3ea54b4f2b
The per-step entry points cannot express a sequence model. CPU_LSTMGates takes
the ENTIRE flattened input as one timestep, and CPU_LSTMGateGradient has no
parameter for dc arriving from the following step - so the recurrent gradient
path does not exist and cannot be assembled from these primitives at any call
pattern. The layer built on them is a gated dense layer that the class comment
already described honestly: "single-timestep-truncated BPTT".
Adds CPU_LSTMSeqForward / CPU_LSTMSeqBackward: the whole unrolled sequence in
one call each, weights shared across timesteps, dW accumulated over all of them
(the per-step CPU_LSTMWeightsGradient assigns rather than accumulates, so it
could not have been reused even with the dc term). Fused rather than dispatched
per step because the recurrence is sequential - T round trips would serialise T
lock/dispatch pairs for a few thousand FLOPs each.
h_{-1} and c_{-1} are zero per sample. The old layer carried its cell state
across forward passes, so under shuffled training every sample inherited the
state of an unrelated one.
DirectML gets the same math host-side (readback, compute in double, upload)
rather than HLSL: the recurrence needs a barrier per timestep, the GPU buffers
are float and BPTT accumulation is where that hurts most, and no D3D12 device
exists on this machine to test a shader against. Documented at the definition.
Verified with lstm_seq_gradcheck.cpp - central-difference check of dW and dX
against an asymmetric loss over the final hidden state. Max relative error
2.3e-10 on both, with a non-trivial gradient magnitude asserted so the check
cannot pass on an all-zero result.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
118 lines
7.4 KiB
C++
118 lines
7.4 KiB
C++
//+------------------------------------------------------------------+
|
|
//| Warrior_EA |
|
|
//| Multithreaded CPU compute fallback - used when neither |
|
|
//| OpenCL nor the D3D12/DirectML tier are available (e.g. |
|
|
//| a VM with no GPU passthrough). Same buffer-handle model |
|
|
//| as WarriorDML.dll but full double precision throughout |
|
|
//| (no GPU float roundtrip) and work is spread across a |
|
|
//| configurable pool of worker threads instead of a device. |
|
|
//+------------------------------------------------------------------+
|
|
// Flat C ABI so MQL5 can #import this DLL directly. Function names and
|
|
// argument order/semantics mirror WarriorDML.h / AI\Network.cl 1:1 so the
|
|
// two backends are interchangeable behind CDirectMLMy in AI\Network.mqh.
|
|
#pragma once
|
|
|
|
#define WARRIORCPU_API extern "C" __declspec(dllexport)
|
|
|
|
// Every exported function (besides CPU_Init/CPU_GetHardwareConcurrency) takes a
|
|
// CpuHandle as its first argument - an opaque pointer to a heap-allocated,
|
|
// self-contained context (its own thread pool, buffer table and mutex) that
|
|
// CPU_Init() allocates and the caller (CDirectMLMy on the MQL5 side) is
|
|
// responsible for remembering and passing back on every subsequent call, then
|
|
// releasing via CPU_Shutdown(). No state is shared between contexts and this
|
|
// DLL keeps no global/static mutable state of its own, so it can be loaded any
|
|
// number of times and driven by any number of instances/threads in parallel -
|
|
// a fault or a wedged call against one context can never poison another
|
|
// context's calls, unlike a process-wide singleton would.
|
|
// A plain `long long` (not C++ `long`, which is only 32 bits on Windows) so it
|
|
// round-trips exactly through MQL5's 64-bit `long` on the #import side.
|
|
typedef long long CpuHandle;
|
|
|
|
// Lifecycle. CPU_Init(threads) always succeeds (returns a non-zero handle)
|
|
// since it needs no hardware - threads<=0 means "use
|
|
// std::thread::hardware_concurrency()". Returns 0 on failure.
|
|
WARRIORCPU_API CpuHandle __stdcall CPU_Init(int threads);
|
|
WARRIORCPU_API void __stdcall CPU_Shutdown(CpuHandle ctx);
|
|
WARRIORCPU_API int __stdcall CPU_GetLastError(CpuHandle ctx);
|
|
WARRIORCPU_API int __stdcall CPU_GetThreadCount(CpuHandle ctx);
|
|
// Stateless: true std::thread::hardware_concurrency(), independent of any
|
|
// context's pool size. Use this (not a CPU_Init(0)/GetThreadCount()/Shutdown()
|
|
// probe) to size a CPU_Init() request.
|
|
WARRIORCPU_API int __stdcall CPU_GetHardwareConcurrency();
|
|
|
|
// Buffer management. Handles are small non-negative integers scoped to ctx;
|
|
// -1 means failure. Buffers are plain double vectors owned by ctx; Write/Read
|
|
// just memcpy in/out, there is no upload/download step on CPU.
|
|
WARRIORCPU_API int __stdcall CPU_BufferCreate(CpuHandle ctx, int elementCount);
|
|
WARRIORCPU_API int __stdcall CPU_BufferWrite(CpuHandle ctx, int handle, const double *data, int count);
|
|
WARRIORCPU_API int __stdcall CPU_BufferRead(CpuHandle ctx, int handle, double *data, int count);
|
|
WARRIORCPU_API void __stdcall CPU_BufferFree(CpuHandle ctx, int handle);
|
|
|
|
// Compute kernels - argument order/semantics mirror AI\Network.cl and
|
|
// WarriorDML.h 1:1. activation: 0 = TANH, 1 = SIGMOID, 2 = PReLU (conv only).
|
|
WARRIORCPU_API int __stdcall CPU_FeedForward(CpuHandle ctx, int wHandle, int iHandle, int oHandle,
|
|
int inputs, int activation);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_CalcOutputGradient(CpuHandle ctx, int tHandle, int oHandle, int igHandle,
|
|
int activation, int count);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_CalcHiddenGradient(CpuHandle ctx, int wHandle, int gHandle, int oHandle, int igHandle,
|
|
int outputs, int activation, int count);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_UpdateWeightsMomentum(CpuHandle ctx, int wHandle, int gHandle, int iHandle, int dwHandle,
|
|
int inputs, double learningRate, double momentumRate, int neurons, int optimizer);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_UpdateWeightsAdam(CpuHandle ctx, int wHandle, int gHandle, int iHandle,
|
|
int mHandle, int vHandle,
|
|
int inputs, double lt, double b1, double b2, int neurons);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_FeedForwardConv(CpuHandle ctx, int wHandle, int iHandle, int oHandle,
|
|
int inputs, int step, int windowIn, int windowOut, int activation, int positions);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_CalcHiddenGradientConv(CpuHandle ctx, int wHandle, int gHandle, int oHandle, int igHandle,
|
|
int outputs, int step, int windowIn, int windowOut, int activation, int inputCount);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_UpdateWeightsConvMomentum(CpuHandle ctx, int wHandle, int gHandle, int iHandle, int dwHandle,
|
|
int inputs, double learningRate, double momentumRate, int windowIn, int windowOut, int step, int optimizer);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_UpdateWeightsConvAdam(CpuHandle ctx, int wHandle, int gHandle, int iHandle,
|
|
int mHandle, int vHandle,
|
|
int inputs, double lt, double b1, double b2, int windowIn, int windowOut, int step);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMGates(CpuHandle ctx, int wHandle, int hiddenPrevHandle, int inputsHandle,
|
|
int concatenatedHandle, int hiddenSize, int inputSize);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMState(CpuHandle ctx, int concatenatedHandle, int memoryHandle, int hiddenPrevHandle,
|
|
int hiddenCacheHandle, int outputHandle, int hiddenSize);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMGateGradient(CpuHandle ctx, int gradientHandle, int memoryHandle, int concatenatedHandle,
|
|
int concatenatedGradientHandle, int hiddenSize);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMWeightsGradient(CpuHandle ctx, int concatenatedGradientHandle, int hiddenCacheHandle,
|
|
int inputsHandle, int weightsGradientHandle, int hiddenSize, int inputSize);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMInputsGradient(CpuHandle ctx, int concatenatedGradientHandle, int wHandle,
|
|
int inputsGradientHandle, int hiddenSize, int inputSize);
|
|
|
|
// Fused unrolled sequence LSTM - see the block comment above the definitions in WarriorCPU.cpp.
|
|
// These replace the per-step entry points above for sequence models: the per-step ones treat the whole
|
|
// input as ONE timestep, and CPU_LSTMGateGradient cannot accept dc from the following step, so real
|
|
// backpropagation-through-time cannot be assembled from them.
|
|
WARRIORCPU_API int __stdcall CPU_LSTMSeqForward(CpuHandle ctx, int wHandle, int inputsHandle,
|
|
int cacheGatesHandle, int cacheCellHandle, int cacheHiddenHandle, int outputHandle,
|
|
int hiddenSize, int stepInputs, int steps);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMSeqBackward(CpuHandle ctx, int wHandle, int inputsHandle,
|
|
int cacheGatesHandle, int cacheCellHandle, int cacheHiddenHandle, int outGradientHandle,
|
|
int weightsGradientHandle, int inputsGradientHandle, int hiddenSize, int stepInputs, int steps);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMUpdateWeightsAdam(CpuHandle ctx, int wHandle, int weightsGradientHandle,
|
|
int mHandle, int vHandle, double l, double b1, double b2, int total);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_LSTMUpdateWeightsMomentum(CpuHandle ctx, int wHandle, int weightsGradientHandle,
|
|
int dwHandle, double learningRate, double momentumRate, int total, int optimizer);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_FeedForwardProof(CpuHandle ctx, int iHandle, int oHandle, int inputs, int window, int step, int outputs);
|
|
|
|
WARRIORCPU_API int __stdcall CPU_CalcInputGradientProof(CpuHandle ctx, int iHandle, int gHandle, int oHandle, int igHandle,
|
|
int outputs, int window, int step, int inputs);
|