597 lines
25 KiB
MQL5
597 lines
25 KiB
MQL5
|
//+------------------------------------------------------------------+
|
||
|
|
//| NetForward.mqh |
|
||
|
|
//| |
|
||
|
|
//| CNet inference and training passes: feedForward, backProp, |
|
||
|
|
//| getResults, logit adjustment. |
|
||
|
|
//| |
|
||
|
|
//| Included from AI\Network.mqh AFTER every class declaration - |
|
||
|
|
//| bodies only, no declarations. Relocation is behaviour-neutral by |
|
||
|
|
//| construction: nothing here is reachable until Network.mqh ends. |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
#ifndef WARRIOR_AI_IMPL_NETFORWARD_MQH
|
||
|
|
#define WARRIOR_AI_IMPL_NETFORWARD_MQH
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
bool CNet::feedForward(CArrayDouble *inputVals)
|
||
|
|
{
|
||
|
|
if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1)
|
||
|
|
return false;
|
||
|
|
//--- Pure-MQL5 inference: OCL-format neurons loaded host-only, computed in double precision. Separate
|
||
|
|
//--- path because the branches below assume either a device backend or the plain CNeuronBase model.
|
||
|
|
if(m_cpuInference)
|
||
|
|
return feedForwardCPU(inputVals);
|
||
|
|
//---
|
||
|
|
CLayer *previous = NULL;
|
||
|
|
CLayer *current = layers.At(0);
|
||
|
|
int total = MathMin(current.Total(), inputVals.Total());
|
||
|
|
CNeuronBase *neuron = NULL;
|
||
|
|
bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID);
|
||
|
|
if(!gpuActive)
|
||
|
|
{
|
||
|
|
for(int i = 0; i < total; i++)
|
||
|
|
{
|
||
|
|
neuron = current.At(i);
|
||
|
|
if(CheckPointer(neuron) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
neuron.setOutputVal(inputVals.At(i));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
CNeuronBaseOCL *neuron_ocl = current.At(0);
|
||
|
|
int total_data = inputVals.Total();
|
||
|
|
bool written;
|
||
|
|
if(CheckPointer(opencl) != POINTER_INVALID)
|
||
|
|
{
|
||
|
|
//--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own
|
||
|
|
//--- BufferWrite(), this call bypasses that class entirely (it writes straight into the
|
||
|
|
//--- first layer's Output buffer by index), so the narrow-to-float has to happen here too.
|
||
|
|
float array[];
|
||
|
|
if(ArrayResize(array, total_data) < 0)
|
||
|
|
return false;
|
||
|
|
for(int d = 0; d < total_data; d++)
|
||
|
|
array[d] = (float)inputVals.At(d);
|
||
|
|
written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
double array[];
|
||
|
|
if(ArrayResize(array, total_data) < 0)
|
||
|
|
return false;
|
||
|
|
for(int d = 0; d < total_data; d++)
|
||
|
|
array[d] = inputVals.At(d);
|
||
|
|
written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data);
|
||
|
|
}
|
||
|
|
if(!written)
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
CObject *temp = NULL;
|
||
|
|
for(int l = 1; l < layers.Total(); l++)
|
||
|
|
{
|
||
|
|
previous = current;
|
||
|
|
current = layers.At(l);
|
||
|
|
if(CheckPointer(current) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
//---
|
||
|
|
if(gpuActive)
|
||
|
|
{
|
||
|
|
CNeuronBaseOCL *current_ocl = current.At(0);
|
||
|
|
if(!current_ocl.feedForward(previous.At(0)))
|
||
|
|
return false;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
total = current.Total();
|
||
|
|
if(current.At(0).Type() == defNeuron)
|
||
|
|
total--;
|
||
|
|
//---
|
||
|
|
for(int n = 0; n < total; n++)
|
||
|
|
{
|
||
|
|
neuron = current.At(n);
|
||
|
|
if(CheckPointer(neuron) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
if(previous.At(0).Type() == defNeuron)
|
||
|
|
{
|
||
|
|
temp = previous;
|
||
|
|
if(!neuron.feedForward(temp))
|
||
|
|
return false;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if(neuron.Type() == defNeuron)
|
||
|
|
{
|
||
|
|
if(n == 0)
|
||
|
|
{
|
||
|
|
CLayer *temp_l = new CLayer(total);
|
||
|
|
if(CheckPointer(temp_l) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
CNeuronPool *Pool = NULL;
|
||
|
|
for(int p = 0; p < previous.Total(); p++)
|
||
|
|
{
|
||
|
|
Pool = previous.At(p);
|
||
|
|
if(CheckPointer(Pool) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
temp_l.AddArray(Pool.getOutputLayer());
|
||
|
|
}
|
||
|
|
temp = temp_l;
|
||
|
|
}
|
||
|
|
if(!neuron.feedForward(temp))
|
||
|
|
return false;
|
||
|
|
if(n == total - 1)
|
||
|
|
{
|
||
|
|
CLayer *temp_l = temp;
|
||
|
|
temp_l.FreeMode(false);
|
||
|
|
temp_l.Shutdown();
|
||
|
|
delete temp_l;
|
||
|
|
}
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
temp = previous.At(n);
|
||
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
if(!neuron.feedForward(temp))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Pure-MQL5 forward pass over an OCL-format network loaded host-only|
|
||
|
|
//| (no OpenCL/DirectML/DLL). Layer 0 is fed from inputVals; each |
|
||
|
|
//| subsequent layer's OCL neuron computes via its virtual |
|
||
|
|
//| feedForwardCPU() (dense/conv/pool/LSTM). See SetCpuInference(). |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
bool CNet::feedForwardCPU(CArrayDouble *inputVals)
|
||
|
|
{
|
||
|
|
CLayer *current = layers.At(0);
|
||
|
|
if(CheckPointer(current) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
CNeuronBaseOCL *in0 = current.At(0);
|
||
|
|
if(CheckPointer(in0) == POINTER_INVALID || !in0.SetInputsCPU(inputVals))
|
||
|
|
return false;
|
||
|
|
for(int l = 1; l < layers.Total(); l++)
|
||
|
|
{
|
||
|
|
CLayer *previous = current;
|
||
|
|
current = layers.At(l);
|
||
|
|
if(CheckPointer(current) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
CNeuronBaseOCL *cur = current.At(0);
|
||
|
|
CNeuronBaseOCL *prev = previous.At(0);
|
||
|
|
if(CheckPointer(cur) == POINTER_INVALID || CheckPointer(prev) == POINTER_INVALID)
|
||
|
|
return false;
|
||
|
|
if(!cur.feedForwardCPU(prev))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
void CNet::backProp(CArrayDouble *targetVals, double sampleWeight)
|
||
|
|
{
|
||
|
|
if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
|
||
|
|
{
|
||
|
|
backPropOCL(targetVals, sampleWeight);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
CLayer *outputLayer = layers.At(layers.Total() - 1);
|
||
|
|
if(CheckPointer(outputLayer) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
//---
|
||
|
|
//--- Defensive: this is the pure-MQL5 CPU backward pass and it walks CNeuron/CNeuronBase objects
|
||
|
|
//--- with SCALAR getOutputVal()/getGradient()/setGradient() accessors. CNeuronBaseOCL & friends do
|
||
|
|
//--- NOT derive from CNeuronBase and expose only ARRAY accessors over a device buffer, so they can
|
||
|
|
//--- never be walked here. That is normally impossible to reach: the early return above hands any
|
||
|
|
//--- backend-backed net to backPropOCL(), and CNet::Create() only ever constructs OCL neurons when
|
||
|
|
//--- a backend exists. The one way to hold OCL neurons with no backend is pure-MQL5 inference mode
|
||
|
|
//--- (SetCpuInference -> CLayer::CreateElementScaled's host-only branch), which is inference-only
|
||
|
|
//--- and never backprops (OnlineLearnStep() guards on Net.CpuInference()). Bail out loudly rather
|
||
|
|
//--- than mis-cast if that ever changes.
|
||
|
|
//--- 2026-07-28: a set of inline "handle OCL neuron types" branches was added throughout this
|
||
|
|
//--- function to cover that impossible case. They could not compile - they called the scalar
|
||
|
|
//--- accessors on CNeuronBaseOCL - and were removed; this guard replaces them.
|
||
|
|
CObject *probe = outputLayer.At(0);
|
||
|
|
if(CheckPointer(probe) != POINTER_INVALID)
|
||
|
|
{
|
||
|
|
int t0 = probe.Type();
|
||
|
|
if(t0 == defNeuronBaseOCL || t0 == defNeuronConvOCL || t0 == defNeuronPoolOCL || t0 == defNeuronLSTMOCL ||
|
||
|
|
t0 == defNeuronBatchNormOCL)
|
||
|
|
{
|
||
|
|
Print(__FUNCTION__ + ": REFUSED - CPU backward pass reached a net built from OpenCL/DirectML neurons with no compute backend attached. Nothing was trained this step.");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
double error = 0.0;
|
||
|
|
int total = outputLayer.Total() - 1;
|
||
|
|
//--- 3-output classification case: true softmax + categorical-cross-entropy gradient
|
||
|
|
//--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section
|
||
|
|
//--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID
|
||
|
|
//--- (bounded, avoids the historical logit-runaway collapse documented at
|
||
|
|
//--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD
|
||
|
|
//--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly,
|
||
|
|
//--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/
|
||
|
|
//--- Neutral together during training: raising one class's softmax probability now structurally
|
||
|
|
//--- lowers the other two's (via the shared normalizing sum), giving real competition instead of
|
||
|
|
//--- three independent binary regressions that can all drift toward "predict Neutral" together -
|
||
|
|
//--- root cause of the "overshoot to all-Neutral" convergence failure this replaces.
|
||
|
|
bool useSoftmaxGrad = (total == 3);
|
||
|
|
double smax[3];
|
||
|
|
if(useSoftmaxGrad)
|
||
|
|
{
|
||
|
|
//--- Logit adjustment added BEFORE the max-subtraction so the shift stays numerically safe.
|
||
|
|
double logit[3];
|
||
|
|
double maxLogit = -DBL_MAX;
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
{
|
||
|
|
CNeuron *nrn = outputLayer.At(n);
|
||
|
|
logit[n] = CLASS_LOGIT_SCALE * nrn.getOutputVal() + (bLogitAdjust ? dLogitAdjust[n] : 0.0);
|
||
|
|
maxLogit = MathMax(maxLogit, logit[n]);
|
||
|
|
}
|
||
|
|
double sum = 0.0;
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
{
|
||
|
|
smax[n] = exp(logit[n] - maxLogit);
|
||
|
|
sum += smax[n];
|
||
|
|
}
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
smax[n] /= sum;
|
||
|
|
}
|
||
|
|
for(int n = 0; n < total && !IsStopped(); n++)
|
||
|
|
{
|
||
|
|
CNeuron *neuron = outputLayer.At(n);
|
||
|
|
double target = targetVals.At(n);
|
||
|
|
double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
|
||
|
|
double delta = clampedTarget - neuron.getOutputVal();
|
||
|
|
error += delta * delta;
|
||
|
|
if(useSoftmaxGrad)
|
||
|
|
neuron.setGradient(clampedTarget - smax[n]);
|
||
|
|
else
|
||
|
|
neuron.calcOutputGradients(targetVals.At(n));
|
||
|
|
//--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
|
||
|
|
//--- sampleWeight is derived) - scales the just-computed output gradient in place, before the
|
||
|
|
//--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted
|
||
|
|
//--- signal without needing its own separate weighting logic.
|
||
|
|
if(sampleWeight != 1.0)
|
||
|
|
neuron.setGradient(neuron.getGradient() * sampleWeight);
|
||
|
|
}
|
||
|
|
error /= total;
|
||
|
|
error = sqrt(error);
|
||
|
|
recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
|
||
|
|
//---
|
||
|
|
CNeuronBase *neuron = NULL;
|
||
|
|
CObject *temp = NULL;
|
||
|
|
for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--)
|
||
|
|
{
|
||
|
|
CLayer *hiddenLayer = layers.At(layerNum);
|
||
|
|
CLayer *nextLayer = layers.At(layerNum + 1);
|
||
|
|
total = hiddenLayer.Total();
|
||
|
|
for(int n = 0; n < total && !IsStopped(); ++n)
|
||
|
|
{
|
||
|
|
neuron = hiddenLayer.At(n);
|
||
|
|
if(nextLayer.At(0).Type() == defNeuron)
|
||
|
|
{
|
||
|
|
temp = nextLayer;
|
||
|
|
neuron.calcHiddenGradients(temp);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if(neuron.Type() == defNeuron)
|
||
|
|
{
|
||
|
|
double g = 0;
|
||
|
|
for(int i = 0; i < nextLayer.Total(); i++)
|
||
|
|
{
|
||
|
|
temp = nextLayer.At(i);
|
||
|
|
neuron.calcHiddenGradients(temp);
|
||
|
|
g += neuron.getGradient();
|
||
|
|
}
|
||
|
|
neuron.setGradient(g);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
temp = nextLayer.At(n);
|
||
|
|
neuron.calcHiddenGradients(temp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--)
|
||
|
|
{
|
||
|
|
CLayer *layer = layers.At(layerNum);
|
||
|
|
CLayer *prevLayer = layers.At(layerNum - 1);
|
||
|
|
total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0);
|
||
|
|
int n_conv = 0;
|
||
|
|
for(int n = 0; n < total && !IsStopped(); n++)
|
||
|
|
{
|
||
|
|
neuron = layer.At(n);
|
||
|
|
if(CheckPointer(neuron) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
if(neuron.Type() == defNeuronPool)
|
||
|
|
continue;
|
||
|
|
switch(prevLayer.At(0).Type())
|
||
|
|
{
|
||
|
|
case defNeuron:
|
||
|
|
temp = prevLayer;
|
||
|
|
neuron.updateInputWeights(temp);
|
||
|
|
break;
|
||
|
|
case defNeuronConv:
|
||
|
|
case defNeuronPool:
|
||
|
|
case defNeuronLSTM:
|
||
|
|
if(neuron.Type() == defNeuron)
|
||
|
|
{
|
||
|
|
for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++)
|
||
|
|
{
|
||
|
|
temp = prevLayer.At(n_conv);
|
||
|
|
neuron.updateInputWeights(temp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
temp = prevLayer.At(n);
|
||
|
|
neuron.updateInputWeights(temp);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
temp = NULL;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight)
|
||
|
|
{
|
||
|
|
if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID ||
|
||
|
|
(CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID))
|
||
|
|
return;
|
||
|
|
CLayer *currentLayer = (CLayer*)layers.At(layers.Total() - 1);
|
||
|
|
if(CheckPointer(currentLayer) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
//---
|
||
|
|
double error = 0.0;
|
||
|
|
int total = targetVals.Total();
|
||
|
|
double result[];
|
||
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)currentLayer.At(0);
|
||
|
|
if(neuron.getOutputVal(result) < total)
|
||
|
|
return;
|
||
|
|
for(int n = 0; n < total && !IsStopped(); n++)
|
||
|
|
{
|
||
|
|
double target = targetVals.At(n);
|
||
|
|
// Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever
|
||
|
|
// target==0, so a one-hot classification target only ever counted the true class's own
|
||
|
|
// error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric,
|
||
|
|
// which is what CNet::backProp()'s CPU-fallback path computes for every output
|
||
|
|
// unconditionally, and is what actually drives the dError<0.1 convergence gate in
|
||
|
|
// ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp
|
||
|
|
// / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was.
|
||
|
|
double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n];
|
||
|
|
error += MathPow(delta, 2);
|
||
|
|
}
|
||
|
|
error /= total;
|
||
|
|
error = sqrt(error);
|
||
|
|
recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
|
||
|
|
if(!neuron.calcOutputGradients(targetVals))
|
||
|
|
return;
|
||
|
|
//--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true
|
||
|
|
//--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5
|
||
|
|
//--- from the raw outputs already read back into result[] above - see the matching CNet::backProp()
|
||
|
|
//--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared
|
||
|
|
//--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels
|
||
|
|
//--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted
|
||
|
|
//--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely
|
||
|
|
//--- on the MQL5 side, so none of the 3 compute backends need to change.
|
||
|
|
if(total == 3)
|
||
|
|
{
|
||
|
|
//--- Logit adjustment (see SetLogitAdjustment): tau*log(prior_c) per class, added to the logit
|
||
|
|
//--- before the softmax. Backward pass ONLY - the forward pass and every inference path stay
|
||
|
|
//--- untouched, which is the whole point: the network learns to absorb the offset, so at
|
||
|
|
//--- inference its RAW argmax is already the balanced-error-optimal decision.
|
||
|
|
double logit[3];
|
||
|
|
double maxLogit = -DBL_MAX;
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
{
|
||
|
|
logit[n] = CLASS_LOGIT_SCALE * result[n] + (bLogitAdjust ? dLogitAdjust[n] : 0.0);
|
||
|
|
maxLogit = MathMax(maxLogit, logit[n]);
|
||
|
|
}
|
||
|
|
double smax[3];
|
||
|
|
double sm = 0.0;
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
{
|
||
|
|
smax[n] = exp(logit[n] - maxLogit);
|
||
|
|
sm += smax[n];
|
||
|
|
}
|
||
|
|
double gradOverwrite[3];
|
||
|
|
for(int n = 0; n < 3; n++)
|
||
|
|
{
|
||
|
|
smax[n] /= sm;
|
||
|
|
double target = targetVals.At(n);
|
||
|
|
double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
|
||
|
|
gradOverwrite[n] = clampedTarget - smax[n];
|
||
|
|
}
|
||
|
|
neuron.setGradient(gradOverwrite);
|
||
|
|
}
|
||
|
|
//--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
|
||
|
|
//--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta
|
||
|
|
//--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the
|
||
|
|
//--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers
|
||
|
|
//--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends.
|
||
|
|
if(sampleWeight != 1.0)
|
||
|
|
{
|
||
|
|
double gradVals[];
|
||
|
|
int gradCount = neuron.getGradient(gradVals);
|
||
|
|
if(gradCount > 0)
|
||
|
|
{
|
||
|
|
for(int g = 0; g < gradCount; g++)
|
||
|
|
gradVals[g] *= sampleWeight;
|
||
|
|
neuron.setGradient(gradVals);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//--- Calc Hidden Gradients
|
||
|
|
CObject *temp = NULL;
|
||
|
|
total = layers.Total();
|
||
|
|
for(int layerNum = total - 2; layerNum > 0; layerNum--)
|
||
|
|
{
|
||
|
|
CLayer *nextLayer = currentLayer;
|
||
|
|
currentLayer = layers.At(layerNum);
|
||
|
|
neuron = currentLayer.At(0);
|
||
|
|
neuron.calcHiddenGradients(nextLayer.At(0));
|
||
|
|
}
|
||
|
|
//--- Layer-1 LSTM special case. The loop above deliberately stops at layerNum > 0 because the INPUT
|
||
|
|
//--- layer needs no gradient of its own - true for every layer type whose updateInputWeights() derives
|
||
|
|
//--- its own weight deltas from (own gradient x previous output) inside the kernel: dense
|
||
|
|
//--- (UpdateWeightsMomentum/Adam) and conv (UpdateWeightsConvMomentum/Adam) both do.
|
||
|
|
//--- CNeuronLSTMOCL is the ONE exception: LSTM_UpdateWeightsMomentum/Adam do not derive anything, they
|
||
|
|
//--- only CONSUME the WeightsGradient buffer, and that buffer is filled purely as a SIDE EFFECT of
|
||
|
|
//--- CNeuronLSTMOCL::calcInputGradients() - which is invoked by the layer BELOW, via its
|
||
|
|
//--- calcHiddenGradients(target=thisLSTM) dispatch. So an LSTM sitting at layer index 1 (the LSTM_2L
|
||
|
|
//--- preset: input -> LSTM -> dense -> dense -> output) never gets calcInputGradients() called at all:
|
||
|
|
//--- WeightsGradient stays at its BufferInit(total, 0) zeros forever, Adam's mt/vt therefore stay 0 and
|
||
|
|
//--- every weight delta is exactly 0. The LSTM silently never trains - it stays a frozen random
|
||
|
|
//--- recurrent projection while only the dense taper above it learns, which shows up as ~0% Buy/Sell
|
||
|
|
//--- recall and a raw-output range that barely moves off its initialisation.
|
||
|
|
//--- HYBRID_2L (input -> conv -> pool -> LSTM -> dense -> dense -> output) is unaffected: its LSTM is at
|
||
|
|
//--- index 3, so the pool layer below it makes the call in the normal course of the loop.
|
||
|
|
//--- Done here rather than by relaxing the loop bound so the loop's currentLayer/nextLayer bookkeeping
|
||
|
|
//--- is untouched, and so no other topology pays for an extra kernel dispatch it does not need.
|
||
|
|
if(total >= 3)
|
||
|
|
{
|
||
|
|
CLayer *firstHidden = layers.At(1);
|
||
|
|
CLayer *inputLayer = layers.At(0);
|
||
|
|
if(CheckPointer(firstHidden) != POINTER_INVALID && CheckPointer(inputLayer) != POINTER_INVALID &&
|
||
|
|
CheckPointer(firstHidden.At(0)) != POINTER_INVALID && CheckPointer(inputLayer.At(0)) != POINTER_INVALID &&
|
||
|
|
firstHidden.At(0).Type() == defNeuronLSTMOCL)
|
||
|
|
{
|
||
|
|
CNeuronLSTMOCL *lstm = firstHidden.At(0);
|
||
|
|
CNeuronBaseOCL *inputNeuron = inputLayer.At(0);
|
||
|
|
//--- Safe to run now and only now: the loop above has already filled this LSTM's own Gradient
|
||
|
|
//--- (it processed layerNum == 1), which is exactly what LSTMGateGradient reads.
|
||
|
|
if(!lstm.calcInputGradients(inputNeuron))
|
||
|
|
printf("%s: LSTM at layer 1 failed to compute its weight gradients - it will not train this step", __FUNCTION__);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
CLayer *prevLayer = layers.At(total - 1);
|
||
|
|
for(int layerNum = total - 1; layerNum > 0; layerNum--)
|
||
|
|
{
|
||
|
|
currentLayer = prevLayer;
|
||
|
|
prevLayer = layers.At(layerNum - 1);
|
||
|
|
neuron = currentLayer.At(0);
|
||
|
|
neuron.updateInputWeights(prevLayer.At(0));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
void CNet::getResults(CArrayDouble *&resultVals)
|
||
|
|
{
|
||
|
|
if(CheckPointer(resultVals) == POINTER_INVALID)
|
||
|
|
{
|
||
|
|
resultVals = new CArrayDouble();
|
||
|
|
if(CheckPointer(resultVals) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
//---
|
||
|
|
resultVals.Clear();
|
||
|
|
if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0)
|
||
|
|
return;
|
||
|
|
//---
|
||
|
|
CLayer *output = layers.At(layers.Total() - 1);
|
||
|
|
if(CheckPointer(output) == POINTER_INVALID)
|
||
|
|
return;
|
||
|
|
//---
|
||
|
|
if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
|
||
|
|
{
|
||
|
|
switch(output.At(0).Type())
|
||
|
|
{
|
||
|
|
case defNeuronBaseOCL:
|
||
|
|
case defNeuronConvOCL:
|
||
|
|
case defNeuronPoolOCL:
|
||
|
|
case defNeuronLSTMOCL:
|
||
|
|
{
|
||
|
|
CNeuronBaseOCL *temp = output.At(0);
|
||
|
|
temp.getOutputVal(resultVals);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//--- pure-MQL5 inference: OCL output neuron computed host-side, read without a device BufferRead.
|
||
|
|
if(m_cpuInference)
|
||
|
|
{
|
||
|
|
switch(output.At(0).Type())
|
||
|
|
{
|
||
|
|
case defNeuronBaseOCL:
|
||
|
|
case defNeuronConvOCL:
|
||
|
|
case defNeuronPoolOCL:
|
||
|
|
case defNeuronLSTMOCL:
|
||
|
|
{
|
||
|
|
CNeuronBaseOCL *temp = output.At(0);
|
||
|
|
temp.GetOutputsCPU(resultVals);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
CNeuronBase *neuron = NULL;
|
||
|
|
CLayer *temp = NULL;
|
||
|
|
int total = output.Total();
|
||
|
|
if(output.At(0).Type() == defNeuron)
|
||
|
|
total--;
|
||
|
|
//---
|
||
|
|
for(int i = 0; i < total; i++)
|
||
|
|
{
|
||
|
|
CObject *obj = output.At(i);
|
||
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
||
|
|
continue;
|
||
|
|
if(obj.Type() == defNeuron)
|
||
|
|
{
|
||
|
|
neuron = (CNeuronBase*)obj;
|
||
|
|
resultVals.Add(neuron.getOutputVal());
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if(obj.Type() == defNeuronPool)
|
||
|
|
{
|
||
|
|
CNeuronPool *n = (CNeuronPool*)obj;
|
||
|
|
temp = n.getOutputLayer();
|
||
|
|
for(int ii = 0; ii < temp.Total(); ii++)
|
||
|
|
{
|
||
|
|
CObject *poolObj = temp.At(ii);
|
||
|
|
if(CheckPointer(poolObj) == POINTER_INVALID)
|
||
|
|
continue;
|
||
|
|
CNeuronBase *poolNeuron = (CNeuronBase*)poolObj;
|
||
|
|
resultVals.Add(poolNeuron.getOutputVal());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Install the per-class logit offsets - see the declaration. |
|
||
|
|
//| Expects tau*log(prior_c) already computed by the caller, which is |
|
||
|
|
//| the only place that knows the training set's class distribution. |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
void CNet::SetLogitAdjustment(const double &offsets[])
|
||
|
|
{
|
||
|
|
if(ArraySize(offsets) < 3)
|
||
|
|
{
|
||
|
|
//--- Refuse rather than half-apply: a partially filled offset vector would silently bias two
|
||
|
|
//--- classes against a third, which is far worse than running unadjusted.
|
||
|
|
bLogitAdjust = false;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
for(int i = 0; i < 3; i++)
|
||
|
|
{
|
||
|
|
//--- Guard against a non-finite offset reaching the softmax (a zero prior would give -inf and
|
||
|
|
//--- turn every gradient into NaN). A class with no examples at all simply gets no push.
|
||
|
|
double v = offsets[i];
|
||
|
|
if(!MathIsValidNumber(v))
|
||
|
|
v = 0.0;
|
||
|
|
dLogitAdjust[i] = v;
|
||
|
|
}
|
||
|
|
bLogitAdjust = true;
|
||
|
|
}
|
||
|
|
#endif
|