geforkt von dng/NN_in_Trading
//+------------------------------------------------------------------+
| |
//| NeuroNet_GraphSparse.mqh |
| |
//| Copyright 2026, DNG |
| |
//| https://www.mql5.com/ru/users/dng |
| |
//+------------------------------------------------------------------+
| |
| |
#ifndef NEURONET_BUILDING_FACADE
| |
#include "NeuroNet.mqh"
| |
#endif // NEURONET_BUILDING_FACADE
| |
//+------------------------------------------------------------------+
| |
//| Logical method implementations generated from NeuroNet.mqh: |
| |
//| GraphSparse. |
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronCGConvOCL layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::Init(uint numOutputs,
| |
uint myIndex,
| |
COpenCLMy * open_cl,
| |
uint window,
| |
uint numNeurons,
| |
ENUM_OPTIMIZATION optimization_type,
| |
uint batch
| |
)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
if(!cInputF.Init(numNeurons, 0, OpenCL, window, optimization, batch))
| |
ReturnFalse;
| |
if(!cInputS.Init(numNeurons, 1, OpenCL, window, optimization, batch))
| |
ReturnFalse;
| |
cInputF.SetActivationFunction(None);
| |
cInputS.SetActivationFunction(None);
| |
//---
| |
if(!cF.Init(0, 2, OpenCL, numNeurons, optimization, batch))
| |
ReturnFalse;
| |
cF.SetActivationFunction(SIGMOID);
| |
if(!cS.Init(0, 3, OpenCL, numNeurons, optimization, batch))
| |
ReturnFalse;
| |
cS.SetActivationFunction(LReLU);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronCGConvOCL feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL)
| |
{
| |
if(!NeuronOCL || !NeuronOCL.getOutput() || NeuronOCL.getOutputIndex() < 0)
| |
ReturnFalse;
| |
//---
| |
if(cInputF.getOutputIndex() != NeuronOCL.getOutputIndex())
| |
{
| |
if(!cInputF.getOutput().BufferSet(NeuronOCL.getOutputIndex()))
| |
ReturnFalse;
| |
cInputF.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation());
| |
}
| |
if(cInputS.getOutputIndex() != NeuronOCL.getOutputIndex())
| |
{
| |
if(!cInputS.getOutput().BufferSet(NeuronOCL.getOutputIndex()))
| |
ReturnFalse;
| |
cInputS.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation());
| |
}
| |
//---
| |
if(!cF.FeedForward(GetPointer(cInputF)))
| |
ReturnFalse;
| |
if(!cS.FeedForward(GetPointer(cInputS)))
| |
ReturnFalse;
| |
//---
| |
if(!ElementMult(cF.getOutput(), cS.getOutput(), Output))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronCGConvOCL |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::calcInputGradients(CNeuronBaseOCL* prevLayer)
| |
{
| |
if(!prevLayer || !prevLayer.getGradient() || prevLayer.getGradientIndex() < 0)
| |
ReturnFalse;
| |
//---
| |
if(!ElementMultGrad(cF.getOutput(), cF.getGradient(), cS.getOutput(), cS.getGradient(), Gradient, cF.Activation(), cS.Activation()))
| |
ReturnFalse;
| |
//---
| |
if(!cInputF.CalcHiddenGradients((CObject *)GetPointer(cF)))
| |
ReturnFalse;
| |
if(!cInputS.CalcHiddenGradients((CObject *)GetPointer(cS)))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cF.getOutput(), cS.getOutput(), prevLayer.getOutput(), 1, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronCGConvOCL trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
| |
{
| |
if(!cF.UpdateInputWeights(cInputF.AsObject()))
| |
ReturnFalse;
| |
if(!cS.UpdateInputWeights(cInputS.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronCGConvOCL state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(!cInputF.Save(file_handle))
| |
ReturnFalse;
| |
if(!cInputS.Save(file_handle))
| |
ReturnFalse;
| |
if(!cF.Save(file_handle))
| |
ReturnFalse;
| |
if(!cS.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronCGConvOCL state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(FileReadInteger(file_handle) != cInputF.Type() || !cInputF.Load(file_handle))
| |
ReturnFalse;
| |
if(FileReadInteger(file_handle) != cInputS.Type() || !cInputS.Load(file_handle))
| |
ReturnFalse;
| |
if(FileReadInteger(file_handle) != cF.Type() || !cF.Load(file_handle))
| |
ReturnFalse;
| |
if(FileReadInteger(file_handle) != cS.Type() || !cS.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronCGConvOCL weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronCGConvOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
if(!cInputF.WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
if(!cInputS.WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
if(!cF.WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
if(!cS.WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronCGConvOCL GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronCGConvOCL::SetOpenCL(COpenCLMy * obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cInputF.SetOpenCL(OpenCL);
| |
cInputS.SetOpenCL(OpenCL);
| |
cF.SetOpenCL(OpenCL);
| |
cS.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronAGCN layer and allocates its GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint dimension,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
| |
units_count * dimension, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cEa.SetActivationFunction(None);
| |
index++;
| |
if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cWconcat_ex.SetActivationFunction(None);
| |
index++;
| |
if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cEn.SetActivationFunction(None);
| |
index++;
| |
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(GELU);
| |
index++;
| |
if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cAadapt.SetHeads(units_count);
| |
index++;
| |
if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cAadaptX.SetActivationFunction(None);
| |
index++;
| |
if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cApreX.SetActivationFunction(None);
| |
index++;
| |
if(!cWadapt.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cWadapt.SetActivationFunction(None);
| |
index++;
| |
if(!cWpre.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cWpre.SetActivationFunction(None);
| |
SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronAGCN feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
| |
{
| |
if(!SecondInput || SecondInput.Total() < cAadapt.Neurons())
| |
ReturnFalse;
| |
//---
| |
if(bTrain)
| |
{
| |
if(!cEa.FeedForward())
| |
ReturnFalse;
| |
if(!cWe.FeedForward(cEa.AsObject()))
| |
ReturnFalse;
| |
}
| |
//---
| |
if(!cWx.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
| |
ReturnFalse;
| |
if(!cEn.FeedForward(cWconcat_ex.AsObject()))
| |
ReturnFalse;
| |
if(!cEnT.FeedForward(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount()))
| |
ReturnFalse;
| |
if(cEnEnT.Activation() != None)
| |
if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation()))
| |
ReturnFalse;
| |
if(!cAadapt.FeedForward(cEnEnT.AsObject()))
| |
ReturnFalse;
| |
if(!IdentSum(cAadapt.getOutput(), cAadapt.getOutput(), cAadapt.Heads()))
| |
ReturnFalse;
| |
if(!MatMul(SecondInput, NeuronOCL.getOutput(), cApreX.getOutput(),
| |
cWpre.GetUnits(), cWpre.GetUnits(), cWpre.GetWindow()))
| |
ReturnFalse;
| |
if(!MatMul(cAadapt.getOutput(), NeuronOCL.getOutput(), cAadaptX.getOutput(),
| |
cWadapt.GetUnits(), cWadapt.GetUnits(), cWadapt.GetWindow()))
| |
ReturnFalse;
| |
if(!cWpre.FeedForward(cApreX.AsObject()))
| |
ReturnFalse;
| |
if(!cWadapt.FeedForward(cAadaptX.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cWadapt.getOutput(), cWpre.getOutput(), Output, cWadapt.GetWindowOut(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronAGCN |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
| |
CBufferFloat *SecondInput,
| |
CBufferFloat *SecondGradient,
| |
ENUM_ACTIVATION SecondActivation = None)
| |
{
| |
if(!NeuronOCL || !SecondInput || !SecondGradient ||
| |
SecondInput.Total() < cAadapt.Neurons() ||
| |
SecondGradient.Total() < cAadapt.Neurons())
| |
ReturnFalse;
| |
//---
| |
if(!DeActivation(cWadapt.getOutput(), cWadapt.getGradient(), Gradient, cWadapt.Activation()))
| |
ReturnFalse;
| |
if(!DeActivation(cWpre.getOutput(), cWpre.getGradient(), Gradient, cWpre.Activation()))
| |
ReturnFalse;
| |
if(!cApreX.CalcHiddenGradients(cWpre.AsObject()))
| |
ReturnFalse;
| |
if(!cAadaptX.CalcHiddenGradients(cWadapt.AsObject()))
| |
ReturnFalse;
| |
if(!MatMulGrad(SecondInput, SecondGradient,
| |
NeuronOCL.getOutput(), NeuronOCL.getGradient(),
| |
cApreX.getGradient(), cWpre.GetUnits(),
| |
cWpre.GetUnits(), cWpre.GetWindow()))
| |
ReturnFalse;
| |
if(SecondActivation != None)
| |
if(!DeActivation(SecondInput, SecondGradient,
| |
SecondGradient, SecondActivation))
| |
ReturnFalse;
| |
//---
| |
if(!MatMulGrad(cAadapt.getOutput(), cAadapt.getGradient(),
| |
NeuronOCL.getOutput(), PrevOutput,
| |
cAadaptX.getGradient(), cWadapt.GetUnits(),
| |
cWadapt.GetUnits(), cWadapt.GetWindow()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
| |
PrevOutput, cWx.GetWindow(), false))
| |
ReturnFalse;
| |
if(NeuronOCL.Activation() != None)
| |
if(!DeActivation(NeuronOCL.getOutput(), PrevOutput,
| |
PrevOutput, NeuronOCL.Activation()))
| |
ReturnFalse;
| |
if(!cEnEnT.CalcHiddenGradients(cAadapt.AsObject()))
| |
ReturnFalse;
| |
if(cEnEnT.Activation() != None)
| |
if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(),
| |
cEnEnT.getGradient(), cEnEnT.Activation()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
| |
cEnT.getOutput(), cEnT.getGradient(),
| |
cEnEnT.getGradient(), cEnT.GetCount(),
| |
cEnT.GetWindow(), cEnT.GetCount()))
| |
ReturnFalse;
| |
if(!cEn.CalcHiddenGradients(cEnT.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(),
| |
cEn.getGradient(), cEnT.GetWindow(), false))
| |
ReturnFalse;
| |
if(cEn.Activation() != None)
| |
if(!DeActivation(cEn.getOutput(), cEn.getGradient(),
| |
cEn.getGradient(), cEn.Activation()))
| |
ReturnFalse;
| |
if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(),
| |
cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
| |
ReturnFalse;
| |
if(cWx.Activation() != None)
| |
if(!DeActivation(cWx.getOutput(), cWx.getGradient(),
| |
cWx.getGradient(), cWx.Activation()))
| |
ReturnFalse;
| |
if(cWe.Activation() != None)
| |
if(!DeActivation(cWe.getOutput(), cWe.getGradient(),
| |
cWe.getGradient(), cWe.Activation()))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
| |
NeuronOCL.getGradient(), cWx.GetWindow(), false))
| |
ReturnFalse;
| |
if(!cEa.CalcHiddenGradients(cWe.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronAGCN trainable weights from the accumulated |
| |
//| gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)
| |
{
| |
if(!cEa.UpdateInputWeights())
| |
ReturnFalse;
| |
if(!cWe.UpdateInputWeights(cEa.AsObject()))
| |
ReturnFalse;
| |
if(!cWx.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject()))
| |
ReturnFalse;
| |
if(!cWpre.UpdateInputWeights(cApreX.AsObject()))
| |
ReturnFalse;
| |
if(!cWadapt.UpdateInputWeights(cAadaptX.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronAGCN weights toward the source model with |
| |
//| factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronAGCN* Source = source;
| |
if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWpre.WeightsUpdate(Source.cWpre.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWadapt.WeightsUpdate(Source.cWadapt.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronAGCN state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(!cEa.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWe.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWx.Save(file_handle))
| |
ReturnFalse;
| |
if(!cEn.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWpre.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWadapt.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronAGCN state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAGCN::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEa.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWe.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWx.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWpre.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWadapt.AsObject()))
| |
ReturnFalse;
| |
//---
| |
uint units_count = GetUnits();
| |
uint dimension = GetWindow();
| |
//---
| |
int index = 3;
| |
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cWconcat_ex.SetActivationFunction(None);
| |
index += 2;
| |
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(GELU);
| |
index++;
| |
if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cAadapt.SetHeads(units_count);
| |
index++;
| |
if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cAadaptX.SetActivationFunction(None);
| |
index++;
| |
if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cApreX.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronAGCN GPU buffers to the given OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronAGCN::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cEa.SetOpenCL(OpenCL);
| |
cWe.SetOpenCL(OpenCL);
| |
cWx.SetOpenCL(OpenCL);
| |
cEn.SetOpenCL(OpenCL);
| |
cWpre.SetOpenCL(OpenCL);
| |
cWadapt.SetOpenCL(OpenCL);
| |
cWconcat_ex.SetOpenCL(OpenCL);
| |
cEnT.SetOpenCL(OpenCL);
| |
cEnEnT.SetOpenCL(OpenCL);
| |
cAadapt.SetOpenCL(OpenCL);
| |
cAadaptX.SetOpenCL(OpenCL);
| |
cApreX.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronAGCN layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronAGCN::TrainMode(bool flag)
| |
{
| |
CNeuronBaseOCL::TrainMode(flag);
| |
cEa.TrainMode(bTrain);
| |
cWe.TrainMode(bTrain);
| |
cWx.TrainMode(bTrain);
| |
cEn.TrainMode(bTrain);
| |
cWpre.TrainMode(bTrain);
| |
cWadapt.TrainMode(bTrain);
| |
cWconcat_ex.TrainMode(bTrain);
| |
cEnT.TrainMode(bTrain);
| |
cEnEnT.TrainMode(bTrain);
| |
cAadapt.TrainMode(bTrain);
| |
cAadaptX.TrainMode(bTrain);
| |
cApreX.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronGinARCell layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint dimension,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
| |
units_count * dimension, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cX_IA.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cX_AGCN.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cForgetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cResetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cContext.Init(0, index, OpenCL, units_count * dimension, optimization, iBatch))
| |
ReturnFalse;
| |
if(!cContext.getPrevOutput().Fill(0))
| |
ReturnFalse;
| |
cContext.SetActivationFunction(None);
| |
bTemp.BufferFree();
| |
if(!bTemp.BufferInit(units_count * units_count, 0) ||
| |
!bTemp.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronGinARCell feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
| |
{
| |
if(!cX_IA.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cX_AGCN.FeedForward(cX_IA.AsObject(), SecondInput))
| |
ReturnFalse;
| |
if(!cForgetGate.FeedForward(cX_IA.AsObject(), SecondInput))
| |
ReturnFalse;
| |
if(!cResetGate.FeedForward(cX_IA.AsObject(), SecondInput))
| |
ReturnFalse;
| |
if(!Activation(cForgetGate.getOutput(), cForgetGate.getOutput(), GELU))
| |
ReturnFalse;
| |
if(!Activation(cResetGate.getOutput(), cResetGate.getOutput(), GELU))
| |
ReturnFalse;
| |
//--- Context
| |
if(!cContext.SwapOutputs())
| |
ReturnFalse;
| |
if(!GateElementMult(cContext.getPrevOutput(), cX_AGCN.getOutput(), cForgetGate.getOutput(), cContext.getOutput()))
| |
ReturnFalse;
| |
//--- Output
| |
if(!Activation(cContext.getOutput(), cContext.getPrevOutput(), ELU))
| |
ReturnFalse;
| |
if(!GateElementMult(cContext.getPrevOutput(), cX_IA.getOutput(), cResetGate.getOutput(), Output))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronGinARCell |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
| |
CBufferFloat *SecondInput,
| |
CBufferFloat *SecondGradient,
| |
ENUM_ACTIVATION SecondActivation = None)
| |
{
| |
if(!NeuronOCL || !SecondInput || !SecondGradient)
| |
ReturnFalse;
| |
//--- Output
| |
if(!GateElementMultGrad(cContext.getPrevOutput(), cContext.getGradient(),
| |
cX_IA.getOutput(), cX_IA.getPrevOutput(),
| |
cResetGate.getOutput(), cResetGate.getGradient(),
| |
Gradient, ELU, cX_IA.Activation(), GELU))
| |
ReturnFalse;
| |
//--- Context
| |
if(!GateElementMultGrad(cContext.getOutput(), cContext.getPrevOutput(),
| |
cX_AGCN.getOutput(), cX_AGCN.getGradient(),
| |
cForgetGate.getOutput(), cForgetGate.getGradient(),
| |
cContext.getGradient(), None,
| |
cX_AGCN.Activation(), GELU))
| |
ReturnFalse;
| |
//--- Gradient to Interposition Attention
| |
if(!cX_IA.CalcHiddenGradients(cX_AGCN.AsObject(), SecondInput, SecondGradient, SecondActivation) ||
| |
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!cX_IA.CalcHiddenGradients(cForgetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) ||
| |
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) ||
| |
!SumAndNormalize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!cX_IA.CalcHiddenGradients(cResetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) ||
| |
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) ||
| |
!SumAndNormalize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
if(!NeuronOCL.CalcHiddenGradients(cX_IA.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronGinARCell trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)
| |
{
| |
if(!cX_IA.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cX_AGCN.UpdateInputWeights(cX_IA.AsObject(), second))
| |
ReturnFalse;
| |
if(!cForgetGate.UpdateInputWeights(cX_IA.AsObject(), second))
| |
ReturnFalse;
| |
if(!cResetGate.UpdateInputWeights(cX_IA.AsObject(), second))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronGinARCell weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronGinARCell* Source = source;
| |
if(!cX_IA.WeightsUpdate(Source.cX_IA.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cX_AGCN.WeightsUpdate(Source.cX_AGCN.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cForgetGate.WeightsUpdate(Source.cForgetGate.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cResetGate.WeightsUpdate(Source.cResetGate.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronGinARCell state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(!cX_IA.Save(file_handle))
| |
ReturnFalse;
| |
if(!cX_AGCN.Save(file_handle))
| |
ReturnFalse;
| |
if(!cForgetGate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cResetGate.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronGinARCell state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cX_IA.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cX_AGCN.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cForgetGate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cResetGate.AsObject()))
| |
ReturnFalse;
| |
//---
| |
int index = 4;
| |
if(!cContext.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
if(!cContext.getPrevOutput().Fill(0))
| |
ReturnFalse;
| |
cContext.SetActivationFunction(None);
| |
bTemp.BufferFree();
| |
if(!bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0) ||
| |
!bTemp.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronGinARCell GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGinARCell::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cX_IA.SetOpenCL(OpenCL);
| |
cX_AGCN.SetOpenCL(OpenCL);
| |
cForgetGate.SetOpenCL(OpenCL);
| |
cResetGate.SetOpenCL(OpenCL);
| |
cContext.SetOpenCL(OpenCL);
| |
bTemp.BufferFree();
| |
bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0);
| |
bTemp.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronGinARCell layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGinARCell::TrainMode(bool flag)
| |
{
| |
CNeuronBaseOCL::TrainMode(flag);
| |
cX_IA.TrainMode(bTrain);
| |
cX_AGCN.TrainMode(bTrain);
| |
cForgetGate.TrainMode(bTrain);
| |
cResetGate.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronGinARCell execution state and its GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinARCell::Clear(void)
| |
{
| |
if(!CNeuronBaseOCL::Clear())
| |
ReturnFalse;
| |
if(!cContext.getPrevOutput() ||
| |
!cContext.getPrevOutput().Fill(0))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronGinAR layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint dimension,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronSwiGLUOCL::Init(numOutputs, myIndex, open_cl, caCells.Size()*dimension,
| |
caCells.Size()*dimension, dimension, units_count, 1,
| |
optimization_type, batch))
| |
ReturnFalse;
| |
int index = 0;
| |
if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cEa.SetActivationFunction(None);
| |
index++;
| |
if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cWconcat_ex.SetActivationFunction(None);
| |
index++;
| |
if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cEn.SetActivationFunction(None);
| |
index++;
| |
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(GELU);
| |
index++;
| |
if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cApre.SetHeads(units_count);
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
{
| |
index++;
| |
if(!caCells[i].Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
}
| |
index++;
| |
if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronGinAR feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
//--- Calculate Apre
| |
if(bTrain)
| |
{
| |
if(!cEa.FeedForward())
| |
ReturnFalse;
| |
if(!cWe.FeedForward(cEa.AsObject()))
| |
ReturnFalse;
| |
}
| |
//---
| |
if(!cWx.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
| |
ReturnFalse;
| |
if(!cEn.FeedForward(cWconcat_ex.AsObject()))
| |
ReturnFalse;
| |
if(!cEnT.FeedForward(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount()))
| |
ReturnFalse;
| |
if(cEnEnT.Activation() != None)
| |
if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation()))
| |
ReturnFalse;
| |
if(!cApre.FeedForward(cEnEnT.AsObject()))
| |
ReturnFalse;
| |
if(!IdentSum(cApre.getOutput(), cApre.getOutput(), cApre.Heads()))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
CNeuronBaseOCL *temp = NeuronOCL;
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
{
| |
if(!caCells[i].FeedForward(temp, cApre.getOutput()))
| |
ReturnFalse;
| |
temp = caCells[i].AsObject();
| |
}
| |
//---
| |
if(!Concat(caCells[0].getOutput(), caCells[1].getOutput(), caCells[2].getOutput(), caCells[3].getOutput(),
| |
cConcat.getOutput(), GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetUnits()))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronSwiGLUOCL::feedForward(cConcat.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronGinAR |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronSwiGLUOCL::calcInputGradients(cConcat.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(caCells[0].getPrevOutput(), caCells[1].getPrevOutput(),
| |
caCells[2].getPrevOutput(), caCells[3].getGradient(),
| |
cConcat.getGradient(), GetWindow() / 4,
| |
GetWindow() / 4, GetWindow() / 4,
| |
GetWindow() / 4, GetUnits()))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
cApre.getGradient().Fill(0);
| |
for(uint i = caCells.Size() - 1; i > 0; i--)
| |
if(!caCells[i - 1].CalcHiddenGradients(caCells[i].AsObject(), cApre.getOutput(),
| |
cApre.getPrevOutput(),
| |
(ENUM_ACTIVATION)cApre.Activation()) ||
| |
!SumAndNormalize(caCells[i - 1].getGradient(), caCells[i - 1].getPrevOutput(),
| |
caCells[i - 1].getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1) ||
| |
!SumAndNormalize(cApre.getGradient(), cApre.getPrevOutput(),
| |
cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(caCells[0].AsObject(), cApre.getOutput(),
| |
cApre.getPrevOutput(), (ENUM_ACTIVATION)cApre.Activation()) ||
| |
!SumAndNormalize(cApre.getGradient(), cApre.getPrevOutput(),
| |
cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
if(!cEnEnT.CalcHiddenGradients(cApre.AsObject()))
| |
ReturnFalse;
| |
if(cEnEnT.Activation() != None)
| |
if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(),
| |
cEnEnT.getGradient(), cEnEnT.Activation()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
| |
cEnT.getOutput(), cEnT.getGradient(),
| |
cEnEnT.getGradient(), cEnT.GetCount(),
| |
cEnT.GetWindow(), cEnT.GetCount()))
| |
ReturnFalse;
| |
if(!cEn.CalcHiddenGradients(cEnT.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(),
| |
cEn.getGradient(), cEnT.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(cEn.Activation() != None)
| |
if(!DeActivation(cEn.getOutput(), cEn.getGradient(),
| |
cEn.getGradient(), cEn.Activation()))
| |
ReturnFalse;
| |
if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(),
| |
cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
| |
ReturnFalse;
| |
if(cWx.Activation() != None)
| |
if(!DeActivation(cWx.getOutput(), cWx.getGradient(),
| |
cWx.getGradient(), cWx.Activation()))
| |
ReturnFalse;
| |
if(cWe.Activation() != None)
| |
if(!DeActivation(cWe.getOutput(), cWe.getGradient(),
| |
cWe.getGradient(), cWe.Activation()))
| |
ReturnFalse;
| |
CBufferFloat* temp = NeuronOCL.getGradient();
| |
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), temp,
| |
NeuronOCL.getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!NeuronOCL.SetGradient(temp, false))
| |
ReturnFalse;
| |
if(!cEa.CalcHiddenGradients(cWe.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronGinAR trainable weights from the accumulated |
| |
//| gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
//--- Apre
| |
if(!cEa.UpdateInputWeights())
| |
ReturnFalse;
| |
if(!cWe.UpdateInputWeights(cEa.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(!cWx.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject()))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
CNeuronBaseOCL *temp = NeuronOCL;
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
{
| |
if(!caCells[i].UpdateInputWeights(temp, cApre.getOutput()))
| |
ReturnFalse;
| |
temp = caCells[i].AsObject();
| |
}
| |
//---
| |
return(CNeuronSwiGLUOCL::updateInputWeights(cConcat.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronGinAR weights toward the source model with |
| |
//| factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronSwiGLUOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronGinAR* Source = source;
| |
//--- Apre
| |
if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
if(!caCells[i].WeightsUpdate(Source.caCells[i].AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronGinAR state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::Save(const int file_handle)
| |
{
| |
if(!CNeuronSwiGLUOCL::Save(file_handle))
| |
ReturnFalse;
| |
//--- Apre
| |
if(!cEa.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWe.Save(file_handle))
| |
ReturnFalse;
| |
if(!cWx.Save(file_handle))
| |
ReturnFalse;
| |
if(!cEn.Save(file_handle))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
if(!caCells[i].Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronGinAR state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::Load(const int file_handle)
| |
{
| |
if(!CNeuronSwiGLUOCL::Load(file_handle))
| |
ReturnFalse;
| |
//--- Apre
| |
if(!LoadInsideLayer(file_handle, cEa.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWe.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cWx.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
| |
ReturnFalse;
| |
//--- GimAR Cells
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
if(!LoadInsideLayer(file_handle, caCells[i].AsObject()))
| |
ReturnFalse;
| |
//---
| |
uint units_count = GetUnits();
| |
uint dimension = cWx.GetWindow();
| |
uint index = 3;
| |
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cWconcat_ex.SetActivationFunction(None);
| |
index += 2;
| |
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(GELU);
| |
index++;
| |
if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
| |
ReturnFalse;
| |
cApre.SetHeads(units_count);
| |
index += caCells.Size();
| |
if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronGinAR GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGinAR::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronSwiGLUOCL::SetOpenCL(obj);
| |
//--- Apre
| |
cEa.SetOpenCL(OpenCL);
| |
cWe.SetOpenCL(OpenCL);
| |
cWx.SetOpenCL(OpenCL);
| |
cEn.SetOpenCL(OpenCL);
| |
//--- GimAR Cells
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
caCells[i].SetOpenCL(OpenCL);
| |
//---
| |
cWconcat_ex.SetOpenCL(OpenCL);
| |
cEnT.SetOpenCL(OpenCL);
| |
cEnEnT.SetOpenCL(OpenCL);
| |
cApre.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronGinAR layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGinAR::TrainMode(bool flag)
| |
{
| |
CNeuronSwiGLUOCL::TrainMode(flag);
| |
//--- Apre
| |
cEa.TrainMode(bTrain);
| |
cWe.TrainMode(bTrain);
| |
cWx.TrainMode(bTrain);
| |
cEn.TrainMode(bTrain);
| |
//--- GimAR Cells
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
caCells[i].TrainMode(bTrain);
| |
//---
| |
cWconcat_ex.TrainMode(bTrain);
| |
cEnT.TrainMode(bTrain);
| |
cEnEnT.TrainMode(bTrain);
| |
cApre.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronGinAR execution state and its GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGinAR::Clear(void)
| |
{
| |
if(!CNeuronSwiGLUOCL::Clear())
| |
ReturnFalse;
| |
for(uint i = 0; i < caCells.Size(); i++)
| |
if(!caCells[i].Clear())
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronAdaptSpatialNorm layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint variables,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * variables, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
iVariables = variables;
| |
iCount = units_count;
| |
//---
| |
uint dimension = (iVariables + 1) / 2;
| |
uint index = 0;
| |
if(!cEn.Init(0, index, OpenCL, iVariables * dimension, optimization, iBatch))
| |
ReturnFalse;
| |
cEn.SetActivationFunction(None);
| |
index++;
| |
if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
cEnT.SetActivationFunction(None);
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(None);
| |
index++;
| |
if(!cAttan.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
| |
ReturnFalse;
| |
cAttan.SetHeads(iVariables);
| |
//---
| |
index++;
| |
if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cMeanSTDevs.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronAdaptSpatialNorm adapt spatial norm |
| |
//| operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::AdaptSpatialNorm(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!OpenCL || !NeuronOCL)
| |
ReturnFalse;
| |
uint global_work_offset[3] = { 0 };
| |
uint global_work_size[] = { iCount, MathMin(iVariables, uint(OpenCL.GetMaxLocalSize(1))), iVariables};
| |
uint local_work_size[] = { 1, global_work_size[1], 1};
| |
//---
| |
uint kernel = def_k_AdaptSpatialNorm;
| |
setBuffer(kernel, def_k_asn_inputs, NeuronOCL.getOutputIndex())
| |
setBuffer(kernel, def_k_asn_mean_stdevs, cMeanSTDevs.getOutputIndex())
| |
setBuffer(kernel, def_k_asn_attention, cAttan.getOutputIndex())
| |
setBuffer(kernel, def_k_asn_outputs, getOutputIndex())
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!Output.BufferRead())
| |
ReturnFalse;
| |
#endif
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronAdaptSpatialNorm adapt spatial norm grad |
| |
//| operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::AdaptSpatialNormGrad(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!OpenCL || !NeuronOCL)
| |
ReturnFalse;
| |
uint global_work_offset[3] = { 0 };
| |
uint global_work_size[] = { MathMax(iVariables, iCount), MathMin(MathMax(iVariables, iCount), uint(OpenCL.GetMaxLocalSize(1))), iVariables};
| |
uint local_work_size[] = { 1, global_work_size[1], 1};
| |
//---
| |
uint kernel = def_k_AdaptSpatialNormGrad;
| |
setBuffer(kernel, def_k_asng_inputs, NeuronOCL.getOutputIndex())
| |
setBuffer(kernel, def_k_asng_inputs_gr, NeuronOCL.getGradientIndex())
| |
setBuffer(kernel, def_k_asng_mean_stdevs, cMeanSTDevs.getOutputIndex())
| |
setBuffer(kernel, def_k_asng_mean_stdevs_gr, cMeanSTDevs.getGradientIndex())
| |
setBuffer(kernel, def_k_asng_attention, cAttan.getOutputIndex())
| |
setBuffer(kernel, def_k_asng_attention_gr, cAttan.getGradientIndex())
| |
setBuffer(kernel, def_k_asng_outputs_gr, getGradientIndex())
| |
setArgument(kernel, def_k_asng_total_inputs, iCount)
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!NeuronOCL.getGradient().BufferRead())
| |
ReturnFalse;
| |
#endif
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronAdaptSpatialNorm feed-forward pass and writes |
| |
//| the layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(bTrain)
| |
{
| |
if(!cEn.FeedForward())
| |
ReturnFalse;
| |
if(!cEnT.FeedForward(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(),
| |
iVariables, cEnT.GetWindow(), iVariables, 1, false))
| |
ReturnFalse;
| |
if(!cAttan.FeedForward(cEnEnT.AsObject()))
| |
ReturnFalse;
| |
}
| |
//---
| |
return(AdaptSpatialNorm(NeuronOCL));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronAdaptSpatialNorm layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!AdaptSpatialNormGrad(NeuronOCL))
| |
ReturnFalse;
| |
if(!cEnEnT.CalcHiddenGradients(cAttan.AsObject()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
| |
cEnT.getOutput(), cEnT.getGradient(),
| |
cEnEnT.getGradient(), iVariables,
| |
cEnT.GetWindow(), iVariables, 1, false))
| |
ReturnFalse;
| |
if(!cEn.CalcHiddenGradients(cEnT.AsObject()) ||
| |
!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(), cEn.getGradient(),
| |
cEnT.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(cEn.Activation() != None)
| |
if(!DeActivation(cEn.getOutput(), cEn.getGradient(), cEn.getGradient(), cEn.Activation()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronAdaptSpatialNorm trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
return(cEn.UpdateInputWeights());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronAdaptSpatialNorm state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(!cEn.Save(file_handle))
| |
ReturnFalse;
| |
if(!cAttan.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronAdaptSpatialNorm state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cAttan.AsObject()))
| |
ReturnFalse;
| |
//---
| |
iVariables = cAttan.Heads();
| |
iCount = Neurons() / iVariables;
| |
uint dimension = cEn.Neurons() / iVariables;
| |
//---
| |
uint index = 1;
| |
if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch))
| |
ReturnFalse;
| |
cEnT.SetActivationFunction(None);
| |
index++;
| |
if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
| |
ReturnFalse;
| |
cEnEnT.SetActivationFunction(None);
| |
index += 2;
| |
if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cMeanSTDevs.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronAdaptSpatialNorm GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronAdaptSpatialNorm::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cEn.SetOpenCL(OpenCL);
| |
cEnT.SetOpenCL(OpenCL);
| |
cEnEnT.SetOpenCL(OpenCL);
| |
cAttan.SetOpenCL(OpenCL);
| |
cMeanSTDevs.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronAdaptSpatialNorm weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronAdaptSpatialNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronAdaptSpatialNorm* Source = source;
| |
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
if(!cEn.FeedForward())
| |
ReturnFalse;
| |
if(!cEnT.FeedForward(cEn.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(),
| |
iVariables, cEnT.GetWindow(), iVariables, 1, false))
| |
ReturnFalse;
| |
if(!cAttan.FeedForward(cEnEnT.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSCNNEncoder layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint variables, uint forecast,
| |
uint season_period, uint short_period,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast, variables, optimization_type, batch))
| |
ReturnFalse;
| |
SetActivationFunction(None);
| |
//---
| |
uint index = 0;
| |
if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, variables, optimization, iBatch))
| |
ReturnFalse;
| |
cLongNorm.SetActivationFunction(None);
| |
index++;
| |
if(!cSeasonTransp.Init(0, index, OpenCL, variables, units_count / season_period, season_period, optimization, iBatch))
| |
ReturnFalse;
| |
cSeasonTransp.SetActivationFunction(None);
| |
index++;
| |
if(!cSeasonNorm.Init(0, index, OpenCL, cSeasonTransp.GetCount(), season_period, variables, optimization, iBatch))
| |
ReturnFalse;
| |
cSeasonNorm.SetActivationFunction(None);
| |
index++;
| |
if(!cUnSeasonTransp.Init(0, index, OpenCL, variables, season_period, cSeasonTransp.GetCount(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, variables, optimization, iBatch))
| |
ReturnFalse;
| |
cSeasonNorm.SetActivationFunction(None);
| |
index++;
| |
if(!cAdaptSpatNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch))
| |
ReturnFalse;
| |
cAdaptSpatNorm.SetActivationFunction(None);
| |
index++;
| |
//--- inputs
| |
uint concatSize = units_count * variables;
| |
//--- long term
| |
concatSize += cLongNorm.Neurons() + cLongNorm.GetMeanSTDevs().Neurons();
| |
//--- seasons
| |
concatSize += cSeasonNorm.Neurons() + cSeasonNorm.GetMeanSTDevs().Neurons();
| |
//--- short term
| |
concatSize += cShortNorm.Neurons() + cShortNorm.GetMeanSTDevs().Neurons();
| |
//--- spatial
| |
concatSize += cAdaptSpatNorm.Neurons() + cAdaptSpatNorm.GetMeanSTDevs().Neurons();
| |
if(!cConcatenated.Init(0, index, OpenCL, concatSize, optimization, iBatch))
| |
ReturnFalse;
| |
cConcatenated.SetActivationFunction(None);
| |
index++;
| |
if(!cProjection.Init(0, index, OpenCL, concatSize / variables, concatSize / variables, units_count + forecast, 1, variables, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cTranspose.Init(0, index, OpenCL, variables, units_count + forecast, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!caFusion[0].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch))
| |
ReturnFalse;
| |
caFusion[0].SetActivationFunction(TANH);
| |
index++;
| |
if(!caFusion[1].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch))
| |
ReturnFalse;
| |
caFusion[1].SetActivationFunction(SIGMOID);
| |
index++;
| |
if(!cFusionOut.Init(0, index, OpenCL, caFusion[0].Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSCNNEncoder feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cLongNorm.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cSeasonTransp.FeedForward(cLongNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cAdaptSpatNorm.FeedForward(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
uint windows[3] = {NeuronOCL.Neurons() / iWindow,
| |
cLongNorm.GetPeriod()*cLongNorm.GetUnits(),
| |
2 * cLongNorm.GetUnits()
| |
};
| |
if(!Concat(NeuronOCL.getOutput(), cLongNorm.getOutput(), cLongNorm.GetMeanSTDevs().getOutput(),
| |
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[0] = windows[0] + windows[1] + windows[2];
| |
windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits();
| |
windows[2] = 2 * cSeasonNorm.GetUnits();
| |
if(!cConcatenated.SwapOutputs() ||
| |
!Concat(cConcatenated.getPrevOutput(), cSeasonNorm.getOutput(), cSeasonNorm.GetMeanSTDevs().getOutput(),
| |
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[0] = windows[0] + windows[1] + windows[2];
| |
windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits();
| |
windows[2] = 2 * cShortNorm.GetUnits();
| |
if(!cConcatenated.SwapOutputs() ||
| |
!Concat(cConcatenated.getPrevOutput(), cShortNorm.getOutput(), cShortNorm.GetMeanSTDevs().getOutput(),
| |
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[0] = windows[0] + windows[1] + windows[2];
| |
windows[1] = cAdaptSpatNorm.GetUnits();
| |
windows[2] = 2 * cAdaptSpatNorm.GetUnits();
| |
if(!cConcatenated.SwapOutputs() ||
| |
!Concat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getOutput(), cAdaptSpatNorm.GetMeanSTDevs().getOutput(),
| |
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
if(!cProjection.FeedForward(cConcatenated.AsObject()))
| |
ReturnFalse;
| |
if(!cTranspose.FeedForward(cProjection.AsObject()))
| |
ReturnFalse;
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
if(!caFusion[i].FeedForward(cTranspose.AsObject()))
| |
ReturnFalse;
| |
if(!ElementMult(caFusion[0].getOutput(), caFusion[1].getOutput(), cFusionOut.getOutput()))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronTransposeOCL::feedForward(cFusionOut.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronSCNNEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronTransposeOCL::calcInputGradients(cFusionOut.AsObject()))
| |
ReturnFalse;
| |
if(!ElementMultGrad(caFusion[0].getOutput(), caFusion[0].getGradient(),
| |
caFusion[1].getOutput(), caFusion[1].getGradient(),
| |
cFusionOut.getGradient(), caFusion[0].Activation(), caFusion[1].Activation()))
| |
ReturnFalse;
| |
if(!cTranspose.CalcHiddenGradients(caFusion[0].AsObject()))
| |
ReturnFalse;
| |
CBufferFloat* temp = cTranspose.getGradient();
| |
if(!cTranspose.SetGradient(cTranspose.getPrevOutput(), false) ||
| |
!cTranspose.CalcHiddenGradients(caFusion[1].AsObject()) ||
| |
!SumAndNormalize(temp, cTranspose.getGradient(), temp, cTranspose.GetCount(), false, 0, 0, 0, 1) ||
| |
!cTranspose.SetGradient(temp, false))
| |
ReturnFalse;
| |
if(!cProjection.CalcHiddenGradients(cTranspose.AsObject()))
| |
ReturnFalse;
| |
if(!cConcatenated.CalcHiddenGradients(cProjection.AsObject()))
| |
ReturnFalse;
| |
//---
| |
uint windows[3] = {0};
| |
windows[1] = cAdaptSpatNorm.GetUnits();
| |
windows[2] = 2 * cAdaptSpatNorm.GetUnits();
| |
windows[0] = cConcatenated.Neurons() / iWindow - windows[1] - windows[2];
| |
if(!DeConcat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getGradient(),
| |
cAdaptSpatNorm.GetMeanSTDevs().getGradient(),
| |
cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits();
| |
windows[2] = 2 * cShortNorm.GetUnits();
| |
windows[0] = windows[0] - windows[1] - windows[2];
| |
if(!DeConcat(cConcatenated.getGradient(), cShortNorm.getPrevOutput(),
| |
cShortNorm.GetMeanSTDevs().getGradient(),
| |
cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits();
| |
windows[2] = 2 * cSeasonNorm.GetUnits();
| |
windows[0] = windows[0] - windows[1] - windows[2];
| |
if(!DeConcat(cConcatenated.getPrevOutput(), cSeasonNorm.getPrevOutput(),
| |
cSeasonNorm.GetMeanSTDevs().getGradient(),
| |
cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
windows[1] = cLongNorm.GetPeriod() * cLongNorm.GetUnits();
| |
windows[2] = 2 * cLongNorm.GetUnits();
| |
windows[0] = windows[0] - windows[1] - windows[2];
| |
if(!DeConcat(NeuronOCL.getPrevOutput(), cLongNorm.getPrevOutput(),
| |
cLongNorm.GetMeanSTDevs().getGradient(),
| |
cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow))
| |
ReturnFalse;
| |
//---
| |
if(!cShortNorm.CalcHiddenGradients(cAdaptSpatNorm.AsObject()) ||
| |
!SumAndNormalize(cShortNorm.getGradient(), cShortNorm.getPrevOutput(),
| |
cShortNorm.getGradient(), cShortNorm.GetPeriod(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject()) ||
| |
!SumAndNormalize(cSeasonNorm.getGradient(), cSeasonNorm.getPrevOutput(),
| |
cSeasonNorm.getGradient(), cSeasonNorm.GetPeriod(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject()) ||
| |
!SumAndNormalize(cLongNorm.getGradient(), cLongNorm.getPrevOutput(),
| |
cLongNorm.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject()) ||
| |
!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(),
| |
NeuronOCL.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronSCNNEncoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cAdaptSpatNorm.UpdateInputWeights(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cProjection.UpdateInputWeights(cConcatenated.AsObject()))
| |
ReturnFalse;
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
if(!caFusion[i].UpdateInputWeights(cTranspose.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSCNNEncoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cLongNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonTransp.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cUnSeasonTransp.Save(file_handle))
| |
ReturnFalse;
| |
if(!cShortNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cAdaptSpatNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cConcatenated.Save(file_handle))
| |
ReturnFalse;
| |
if(!cProjection.Save(file_handle))
| |
ReturnFalse;
| |
if(!cTranspose.Save(file_handle))
| |
ReturnFalse;
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
if(!caFusion[i].Save(file_handle))
| |
ReturnFalse;
| |
if(!cFusionOut.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSCNNEncoder state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cLongNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cAdaptSpatNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cConcatenated.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
| |
ReturnFalse;
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
if(!LoadInsideLayer(file_handle, caFusion[i].AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cFusionOut.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSCNNEncoder GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSCNNEncoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronTransposeOCL::SetOpenCL(obj);
| |
//---
| |
cLongNorm.SetOpenCL(OpenCL);
| |
cSeasonTransp.SetOpenCL(OpenCL);
| |
cSeasonNorm.SetOpenCL(OpenCL);
| |
cUnSeasonTransp.SetOpenCL(OpenCL);
| |
cShortNorm.SetOpenCL(OpenCL);
| |
cAdaptSpatNorm.SetOpenCL(OpenCL);
| |
cConcatenated.SetOpenCL(OpenCL);
| |
cProjection.SetOpenCL(OpenCL);
| |
cTranspose.SetOpenCL(OpenCL);
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
caFusion[i].SetOpenCL(OpenCL);
| |
cFusionOut.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronSCNNEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSCNNEncoder::TrainMode(bool flag)
| |
{
| |
CNeuronTransposeOCL::TrainMode(flag);
| |
//---
| |
cLongNorm.TrainMode(bTrain);
| |
cSeasonTransp.TrainMode(bTrain);
| |
cSeasonNorm.TrainMode(bTrain);
| |
cUnSeasonTransp.TrainMode(bTrain);
| |
cShortNorm.TrainMode(bTrain);
| |
cAdaptSpatNorm.TrainMode(bTrain);
| |
cConcatenated.TrainMode(bTrain);
| |
cProjection.TrainMode(bTrain);
| |
cTranspose.TrainMode(bTrain);
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
caFusion[i].TrainMode(bTrain);
| |
cFusionOut.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronSCNNEncoder weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronSCNNEncoder* Source = source;
| |
if(!cAdaptSpatNorm.WeightsUpdate(Source.cAdaptSpatNorm.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
| |
ReturnFalse;
| |
for(uint i = 0; i < caFusion.Size(); i++)
| |
if(!caFusion[i].WeightsUpdate(Source.caFusion[i].AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSCNN layer and allocates its GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint variables, uint forecast,
| |
uint season_period, uint short_period, uint layers,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables, optimization_type, batch))
| |
ReturnFalse;
| |
SetActivationFunction(None);
| |
//---
| |
cLayers.Clear();
| |
cLayers.SetOpenCL(OpenCL);
| |
CNeuronSCNNEncoder* encoder = NULL;
| |
CNeuronBaseOCL* residual = NULL;
| |
for(uint l = 0; l < layers; l++)
| |
{
| |
encoder = new CNeuronSCNNEncoder();
| |
if(!encoder)
| |
ReturnFalse;
| |
if(!encoder.Init(0, l, OpenCL, units_count, variables, forecast, season_period, short_period, optimization, iBatch) ||
| |
!cLayers.Add(encoder))
| |
ReturnFalse;
| |
encoder.SetActivationFunction(None);
| |
if((l + 1) == layers)
| |
break;
| |
//---
| |
residual = new CNeuronBaseOCL();
| |
if(!residual)
| |
ReturnFalse;
| |
if(!residual.Init(0, l, OpenCL, units_count * variables, optimization, iBatch) ||
| |
!cLayers.Add(residual))
| |
ReturnFalse;
| |
residual.SetActivationFunction(None);
| |
}
| |
if(!SetGradient(encoder.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSCNN feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
if(!Output.Fill(0))
| |
ReturnFalse;
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronSCNNEncoder* current = NULL;
| |
CNeuronBaseOCL* residual = NULL;
| |
CNeuronBaseOCL* temp = NULL;
| |
int layers = cLayers.Total();
| |
//---
| |
for(int l = 0; l < layers; l += 2)
| |
{
| |
current = cLayers[l];
| |
if(!current ||
| |
!current.FeedForward(inputs) ||
| |
!SumAndNormalize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if((l + 1) == layers)
| |
break;
| |
uint variables = current.GetWindow();
| |
uint dimension = inputs.Neurons() / variables;
| |
uint forecast = current.GetCount() - dimension;
| |
residual = cLayers[l + 1];
| |
if(!residual)
| |
ReturnFalse;
| |
if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(),
| |
dimension, forecast, variables) ||
| |
!SumAndNormalize(residual.getOutput(), inputs.getOutput(), residual.getOutput(),
| |
dimension, true, 0, 0, 0, 1))
| |
ReturnFalse;
| |
inputs = residual;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronSCNN |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
if(!PrevOutput.Fill(0))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* inputs = NULL;
| |
CNeuronSCNNEncoder* current = cLayers[-1];
| |
CNeuronBaseOCL* residual = NULL;
| |
int layers = cLayers.Total() - 2;
| |
//---
| |
for(int l = layers; l >= 0; l--)
| |
switch(cLayers[l].Type())
| |
{
| |
case defNeuronBaseOCL:
| |
inputs = cLayers[l];
| |
if(!inputs ||
| |
!inputs.CalcHiddenGradients(current))
| |
ReturnFalse;
| |
if(!!residual)
| |
if(!SumAndNormalize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
residual = inputs;
| |
break;
| |
case defNeuronSCNNEncoder:
| |
current = cLayers[l];
| |
if(!residual)
| |
break;
| |
inputs = cLayers[l + 2];
| |
if(!inputs)
| |
ReturnFalse;
| |
if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(),
| |
residual.Neurons() / current.GetWindow(),
| |
current.GetCount() - residual.Neurons() / current.GetWindow(),
| |
current.GetWindow()) ||
| |
!SumAndNormalize(current.getGradient(), inputs.getGradient(), current.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
break;
| |
default:
| |
ReturnFalse;
| |
break;
| |
}
| |
//---
| |
if(!NeuronOCL.CalcHiddenGradients(current))
| |
ReturnFalse;
| |
if(!!residual)
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronSCNN trainable weights from the accumulated |
| |
//| gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronBaseOCL* current = NULL;
| |
//---
| |
for(int l = 0; l < cLayers.Total(); l++)
| |
{
| |
current = cLayers[l];
| |
if(!current)
| |
ReturnFalse;
| |
if(current.Type() == defNeuronSCNNEncoder ||
| |
current.Type() == defNeuronSSCNNEncoder)
| |
if(!current.UpdateInputWeights(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronSCNN weights toward the source model with |
| |
//| factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronSCNN* Source = source;
| |
if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSCNN state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(!cLayers.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSCNN state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSCNN::Load(const int file_handle)
| |
{
| |
cLayers.Clear();
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
cLayers.SetOpenCL(OpenCL);
| |
if(!cLayers.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* last = cLayers[-1];
| |
if(!SetGradient(last.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSCNN GPU buffers to the given OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSCNN::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cLayers.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronSCNN layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSCNN::TrainMode(bool flag)
| |
{
| |
CNeuronBaseOCL::TrainMode(flag);
| |
cLayers.TrainMode(flag);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronPolynomialRegression layer and allocates |
| |
//| its GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint window, uint window_out,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window_out, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
if(!cProjection.Init(0, 0, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
if(!cConvolution.Init(0, 1, OpenCL, window_out, window_out, window_out, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cConvolution.SetActivationFunction(None);
| |
if(!cResidual.Init(0, 2, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cResidual.SetActivationFunction(None);
| |
//---
| |
if(!SetGradient(cConvolution.getGradient(), true))
| |
ReturnFalse;
| |
if(!cResidual.SetGradient(getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronPolynomialRegression feed-forward pass and |
| |
//| writes the layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cProjection.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cConvolution.FeedForward(cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!cResidual.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cConvolution.getOutput(), cResidual.getOutput(),
| |
Output, GetWindowOut(), true, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronPolynomialRegression layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!cProjection.CalcHiddenGradients(cConvolution.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject()))
| |
ReturnFalse;
| |
CBufferFloat* temp = NeuronOCL.getGradient();
| |
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) ||
| |
!NeuronOCL.CalcHiddenGradients(cResidual.AsObject()) ||
| |
!SumAndNormalize(temp, NeuronOCL.getGradient(), temp,
| |
GetWindowIn(), false, 0, 0, 0, 1) ||
| |
!NeuronOCL.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronPolynomialRegression trainable weights from |
| |
//| the accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cProjection.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cConvolution.UpdateInputWeights(cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!cResidual.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronPolynomialRegression weights toward the |
| |
//| source model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronPolynomialRegression* Source = source;
| |
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cConvolution.WeightsUpdate(Source.cConvolution.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronPolynomialRegression state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cProjection.Save(file_handle))
| |
ReturnFalse;
| |
if(!cConvolution.Save(file_handle))
| |
ReturnFalse;
| |
if(!cResidual.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronPolynomialRegression state from the file |
| |
//| handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronPolynomialRegression::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cConvolution.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cResidual.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronPolynomialRegression GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronPolynomialRegression::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
//---
| |
cProjection.SetOpenCL(OpenCL);
| |
cConvolution.SetOpenCL(OpenCL);
| |
cResidual.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSSCNNEncoder layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint variables, uint forecast,
| |
uint season_period, uint short_period,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast,
| |
variables, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
int index = 0;
| |
if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, iWindow, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cLongExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cLongExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cLongMeanSTDevTransp.Init(0, index, OpenCL, iWindow, 2, optimization, iBatch))
| |
ReturnFalse;
| |
if(!cLongMeanSTDevTransp.getGradient().Fill(0))
| |
ReturnFalse;
| |
index++;
| |
if(!cLongMeanExtrapolate.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cLongExtrapolate.SetActivationFunction(None);
| |
if(!cLongMeanExtrapolate.getPrevOutput().Fill(1))
| |
ReturnFalse;
| |
index++;
| |
if(!cSeasonTransp.Init(0, index, OpenCL, iWindow, units_count / season_period, season_period, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cSeasonNorm.Init(0, index, OpenCL, season_period, cSeasonTransp.GetCount(), iWindow, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cUnSeasonTransp.Init(0, index, OpenCL, iWindow, season_period, cSeasonTransp.GetCount(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cSeasonExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cSeasonExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cSeasonMeanExtrapolate.Init(0, index, OpenCL, season_period, season_period, iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cSeasonMeanExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, iWindow, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cShortExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cShortExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cShortMeanExtrapolate.Init(0, index, OpenCL, cShortNorm.GetUnits(), cShortNorm.GetUnits(), iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cShortMeanExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cSpatialNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cSpatialExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cSpatialExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cSpatialMeanExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, 1, iWindow, optimization, iBatch))
| |
ReturnFalse;
| |
cSpatialMeanExtrapolate.SetActivationFunction(None);
| |
index++;
| |
if(!cConcatenated.Init(0, index, OpenCL, 8 * Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cTranspose.Init(0, index, OpenCL, 8 * iWindow, iCount, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cFusion.Init(0, index, OpenCL, iCount, 8 * iWindow, iWindow, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSSCNNEncoder feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
//--- Long
| |
if(!cLongNorm.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cLongExtrapolate.FeedForward(cLongNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cLongMeanSTDevTransp.FeedForward(cLongNorm.GetMeanSTDevs()))
| |
ReturnFalse;
| |
if(!MatMul(cLongMeanSTDevTransp.getOutput(), cLongMeanExtrapolate.getPrevOutput(),
| |
cLongMeanExtrapolate.getOutput(), 1, 1, iCount, iWindow, true))
| |
ReturnFalse;
| |
//--- Season
| |
if(!cSeasonTransp.FeedForward(cLongNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonExtrapolate.FeedForward(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonMeanExtrapolate.FeedForward(cSeasonNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Short
| |
if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cShortExtrapolate.FeedForward(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cShortMeanExtrapolate.FeedForward(cShortNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Spatial
| |
if(!cSpatialNorm.FeedForward(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatialExtrapolate.FeedForward(cSpatialNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatialMeanExtrapolate.FeedForward(cSpatialNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Concat
| |
if(!Concat(cLongExtrapolate.getOutput(), cLongMeanExtrapolate.getOutput(),
| |
cSeasonExtrapolate.getOutput(), cSeasonMeanExtrapolate.getOutput(),
| |
cConcatenated.getOutput(), iCount, iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
if(!Concat(cConcatenated.getOutput(), cShortExtrapolate.getOutput(),
| |
cShortMeanExtrapolate.getOutput(), cConcatenated.getPrevOutput(),
| |
4 * iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
if(!Concat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getOutput(),
| |
cSpatialMeanExtrapolate.getOutput(), cConcatenated.getOutput(),
| |
6 * iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
//--- Fusion
| |
if(!cTranspose.FeedForward(cConcatenated.AsObject()))
| |
ReturnFalse;
| |
if(!cFusion.FeedForward(cTranspose.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronTransposeOCL::feedForward(cFusion.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronSSCNNEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//--- Fusion
| |
if(!CNeuronTransposeOCL::calcInputGradients(cFusion.AsObject()))
| |
ReturnFalse;
| |
if(!cTranspose.CalcHiddenGradients(cFusion.AsObject()))
| |
ReturnFalse;
| |
if(!cConcatenated.CalcHiddenGradients(cTranspose.AsObject()))
| |
ReturnFalse;
| |
//--- DeConcat
| |
if(!DeConcat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getGradient(),
| |
cSpatialMeanExtrapolate.getGradient(), cConcatenated.getGradient(),
| |
6 * iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
if(!DeConcat(cConcatenated.getGradient(), cShortExtrapolate.getGradient(),
| |
cShortMeanExtrapolate.getGradient(), cConcatenated.getPrevOutput(),
| |
4 * iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
if(!DeConcat(cLongExtrapolate.getGradient(), cLongMeanExtrapolate.getGradient(),
| |
cSeasonExtrapolate.getGradient(), cSeasonMeanExtrapolate.getGradient(),
| |
cConcatenated.getGradient(), iCount, iCount, iCount, iCount, iWindow))
| |
ReturnFalse;
| |
//--- Spatial
| |
if(!cSpatialNorm.GetMeans().CalcHiddenGradients(cSpatialMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatialNorm.CalcHiddenGradients(cSpatialExtrapolate.AsObject()))
| |
ReturnFalse;
| |
//--- Short
| |
if(!cShortNorm.CalcHiddenGradients(cSpatialNorm.AsObject()))
| |
ReturnFalse;
| |
CBufferFloat* temp = cShortNorm.getGradient();
| |
if(!cShortNorm.SetGradient(cShortNorm.getPrevOutput(), false) ||
| |
!cShortNorm.CalcHiddenGradients(cShortExtrapolate.AsObject()) ||
| |
!SumAndNormalize(temp, cShortNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
| |
!cShortNorm.SetGradient(temp, false))
| |
ReturnFalse;
| |
if(!cShortNorm.GetMeans().CalcHiddenGradients(cShortMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
//--- Season
| |
if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
temp = cUnSeasonTransp.getGradient();
| |
if(!cUnSeasonTransp.SetGradient(cUnSeasonTransp.getPrevOutput(), false) ||
| |
!cUnSeasonTransp.CalcHiddenGradients(cSeasonExtrapolate.AsObject()) ||
| |
!SumAndNormalize(temp, cUnSeasonTransp.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
| |
!cUnSeasonTransp.SetGradient(temp, false))
| |
ReturnFalse;
| |
if(!cSeasonNorm.GetMeans().CalcHiddenGradients(cSeasonMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
//--- Long
| |
if(!MatMulGrad(cLongMeanSTDevTransp.getOutput(), cLongMeanSTDevTransp.getGradient(),
| |
cLongMeanExtrapolate.getPrevOutput(), cLongMeanExtrapolate.getGradient(),
| |
cLongMeanExtrapolate.getGradient(), 1, 1, iCount, iWindow, true))
| |
ReturnFalse;
| |
if(!cLongNorm.GetMeanSTDevs().CalcHiddenGradients(cLongMeanSTDevTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
temp = cLongNorm.getGradient();
| |
if(!cLongNorm.SetGradient(cLongNorm.getPrevOutput(), false) ||
| |
!cLongNorm.CalcHiddenGradients(cLongExtrapolate.AsObject()) ||
| |
!SumAndNormalize(temp, cLongNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
| |
!cLongNorm.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronSSCNNEncoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
//--- Long
| |
if(!cLongNorm.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cLongExtrapolate.UpdateInputWeights(cLongNorm.AsObject()))
| |
ReturnFalse;
| |
//--- Season
| |
if(!cSeasonNorm.UpdateInputWeights(cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonExtrapolate.UpdateInputWeights(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cSeasonMeanExtrapolate.UpdateInputWeights(cSeasonNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Short
| |
if(!cShortNorm.UpdateInputWeights(cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!cShortExtrapolate.UpdateInputWeights(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cShortMeanExtrapolate.UpdateInputWeights(cShortNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Spatial
| |
if(!cSpatialNorm.UpdateInputWeights(cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatialExtrapolate.UpdateInputWeights(cSpatialNorm.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatialMeanExtrapolate.UpdateInputWeights(cSpatialNorm.GetMeans()))
| |
ReturnFalse;
| |
//--- Fusion
| |
if(!cFusion.UpdateInputWeights(cTranspose.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronSSCNNEncoder weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronSSCNNEncoder* Source = source;
| |
//--- Long
| |
if(!cLongNorm.WeightsUpdate(Source.cLongNorm.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cLongExtrapolate.WeightsUpdate(Source.cLongExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
//--- Season
| |
if(!cSeasonNorm.WeightsUpdate(Source.cSeasonNorm.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cSeasonExtrapolate.WeightsUpdate(Source.cSeasonExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cSeasonMeanExtrapolate.WeightsUpdate(Source.cSeasonMeanExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
//--- Short
| |
if(!cShortNorm.WeightsUpdate(Source.cShortNorm.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cShortExtrapolate.WeightsUpdate(Source.cShortExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cShortMeanExtrapolate.WeightsUpdate(Source.cShortMeanExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
//--- Spatial
| |
if(!cSpatialNorm.WeightsUpdate(Source.cSpatialNorm.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cSpatialExtrapolate.WeightsUpdate(Source.cSpatialExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cSpatialMeanExtrapolate.WeightsUpdate(Source.cSpatialMeanExtrapolate.AsObject(), tau))
| |
ReturnFalse;
| |
//--- Fusion
| |
if(!cFusion.WeightsUpdate(Source.cFusion.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSSCNNEncoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cLongNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cLongExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cLongMeanSTDevTransp.Save(file_handle))
| |
ReturnFalse;
| |
if(!cLongMeanExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonTransp.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cUnSeasonTransp.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSeasonMeanExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cShortNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cShortExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cShortMeanExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSpatialNorm.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSpatialExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSpatialMeanExtrapolate.Save(file_handle))
| |
ReturnFalse;
| |
if(!cConcatenated.Save(file_handle))
| |
ReturnFalse;
| |
if(!cTranspose.Save(file_handle))
| |
ReturnFalse;
| |
if(!cFusion.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSSCNNEncoder state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNNEncoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cLongNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cLongExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cLongMeanSTDevTransp.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cLongMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSeasonMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cShortNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cShortExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cShortMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSpatialNorm.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSpatialExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSpatialMeanExtrapolate.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cConcatenated.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cFusion.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSSCNNEncoder GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSSCNNEncoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronTransposeOCL::SetOpenCL(obj);
| |
//---
| |
cLongNorm.SetOpenCL(OpenCL);
| |
cLongExtrapolate.SetOpenCL(OpenCL);
| |
cLongMeanSTDevTransp.SetOpenCL(OpenCL);
| |
cLongMeanExtrapolate.SetOpenCL(OpenCL);
| |
cSeasonTransp.SetOpenCL(OpenCL);
| |
cSeasonNorm.SetOpenCL(OpenCL);
| |
cUnSeasonTransp.SetOpenCL(OpenCL);
| |
cSeasonExtrapolate.SetOpenCL(OpenCL);
| |
cSeasonMeanExtrapolate.SetOpenCL(OpenCL);
| |
cShortNorm.SetOpenCL(OpenCL);
| |
cShortExtrapolate.SetOpenCL(OpenCL);
| |
cShortMeanExtrapolate.SetOpenCL(OpenCL);
| |
cSpatialNorm.SetOpenCL(OpenCL);
| |
cSpatialExtrapolate.SetOpenCL(OpenCL);
| |
cSpatialMeanExtrapolate.SetOpenCL(OpenCL);
| |
cConcatenated.SetOpenCL(OpenCL);
| |
cTranspose.SetOpenCL(OpenCL);
| |
cFusion.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronSSCNNEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSSCNNEncoder::TrainMode(bool flag)
| |
{
| |
CNeuronTransposeOCL::TrainMode(flag);
| |
//---
| |
cLongNorm.TrainMode(bTrain);
| |
cLongExtrapolate.TrainMode(bTrain);
| |
cLongMeanSTDevTransp.TrainMode(bTrain);
| |
cLongMeanExtrapolate.TrainMode(bTrain);
| |
cSeasonTransp.TrainMode(bTrain);
| |
cSeasonNorm.TrainMode(bTrain);
| |
cUnSeasonTransp.TrainMode(bTrain);
| |
cSeasonExtrapolate.TrainMode(bTrain);
| |
cSeasonMeanExtrapolate.TrainMode(bTrain);
| |
cShortNorm.TrainMode(bTrain);
| |
cShortExtrapolate.TrainMode(bTrain);
| |
cShortMeanExtrapolate.TrainMode(bTrain);
| |
cSpatialNorm.TrainMode(bTrain);
| |
cSpatialExtrapolate.TrainMode(bTrain);
| |
cSpatialMeanExtrapolate.TrainMode(bTrain);
| |
cConcatenated.TrainMode(bTrain);
| |
cTranspose.TrainMode(bTrain);
| |
cFusion.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSSCNN layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units_count, uint variables, uint forecast,
| |
uint season_period, uint short_period, uint layers,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables,
| |
optimization_type, batch))
| |
ReturnFalse;
| |
SetActivationFunction(None);
| |
//---
| |
cLayers.Clear();
| |
cLayers.SetOpenCL(OpenCL);
| |
CNeuronSSCNNEncoder* encoder = NULL;
| |
CNeuronBaseOCL* residual = NULL;
| |
for(uint l = 0; l < layers; l++)
| |
{
| |
encoder = new CNeuronSSCNNEncoder();
| |
if(!encoder)
| |
ReturnFalse;
| |
if(!encoder.Init(0, 2 * l, OpenCL, units_count, variables, forecast, season_period,
| |
short_period, optimization, iBatch) ||
| |
!cLayers.Add(encoder))
| |
ReturnFalse;
| |
encoder.SetActivationFunction(None);
| |
if((l + 1) == layers)
| |
break;
| |
//---
| |
residual = new CNeuronBaseOCL();
| |
if(!residual)
| |
ReturnFalse;
| |
if(!residual.Init(0, 2 * l + 1, OpenCL, units_count * variables, optimization, iBatch) ||
| |
!cLayers.Add(residual))
| |
ReturnFalse;
| |
residual.SetActivationFunction(None);
| |
}
| |
if(!SetGradient(encoder.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSSCNN feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNN::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
if(!Output.Fill(0))
| |
ReturnFalse;
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronSSCNNEncoder* current = NULL;
| |
CNeuronBaseOCL* residual = NULL;
| |
CNeuronBaseOCL* temp = NULL;
| |
int layers = cLayers.Total();
| |
//---
| |
for(int l = 0; l < layers; l += 2)
| |
{
| |
current = cLayers[l];
| |
if(!current ||
| |
!current.FeedForward(inputs) ||
| |
!SumAndNormalize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if((l + 1) == layers)
| |
break;
| |
uint variables = current.GetWindow();
| |
uint dimension = inputs.Neurons() / variables;
| |
uint forecast = current.GetCount() - dimension;
| |
residual = cLayers[l + 1];
| |
if(!residual)
| |
ReturnFalse;
| |
if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(),
| |
dimension, forecast, variables) ||
| |
!SumAndNormalize(residual.getOutput(), inputs.getOutput(), residual.getOutput(),
| |
dimension, true, 0, 0, 0, 1))
| |
ReturnFalse;
| |
inputs = residual;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronSSCNN |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
if(!PrevOutput.Fill(0))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* inputs = NULL;
| |
CNeuronSSCNNEncoder* current = cLayers[-1];
| |
CNeuronBaseOCL* residual = NULL;
| |
int layers = cLayers.Total() - 2;
| |
//---
| |
for(int l = layers; l >= 0; l--)
| |
switch(cLayers[l].Type())
| |
{
| |
case defNeuronBaseOCL:
| |
inputs = cLayers[l];
| |
if(!inputs ||
| |
!inputs.CalcHiddenGradients(current))
| |
ReturnFalse;
| |
if(!!residual)
| |
if(!SumAndNormalize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
residual = inputs;
| |
break;
| |
case defNeuronSSCNNEncoder:
| |
current = cLayers[l];
| |
if(!residual)
| |
break;
| |
inputs = cLayers[l + 2];
| |
if(!inputs)
| |
ReturnFalse;
| |
if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(),
| |
residual.Neurons() / current.GetWindow(),
| |
current.GetCount() - residual.Neurons() / current.GetWindow(),
| |
current.GetWindow()) ||
| |
!SumAndNormalize(current.getGradient(), inputs.getGradient(), current.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
break;
| |
default:
| |
ReturnFalse;
| |
break;
| |
}
| |
//---
| |
if(!NeuronOCL.CalcHiddenGradients(current))
| |
ReturnFalse;
| |
if(!!residual)
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(),
| |
current.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CChebPolinom layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CChebPolinom::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension,
| |
uint steps, ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * dimension * steps, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
activation = None;
| |
iDimension = dimension;
| |
iSteps = steps;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CChebPolinom feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CChebPolinom::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension))
| |
ReturnFalse;
| |
//---
| |
uint global_work_offset[3] = { 0 };
| |
uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))),
| |
iDimension, iDimension
| |
};
| |
uint local_work_size[3] = { global_work_size[0], 1, 1 };
| |
//---
| |
uint kernel = def_k_ChebStep;
| |
setBuffer(kernel, def_k_cheb_support, NeuronOCL.getOutputIndex())
| |
setBuffer(kernel, def_k_cheb_outputs, getOutputIndex())
| |
for(int step = MathMin(2, MathMax(int(iSteps) - 1, 0)); step < int(iSteps); step++)
| |
{
| |
setArgument(kernel, def_k_cheb_step, step + 1)
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!Output.BufferRead())
| |
ReturnFalse;
| |
#endif
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CChebPolinom |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CChebPolinom::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension))
| |
ReturnFalse;
| |
//---
| |
uint global_work_offset[3] = { 0 };
| |
uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))),
| |
iDimension, iDimension
| |
};
| |
uint local_work_size[3] = { global_work_size[0], 1, 1 };
| |
//---
| |
uint kernel = def_k_ChebStepGrad;
| |
setBuffer(kernel, def_k_chebgr_support, NeuronOCL.getOutputIndex())
| |
setBuffer(kernel, def_k_chebgr_support_g, NeuronOCL.getGradientIndex())
| |
setBuffer(kernel, def_k_chebgr_outputs, getOutputIndex())
| |
setBuffer(kernel, def_k_chebgr_outputs_g, getGradientIndex())
| |
for(int step = (int(iSteps) - 1); step >= MathMax(MathMin(2, int(iSteps) - 1), 1); step--)
| |
{
| |
setArgument(kernel, def_k_chebgr_step, step)
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!NeuronOCL.getGradient().BufferRead())
| |
ReturnFalse;
| |
#endif
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CChebPolinom state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CChebPolinom::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE)
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, int(iSteps)) < INT_VALUE)
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CChebPolinom state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CChebPolinom::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iDimension = (uint)FileReadInteger(file_handle);
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iSteps = (uint)FileReadInteger(file_handle);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetGrapConv layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGrapConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint count, uint window, uint cheb_k,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronTransposeRCDOCL::Init(numOutputs, myIndex, open_cl, cheb_k,
| |
count, window, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cX_G.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronHimNetGrapConv feed forward operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGrapConv::FeedForward(CNeuronBaseOCL *NeuronOCL,
| |
CChebPolinom *Support)
| |
{
| |
if(!NeuronOCL || !Support)
| |
ReturnFalse;
| |
if(Support.GetDimension() != iWindow ||
| |
Support.GetSteps() != iCount)
| |
ReturnFalse;
| |
if(NeuronOCL.Neurons() != (Neurons() / iCount))
| |
ReturnFalse;
| |
//---
| |
if(!MatMul(Support.getOutput(), NeuronOCL.getOutput(), cX_G.getOutput(),
| |
iWindow, iWindow, GetDimension(), iCount, false))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronTransposeRCDOCL::feedForward(cX_G.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetGrapConv layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGrapConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
| |
CChebPolinom *Support)
| |
{
| |
if(!NeuronOCL || !Support)
| |
ReturnFalse;
| |
if(Support.GetDimension() != iWindow ||
| |
Support.GetSteps() != iCount)
| |
ReturnFalse;
| |
if(NeuronOCL.Neurons() != (Neurons() / iCount))
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronTransposeRCDOCL::calcInputGradients(cX_G.AsObject()))
| |
ReturnFalse;
| |
if(!MatMulGrad(Support.getOutput(), Support.getGradient(),
| |
NeuronOCL.getOutput(), NeuronOCL.getGradient(),
| |
cX_G.getGradient(), iWindow, iWindow,
| |
GetDimension(), iCount, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetGrapConv state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGrapConv::Load(const int file_handle)
| |
{
| |
if(!CNeuronTransposeRCDOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetGrapConv GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetGrapConv::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronTransposeRCDOCL::SetOpenCL(obj);
| |
cX_G.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetGCRU layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint window_out,
| |
uint cheb_k, uint embed_dim,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
int index = 0;
| |
if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch))
| |
ReturnFalse;
| |
cInpAndHidden.SetActivationFunction(None);
| |
index++;
| |
if(!cZ_R.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch))
| |
ReturnFalse;
| |
cZ_R.SetActivationFunction(None);
| |
index++;
| |
if(!cZ_R_emb.Init(0, index, OpenCL, embed_dim, embed_dim,
| |
2 * cheb_k * (window + window_out) * window_out,
| |
units, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cZ_R_emb.SetActivationFunction(None);
| |
index++;
| |
if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cZe_Re.SetActivationFunction(None);
| |
index++;
| |
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cZ.SetActivationFunction(None);
| |
index++;
| |
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cR.SetActivationFunction(None);
| |
index++;
| |
if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cCandidate.SetActivationFunction(None);
| |
index++;
| |
if(!cHC.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch))
| |
ReturnFalse;
| |
cHC.SetActivationFunction(None);
| |
index++;
| |
if(!cHC_emb.Init(0, index, OpenCL, embed_dim, embed_dim,
| |
(window + window_out) * window_out * cheb_k,
| |
units, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cHC_emb.SetActivationFunction(None);
| |
index++;
| |
if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cHCe.SetActivationFunction(None);
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
if(!Output.Fill(0))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronHimNetGCRU feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::feedForward(CNeuronBaseOCL *NeuronOCL,
| |
CChebPolinom *Support,
| |
CNeuronBaseOCL *Embedding)
| |
{
| |
if(!NeuronOCL || !Support || !Embedding)
| |
ReturnFalse;
| |
if(!SwapOutputs())
| |
ReturnFalse;
| |
//---
| |
uint cheb_k = cZ_R.GetChebK();
| |
uint units = cZ_R.GetCount();
| |
uint window_out = Neurons() / units;
| |
uint window = cZ_R.GetWindow() - window_out;
| |
//---
| |
if(!Concat(NeuronOCL.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window_out, units))
| |
ReturnFalse;
| |
if(!cZ_R.FeedForward(cInpAndHidden.AsObject(), Support))
| |
ReturnFalse;
| |
if(!cZ_R_emb.FeedForward(Embedding))
| |
ReturnFalse;
| |
if(!MatMul(cZ_R.getOutput(), cZ_R_emb.getOutput(), cZe_Re.getOutput(), 1, (window + window_out)*cheb_k,
| |
2 * window_out, units, true))
| |
ReturnFalse;
| |
if(!Activation(cZe_Re.getOutput(), cZe_Re.getOutput(), SIGMOID))
| |
ReturnFalse;
| |
if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZe_Re.getOutput(), window_out, window_out, units))
| |
ReturnFalse;
| |
if(!ElementMult(cZ.getOutput(), PrevOutput, cZ.getPrevOutput()))
| |
ReturnFalse;
| |
if(!Concat(NeuronOCL.getOutput(), cZ.getPrevOutput(), cCandidate.getOutput(), window, window_out, units))
| |
ReturnFalse;
| |
if(!cHC.FeedForward(cCandidate.AsObject(), Support))
| |
ReturnFalse;
| |
if(!cHC_emb.FeedForward(Embedding))
| |
ReturnFalse;
| |
if(!MatMul(cHC.getOutput(), cHC_emb.getOutput(), cHCe.getOutput(), 1, (window + window_out)*cheb_k,
| |
window_out, units, true))
| |
ReturnFalse;
| |
if(!Activation(cHCe.getOutput(), cHCe.getOutput(), TANH))
| |
ReturnFalse;
| |
if(!GateElementMult(PrevOutput, cHCe.getOutput(), cR.getOutput(), Output))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetGCRU layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
| |
CChebPolinom *Support,
| |
CNeuronBaseOCL *Embedding)
| |
{
| |
if(!NeuronOCL || !Support || !Embedding)
| |
ReturnFalse;
| |
//---
| |
uint cheb_k = cZ_R.GetChebK();
| |
uint units = cZ_R.GetCount();
| |
uint window_out = Neurons() / units;
| |
uint window = cZ_R.GetWindow() - window_out;
| |
//---
| |
if(!GateElementMultGrad(PrevOutput, cR.getPrevOutput(),
| |
cHCe.getOutput(), cHCe.getGradient(),
| |
cR.getOutput(), cR.getGradient(),
| |
Gradient, None, TANH, cR.Activation()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cHC.getOutput(), cHC.getGradient(),
| |
cHC_emb.getOutput(), cHC_emb.getGradient(),
| |
cHCe.getGradient(), 1, (window + window_out)*cheb_k,
| |
window_out, units, true))
| |
ReturnFalse;
| |
if(!cHC.calcInputGradients(cCandidate.AsObject(), Support))
| |
ReturnFalse;
| |
if(!DeConcat(NeuronOCL.getGradient(), cZ.getPrevOutput(), cCandidate.getGradient(),
| |
window, window_out, units))
| |
ReturnFalse;
| |
if(!ElementMultGrad(cZ.getOutput(), cZ.getGradient(),
| |
PrevOutput, cCandidate.getPrevOutput(),
| |
cZ.getPrevOutput(), None, None))
| |
ReturnFalse;
| |
if(!Concat(cZ.getGradient(), cR.getGradient(), cZe_Re.getGradient(),
| |
window_out, window_out, units))
| |
ReturnFalse;
| |
if(!DeActivation(cZe_Re.getOutput(), cZe_Re.getGradient(), cZe_Re.getGradient(), SIGMOID))
| |
ReturnFalse;
| |
if(!MatMulGrad(cZ_R.getOutput(), cZ_R.getGradient(),
| |
cZ_R_emb.getOutput(), cZ_R_emb.getGradient(),
| |
cZe_Re.getGradient(), 1, (window + window_out)*cheb_k,
| |
2 * window_out, units, true))
| |
ReturnFalse;
| |
CBufferFloat* temp = Support.getGradient();
| |
if(!Support.SetGradient(GetPointer(bSupportAccum), false) ||
| |
!cZ_R.calcInputGradients(cInpAndHidden.AsObject(), Support) ||
| |
!SumAndNormalize(temp, Support.getGradient(), temp, Support.GetDimension(), false, 0, 0, 0, 1) ||
| |
!Support.SetGradient(temp, false))
| |
ReturnFalse;
| |
if(!DeConcat(cInpAndHidden.getPrevOutput(), cCandidate.getPrevOutput(), cInpAndHidden.getGradient(), window, window_out, units))
| |
ReturnFalse;
| |
if(!SumAndNormalize(NeuronOCL.getGradient(), cInpAndHidden.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(NeuronOCL.Activation() != None)
| |
if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation()))
| |
ReturnFalse;
| |
//---
| |
if(!Embedding.CalcHiddenGradients(cHC_emb.AsObject()))
| |
ReturnFalse;
| |
temp = Embedding.getGradient();
| |
if(!Embedding.SetGradient(Embedding.getPrevOutput(), false) ||
| |
!Embedding.CalcHiddenGradients(cZ_R_emb.AsObject()) ||
| |
!SumAndNormalize(temp, Embedding.getGradient(), temp, cZ_R_emb.GetWindow(), false, 0, 0, 0, 1) ||
| |
!Embedding.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronHimNetGCRU trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::updateInputWeights(CNeuronBaseOCL *NeuronOCL,
| |
CChebPolinom *Support,
| |
CNeuronBaseOCL *Embedding)
| |
{
| |
if(!Embedding)
| |
ReturnFalse;
| |
//---
| |
if(!cZ_R_emb.UpdateInputWeights(Embedding))
| |
ReturnFalse;
| |
if(!cHC_emb.UpdateInputWeights(Embedding))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronHimNetGCRU weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetGCRU* Source = source;
| |
if(!cZ_R_emb.WeightsUpdate(Source.cZ_R_emb.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cHC_emb.WeightsUpdate(Source.cHC_emb.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronHimNetGCRU state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cZ_R.Save(file_handle))
| |
ReturnFalse;
| |
if(!cZ_R_emb.Save(file_handle))
| |
ReturnFalse;
| |
if(!cHC.Save(file_handle))
| |
ReturnFalse;
| |
if(!cHC_emb.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetGCRU state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetGCRU::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cZ_R.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cZ_R_emb.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cHC.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cHC_emb.AsObject()))
| |
ReturnFalse;
| |
//---
| |
uint cheb_k = cZ_R.GetChebK();
| |
uint units = cZ_R.GetCount();
| |
uint window_out = Neurons() / units;
| |
uint window = cZ_R.GetWindow() - window_out;
| |
//---
| |
int index = 0;
| |
if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch))
| |
ReturnFalse;
| |
cInpAndHidden.SetActivationFunction(None);
| |
index += 3;
| |
if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cZ.SetActivationFunction(None);
| |
index++;
| |
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cR.SetActivationFunction(None);
| |
index++;
| |
if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cCandidate.SetActivationFunction(None);
| |
index += 3;
| |
if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
cHCe.SetActivationFunction(None);
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
if(!Output.Fill(0))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetGCRU GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetGCRU::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
//---
| |
cZ_R.SetOpenCL(OpenCL);
| |
cZ_R_emb.SetOpenCL(OpenCL);
| |
cHC.SetOpenCL(OpenCL);
| |
cHC_emb.SetOpenCL(OpenCL);
| |
cInpAndHidden.SetOpenCL(OpenCL);
| |
cZe_Re.SetOpenCL(OpenCL);
| |
cZ.SetOpenCL(OpenCL);
| |
cR.SetOpenCL(OpenCL);
| |
cCandidate.SetOpenCL(OpenCL);
| |
cHCe.SetOpenCL(OpenCL);
| |
//---
| |
bSupportAccum.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetTempEncoder layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint window_out, uint cheb_k,
| |
uint layers, uint embed_dim, uint period1,
| |
uint timeframe1, uint period2, uint timeframe2,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(layers <= 0)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!caEmbeddings[0].Init(0, index, OpenCL, embed_dim, period1, optimization, iBatch))
| |
ReturnFalse;
| |
aTimeframes[0] = MathMax(1, timeframe1);
| |
index++;
| |
if(!caEmbeddings[1].Init(0, index, OpenCL, embed_dim, period2, optimization, iBatch))
| |
ReturnFalse;
| |
aTimeframes[1] = MathMax(1, timeframe2);
| |
if(!cConcatEmbeddings.Init(numOutputs, myIndex, open_cl, 2 * embed_dim, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
cGRCUs.Clear();
| |
cGRCUs.SetOpenCL(OpenCL);
| |
index++;
| |
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
ReturnFalse;
| |
for(uint i = 1; i < layers; i++)
| |
{
| |
index++;
| |
temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
ReturnFalse;
| |
}
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronHimNetTempEncoder feed-forward pass and writes |
| |
//| the layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, int label)
| |
{
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
{
| |
int position = (label / int(aTimeframes[i])) % caEmbeddings[i].GetPeriod();
| |
if(!caEmbeddings[i].SetPosition(position) ||
| |
!caEmbeddings[i].FeedForward())
| |
ReturnFalse;
| |
}
| |
if(!Concat(caEmbeddings[0].getOutput(), caEmbeddings[1].getOutput(),
| |
cConcatEmbeddings.getOutput(), 1, 1, caEmbeddings[0].Neurons()))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL *inputs = NeuronOCL;
| |
CNeuronHimNetGCRU *current = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current ||
| |
!current.feedForward(inputs, Support, cConcatEmbeddings.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetTempEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
| |
{
| |
if(!NeuronOCL || !Support)
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL);
| |
CNeuronHimNetGCRU* current = cGRCUs[-1];
| |
if(!current ||
| |
!current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(caEmbeddings[0].getGradient(), caEmbeddings[1].getGradient(),
| |
cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons()))
| |
ReturnFalse;
| |
if(cGRCUs.Total() > 1)
| |
{
| |
CBufferFloat *temp = Support.getGradient();
| |
if(!Support.SetGradient(GetPointer(bSupportAccum), false))
| |
ReturnFalse;
| |
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
| |
{
| |
current = cGRCUs[i];
| |
inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL);
| |
if(!current ||
| |
!current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(temp, Support.getGradient(), temp, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!DeConcat(caEmbeddings[0].getPrevOutput(), caEmbeddings[1].getPrevOutput(),
| |
cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons()))
| |
ReturnFalse;
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
if(!SumAndNormalize(caEmbeddings[i].getGradient(), caEmbeddings[i].getPrevOutput(),
| |
caEmbeddings[i].getGradient(), 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
}
| |
if(!Support.SetGradient(temp, false))
| |
ReturnFalse;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronHimNetTempEncoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
| |
{
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
if(!caEmbeddings[i].UpdateInputWeights())
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronHimNetGCRU* current = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current.updateInputWeights(inputs, Support, cConcatEmbeddings.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronHimNetTempEncoder weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetTempEncoder* Source = source;
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau))
| |
ReturnFalse;
| |
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronHimNetTempEncoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
{
| |
if(!caEmbeddings[i].Save(file_handle))
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, int(aTimeframes[i])) < INT_VALUE)
| |
ReturnFalse;
| |
}
| |
if(!cGRCUs.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetTempEncoder state from the file |
| |
//| handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
{
| |
if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject()))
| |
ReturnFalse;
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
aTimeframes[i] = (uint)FileReadInteger(file_handle);
| |
}
| |
if(!cGRCUs.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
CNeuronHimNetGCRU* temp = cGRCUs[-1];
| |
uint units = temp.GetCount();
| |
uint cheb_k = temp.GetChebK();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetTempEncoder GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetTempEncoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
for(uint i = 0; i < caEmbeddings.Size(); i++)
| |
caEmbeddings[i].SetOpenCL(OpenCL);
| |
cGRCUs.SetOpenCL(OpenCL);
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronHimNetTempEncoder execution state and its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::Clear(void)
| |
{
| |
CNeuronBaseOCL* temp = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
temp = cGRCUs[i];
| |
if(!temp ||
| |
!temp.Clear())
| |
ReturnFalse;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the count of the CNeuronHimNetTempEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetTempEncoder::GetCount(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetCount());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the cheb k of the CNeuronHimNetTempEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetTempEncoder::GetChebK(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetChebK());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the window in of the CNeuronHimNetTempEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetTempEncoder::GetWindowIn(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetWindowIn());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the window out of the CNeuronHimNetTempEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetTempEncoder::GetWindowOut(void) const
| |
{
| |
uint count = GetCount();
| |
if(count <= 0)
| |
return(0);
| |
//---
| |
return(Neurons() / count);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetSpatEncoder layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint window_out, uint cheb_k,
| |
uint layers, uint embed_dim,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(layers <= 0)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cEmbedding.Init(0, index, OpenCL, embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
cGRCUs.Clear();
| |
cGRCUs.SetOpenCL(OpenCL);
| |
index++;
| |
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
{
| |
DeleteObj(temp);
| |
ReturnFalse;
| |
}
| |
for(uint i = 1; i < layers; i++)
| |
{
| |
index++;
| |
temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
{
| |
DeleteObj(temp);
| |
ReturnFalse;
| |
}
| |
}
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.Clear();
| |
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
| |
!bEmbeddingAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronHimNetSpatEncoder feed-forward pass and writes |
| |
//| the layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
| |
{
| |
if(bTrain)
| |
if(!cEmbedding.FeedForward())
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL *inputs = NeuronOCL;
| |
CNeuronHimNetGCRU *current = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current ||
| |
!current.feedForward(inputs, Support, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetSpatEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
| |
{
| |
if(!NeuronOCL || !Support)
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL);
| |
CNeuronHimNetGCRU* current = cGRCUs[-1];
| |
if(!current ||
| |
!current.calcInputGradients(inputs, Support, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(cGRCUs.Total() > 1)
| |
{
| |
CBufferFloat *sup = Support.getGradient();
| |
if(!Support.SetGradient(GetPointer(bSupportAccum), false))
| |
ReturnFalse;
| |
CBufferFloat *emb = cEmbedding.getGradient();
| |
if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false))
| |
ReturnFalse;
| |
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
| |
{
| |
current = cGRCUs[i];
| |
inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL);
| |
if(!current ||
| |
!current.calcInputGradients(inputs, Support, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(sup, Support.getGradient(), sup, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
}
| |
if(!Support.SetGradient(sup, false))
| |
ReturnFalse;
| |
if(!cEmbedding.SetGradient(emb, false))
| |
ReturnFalse;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronHimNetSpatEncoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
| |
{
| |
if(!cEmbedding.UpdateInputWeights())
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronHimNetGCRU* current = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current.updateInputWeights(inputs, Support, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronHimNetSpatEncoder weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetSpatEncoder* Source = source;
| |
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronHimNetSpatEncoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cEmbedding.Save(file_handle))
| |
ReturnFalse;
| |
if(!cGRCUs.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetSpatEncoder state from the file |
| |
//| handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!cGRCUs.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
CNeuronHimNetGCRU* temp = cGRCUs[-1];
| |
uint units = temp.GetCount();
| |
uint cheb_k = temp.GetChebK();
| |
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.Clear();
| |
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
| |
!bEmbeddingAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetSpatEncoder GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetSpatEncoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cEmbedding.SetOpenCL(OpenCL);
| |
cGRCUs.SetOpenCL(OpenCL);
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.BufferCreate(OpenCL);
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronHimNetSpatEncoder execution state and its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::Clear(void)
| |
{
| |
CNeuronBaseOCL* temp = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
temp = cGRCUs[i];
| |
if(!temp ||
| |
!temp.Clear())
| |
ReturnFalse;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the count of the CNeuronHimNetSpatEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetSpatEncoder::GetCount(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetCount());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the cheb k of the CNeuronHimNetSpatEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetSpatEncoder::GetChebK(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetChebK());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the window in of the CNeuronHimNetSpatEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetSpatEncoder::GetWindowIn(void) const
| |
{
| |
if(cGRCUs.Total() <= 0)
| |
return(0);
| |
CNeuronHimNetGCRU* temp = cGRCUs[0];
| |
if(!temp)
| |
return(0);
| |
//---
| |
return(temp.GetWindowIn());
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Returns the window out of the CNeuronHimNetSpatEncoder. |
| |
//+------------------------------------------------------------------+
| |
uint CNeuronHimNetSpatEncoder::GetWindowOut(void) const
| |
{
| |
uint count = GetCount();
| |
if(count <= 0)
| |
return(0);
| |
//---
| |
return(Neurons() / count);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the gradient of the CNeuronHimNetSpatEncoder. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetSpatEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
| |
{
| |
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* temp = cGRCUs[-1];
| |
if(!temp ||
| |
!temp.SetGradient(Gradient, delete_prev))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the gradient of the CNeuronHimNetTempEncoder. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetTempEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
| |
{
| |
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* temp = cGRCUs[-1];
| |
if(!temp ||
| |
!temp.SetGradient(Gradient, delete_prev))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetEncoder layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint window_out, uint cheb_k,
| |
uint layers, uint embed_dim, uint period1,
| |
uint timeframe1, uint period2, uint timeframe2,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cEmbedding.Init(0, index, OpenCL, units * embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
SetActivationFunction(TANH);
| |
index++;
| |
if(!cEmbeddingT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch))
| |
ReturnFalse;
| |
cSupport.SetActivationFunction(None);
| |
index++;
| |
if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cNormSupport.SetHeads(units);
| |
cNormSupport.SetActivationFunction(None);
| |
index++;
| |
if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cTempEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, (embed_dim + 1) / 2, period1, timeframe1, period2, timeframe2, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cSpatEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
if(!SetGradient(cTempEncoder.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronHimNetEncoder feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
| |
{
| |
if(!SecondInput)
| |
ReturnFalse;
| |
//---
| |
if(bTrain)
| |
{
| |
if(!cEmbedding.FeedForward())
| |
ReturnFalse;
| |
if(!cEmbeddingT.FeedForward(cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cEmbedding.getOutput(), cEmbeddingT.getOutput(), cSupport.getOutput(),
| |
cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false))
| |
ReturnFalse;
| |
if(!cNormSupport.FeedForward(cSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cPolinomSupport.FeedForward(cNormSupport.AsObject()))
| |
ReturnFalse;
| |
}
| |
//---
| |
if(!cTempEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject(), int(SecondInput[0])))
| |
ReturnFalse;
| |
if(!cSpatEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cTempEncoder.getOutput(), cSpatEncoder.getOutput(), Output,
| |
cTempEncoder.GetWindowOut(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::calcInputGradients(CNeuronBaseOCL *prevLayer)
| |
{
| |
if(!cTempEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
CBufferFloat* temp = cPolinomSupport.getGradient();
| |
CBufferFloat* prev = prevLayer.getGradient();
| |
if(!cPolinomSupport.SetGradient(cPolinomSupport.getPrevOutput(), false) ||
| |
!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
| |
!cSpatEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject()) ||
| |
!SumAndNormalize(temp, cPolinomSupport.getGradient(), temp,
| |
cSpatEncoder.GetCount(), false, 0, 0, 0, 1) ||
| |
!SumAndNormalize(prev, prevLayer.getGradient(), prev,
| |
cSpatEncoder.GetCount(), false, 0, 0, 0, 1) ||
| |
!cPolinomSupport.SetGradient(temp, false) ||
| |
!prevLayer.SetGradient(prev, false))
| |
ReturnFalse;
| |
if(prevLayer.Activation() != None)
| |
if(!DeActivation(prevLayer.getOutput(), prev, prev, prevLayer.Activation()))
| |
ReturnFalse;
| |
//---
| |
if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cEmbedding.getOutput(), cEmbedding.getPrevOutput(),
| |
cEmbeddingT.getOutput(), cEmbeddingT.getGradient(),
| |
cSupport.getGradient(),
| |
cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false))
| |
ReturnFalse;
| |
if(!cEmbedding.CalcHiddenGradients(cEmbeddingT.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cEmbedding.getGradient(), cEmbedding.getPrevOutput(), cEmbedding.getGradient(),
| |
cEmbeddingT.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(cEmbedding.Activation() != None)
| |
if(!DeActivation(cEmbedding.getOutput(), cEmbedding.getGradient(),
| |
cEmbedding.getGradient(), cEmbedding.Activation()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronHimNetEncoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cEmbedding.UpdateInputWeights())
| |
ReturnFalse;
| |
if(!cTempEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cSpatEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronHimNetEncoder weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetEncoder* Source = source;
| |
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cTempEncoder.WeightsUpdate(Source.cTempEncoder.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cSpatEncoder.WeightsUpdate(Source.cSpatEncoder.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronHimNetEncoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cEmbedding.Save(file_handle))
| |
ReturnFalse;
| |
if(!cEmbeddingT.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cNormSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cPolinomSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cTempEncoder.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSpatEncoder.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetEncoder state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEmbeddingT.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSupport.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cNormSupport.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cTempEncoder.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSpatEncoder.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(!SetGradient(cTempEncoder.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetEncoder GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetEncoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
//---
| |
cEmbedding.SetOpenCL(OpenCL);
| |
cEmbeddingT.SetOpenCL(OpenCL);
| |
cSupport.SetOpenCL(OpenCL);
| |
cNormSupport.SetOpenCL(OpenCL);
| |
cPolinomSupport.SetOpenCL(OpenCL);
| |
cTempEncoder.SetOpenCL(OpenCL);
| |
cSpatEncoder.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronHimNetEncoder layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetEncoder::TrainMode(bool flag)
| |
{
| |
CNeuronBaseOCL::TrainMode(flag);
| |
//---
| |
cEmbedding.TrainMode(bTrain);
| |
cEmbeddingT.TrainMode(bTrain);
| |
cSupport.TrainMode(bTrain);
| |
cNormSupport.TrainMode(bTrain);
| |
cPolinomSupport.TrainMode(bTrain);
| |
cTempEncoder.TrainMode(bTrain);
| |
cSpatEncoder.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronHimNetEncoder execution state and its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::Clear(void)
| |
{
| |
if(!CNeuronBaseOCL::Clear())
| |
ReturnFalse;
| |
//---
| |
if(!cEmbedding.Clear())
| |
ReturnFalse;
| |
if(!cEmbeddingT.Clear())
| |
ReturnFalse;
| |
if(!cSupport.Clear())
| |
ReturnFalse;
| |
if(!cNormSupport.Clear())
| |
ReturnFalse;
| |
if(!cPolinomSupport.Clear())
| |
ReturnFalse;
| |
if(!cTempEncoder.Clear())
| |
ReturnFalse;
| |
if(!cSpatEncoder.Clear())
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the gradient of the CNeuronHimNetEncoder. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
| |
{
| |
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
| |
ReturnFalse;
| |
//---
| |
if(!cTempEncoder.SetGradient(Gradient, delete_prev))
| |
ReturnFalse;
| |
if(!cSpatEncoder.SetGradient(Gradient, delete_prev))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronHimNetDecoder layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint window_out,
| |
uint cheb_k, uint layers, uint embed_dim,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cProjection.Init(0, index, OpenCL, window, window, embed_dim, units, 1, optimization, iBatch))
| |
ReturnFalse;
| |
SetActivationFunction(TANH);
| |
index++;
| |
if(!cProjectionT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cEmbedding.Init(0, index, OpenCL, units, units, 1, 1, embed_dim, optimization, iBatch))
| |
ReturnFalse;
| |
SetActivationFunction(SIGMOID);
| |
index++;
| |
if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch))
| |
ReturnFalse;
| |
cSupport.SetActivationFunction(None);
| |
index++;
| |
if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
cNormSupport.SetHeads(units);
| |
cNormSupport.SetActivationFunction(None);
| |
index++;
| |
if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
cGRCUs.Clear();
| |
cGRCUs.SetOpenCL(OpenCL);
| |
index++;
| |
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
{
| |
DeleteObj(temp);
| |
ReturnFalse;
| |
}
| |
for(uint i = 1; i < layers; i++)
| |
{
| |
index++;
| |
temp = new CNeuronHimNetGCRU();
| |
if(!temp ||
| |
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
| |
!cGRCUs.Add(temp))
| |
{
| |
DeleteObj(temp);
| |
ReturnFalse;
| |
}
| |
}
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.Clear();
| |
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
| |
!bEmbeddingAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronHimNetDecoder feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cProjection.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cProjectionT.FeedForward(cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!MatMul(cProjection.getOutput(), cProjectionT.getOutput(), cSupport.getOutput(),
| |
cProjectionT.GetCount(), cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false))
| |
ReturnFalse;
| |
if(!cNormSupport.FeedForward(cSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cPolinomSupport.FeedForward(cNormSupport.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(!cEmbedding.FeedForward(cProjectionT.AsObject()))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetGCRU* current = NULL;
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current ||
| |
!current.feedForward(NeuronOCL, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronHimNetDecoder layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::calcInputGradients(CNeuronBaseOCL *prevLayer)
| |
{
| |
if(!prevLayer)
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : prevLayer);
| |
CNeuronHimNetGCRU* current = cGRCUs[-1];
| |
if(!current ||
| |
!current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(cGRCUs.Total() > 1)
| |
{
| |
CBufferFloat *sup = cPolinomSupport.getGradient();
| |
if(!cPolinomSupport.SetGradient(GetPointer(bSupportAccum), false))
| |
ReturnFalse;
| |
CBufferFloat *emb = cEmbedding.getGradient();
| |
if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false))
| |
ReturnFalse;
| |
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
| |
{
| |
current = cGRCUs[i];
| |
inputs = (i > 0 ? cGRCUs[i - 1] : prevLayer);
| |
if(!current ||
| |
!current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cPolinomSupport.getGradient(), sup, sup, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
}
| |
if(!cPolinomSupport.SetGradient(sup, false))
| |
ReturnFalse;
| |
if(!cEmbedding.SetGradient(emb, false))
| |
ReturnFalse;
| |
}
| |
if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cProjection.getOutput(), cProjection.getPrevOutput(),
| |
cProjectionT.getOutput(), cProjectionT.getPrevOutput(),
| |
cSupport.getGradient(), cProjectionT.GetCount(),
| |
cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false))
| |
ReturnFalse;
| |
if(!cProjectionT.CalcHiddenGradients(cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(cProjectionT.Activation() != None)
| |
if(!DeActivation(cProjectionT.getOutput(), cProjectionT.getPrevOutput(),
| |
cProjectionT.getPrevOutput(), cProjectionT.Activation()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cProjectionT.getGradient(), cProjectionT.getPrevOutput(),
| |
cProjectionT.getGradient(), cProjectionT.GetCount(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!cProjection.CalcHiddenGradients(cProjectionT.AsObject()))
| |
ReturnFalse;
| |
if(cProjection.Activation() != None)
| |
if(!DeActivation(cProjection.getOutput(), cProjection.getPrevOutput(),
| |
cProjection.getPrevOutput(), cProjection.Activation()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cProjection.getGradient(), cProjection.getPrevOutput(),
| |
cProjection.getGradient(), cProjectionT.GetWindow(), false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
//---
| |
CBufferFloat* temp = prevLayer.getGradient();
| |
if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
| |
!prevLayer.CalcHiddenGradients(cProjection.AsObject()) ||
| |
!SumAndNormalize(temp, prevLayer.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
| |
!prevLayer.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronHimNetDecoder trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cProjection.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cEmbedding.UpdateInputWeights(cProjectionT.AsObject()))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* inputs = NeuronOCL;
| |
CNeuronHimNetGCRU* current = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
current = cGRCUs[i];
| |
if(!current.updateInputWeights(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronHimNetDecoder weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronHimNetDecoder* Source = source;
| |
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronHimNetDecoder state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cProjection.Save(file_handle))
| |
ReturnFalse;
| |
if(!cProjectionT.Save(file_handle))
| |
ReturnFalse;
| |
if(!cEmbedding.Save(file_handle))
| |
ReturnFalse;
| |
if(!cSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cNormSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cPolinomSupport.Save(file_handle))
| |
ReturnFalse;
| |
if(!cGRCUs.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronHimNetDecoder state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cProjectionT.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cSupport.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cNormSupport.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject()))
| |
ReturnFalse;
| |
if(!cGRCUs.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.Clear();
| |
CNeuronHimNetGCRU* temp = cGRCUs[-1];
| |
if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) ||
| |
!bSupportAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.Clear();
| |
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
| |
!bEmbeddingAccum.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
| |
if(!SetOutput(temp.getOutput(), true) ||
| |
!SetGradient(temp.getGradient(), true))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronHimNetDecoder GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronHimNetDecoder::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
//---
| |
cProjection.SetOpenCL(OpenCL);
| |
cProjectionT.SetOpenCL(OpenCL);
| |
cEmbedding.SetOpenCL(OpenCL);
| |
cSupport.SetOpenCL(OpenCL);
| |
cNormSupport.SetOpenCL(OpenCL);
| |
cPolinomSupport.SetOpenCL(OpenCL);
| |
cGRCUs.SetOpenCL(OpenCL);
| |
//---
| |
bSupportAccum.BufferFree();
| |
bSupportAccum.BufferCreate(OpenCL);
| |
bEmbeddingAccum.BufferFree();
| |
bEmbeddingAccum.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Clears the CNeuronHimNetDecoder execution state and its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronHimNetDecoder::Clear(void)
| |
{
| |
if(!CNeuronBaseOCL::Clear())
| |
ReturnFalse;
| |
//---
| |
if(!cProjection.Clear())
| |
ReturnFalse;
| |
if(!cProjectionT.Clear())
| |
ReturnFalse;
| |
if(!cEmbedding.Clear())
| |
ReturnFalse;
| |
if(!cSupport.Clear())
| |
ReturnFalse;
| |
if(!cNormSupport.Clear())
| |
ReturnFalse;
| |
if(!cPolinomSupport.Clear())
| |
ReturnFalse;
| |
CNeuronBaseOCL* temp = NULL;
| |
for(int i = 0; i < cGRCUs.Total(); i++)
| |
{
| |
temp = cGRCUs[i];
| |
if(!temp || !temp.Clear())
| |
ReturnFalse;
| |
}
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronFastGConv layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint sparse_dimension,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
int index = 0;
| |
if(!cInpAndHidden.Init(0, index, OpenCL, 2 * units * window, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cZ_R.Init(0, index, OpenCL, 2 * window, 2 * window, 2 * window, units, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cZ_R.SetActivationFunction(SIGMOID);
| |
index++;
| |
if(!cZ.Init(0, index, OpenCL, window * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cR.Init(0, index, OpenCL, window * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cCandidate.Init(0, index, OpenCL, 2 * window * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cHC.Init(0, index, OpenCL, 2 * window, 2 * window, window, units, 1, optimization, iBatch))
| |
ReturnFalse;
| |
cHC.SetActivationFunction(TANH);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronFastGConv random walk operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::RandomWalk(CBufferFloat *data,
| |
CBufferFloat *normal,
| |
CBufferFloat *inv_diag,
| |
const int rows,
| |
const int cols)
| |
{
| |
if(!OpenCL)
| |
ReturnFalse;
| |
uint global_work_offset[2] = {0, 0};
| |
uint global_work_size[2] = {(uint)rows, MathMin((uint)cols, (uint)OpenCL.GetMaxLocalSize(1))};
| |
uint local_work_size[2] = {1, global_work_size[1]};
| |
uint kernel = def_k_RandomWalk;
| |
setBuffer(kernel, def_k_rw_data, data.GetIndex())
| |
setBuffer(kernel, def_k_rw_norm, normal.GetIndex())
| |
setBuffer(kernel, def_k_rw_inv_diag, inv_diag.GetIndex())
| |
setArgument(kernel, def_k_rw_total_cols, cols)
| |
//---
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!inv_diag.BufferRead())
| |
ReturnFalse;
| |
#endif
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronFastGConv feed forward operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::FeedForward(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent)
| |
{
| |
if(!SourceData || !SparseAttent)
| |
ReturnFalse;
| |
//---
| |
const uint units = GetCount();
| |
const uint window = GetWindow();
| |
const uint sparse = GetSparseDimension();
| |
//---
| |
if(!RandomWalk(SparseAttent.getOutput(), cNormAttention.getOutput(), cInvDiag.getOutput(), window, sparse))
| |
ReturnFalse;
| |
if(!SwapOutputs())
| |
ReturnFalse;
| |
if(!Concat(SourceData.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window, units))
| |
ReturnFalse;
| |
//---
| |
if(!SparseMatMul(SparseAttent.GetIndexes(), cNormAttention.getOutput(), cInpAndHidden.getOutput(),
| |
cAX.getOutput(), units, sparse, units, 2 * window))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cAX.getOutput(), cInpAndHidden.getOutput(), cAXplusX.getOutput(), 2 * window, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!DiagMatMul(cInvDiag.getOutput(), cAXplusX.getOutput(), cNormAXplusX.getOutput(), units, 2 * window, 1, None))
| |
ReturnFalse;
| |
if(!cZ_R.FeedForward(cNormAXplusX.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZ_R.getOutput(), window, window, units))
| |
ReturnFalse;
| |
if(!ElementMult(cR.getOutput(), PrevOutput, cR.getPrevOutput()))
| |
ReturnFalse;
| |
if(!Concat(SourceData.getOutput(), cR.getPrevOutput(), cCandidate.getOutput(), window, window, units))
| |
ReturnFalse;
| |
if(!cHC.FeedForward(cCandidate.AsObject()))
| |
ReturnFalse;
| |
if(!GateElementMult(PrevOutput, cHC.getOutput(), cZ.getOutput(), Output))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Handles the CNeuronFastGConv calc input gradients operation. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::CalcInputGradients(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent)
| |
{
| |
if(!SourceData || !SparseAttent)
| |
ReturnFalse;
| |
//---
| |
const uint units = GetCount();
| |
const uint window = GetWindow();
| |
const uint sparse = GetSparseDimension();
| |
//---
| |
if(!GateElementMultGrad(PrevOutput, cInpAndHidden.getGradient(),
| |
cHC.getOutput(), cHC.getGradient(),
| |
cZ.getOutput(), cZ.getGradient(),
| |
Gradient, None, cHC.Activation(), cZ_R.Activation()))
| |
ReturnFalse;
| |
if(!cCandidate.CalcHiddenGradients(cHC.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(SourceData.getGradient(), cR.getPrevOutput(),
| |
cCandidate.getGradient(), window, window, units))
| |
ReturnFalse;
| |
if(!ElementMultGrad(cR.getOutput(), cR.getGradient(),
| |
PrevOutput, cInpAndHidden.getGradient(),
| |
cR.getPrevOutput(), cZ_R.Activation(), None))
| |
ReturnFalse;
| |
if(!Concat(cZ.getGradient(), cR.getGradient(),
| |
cZ_R.getGradient(), window, window, units))
| |
ReturnFalse;
| |
if(!cNormAXplusX.CalcHiddenGradients(cZ_R.AsObject()))
| |
ReturnFalse;
| |
if(!DiagMatMulGrad(cInvDiag.getOutput(), cInvDiag.getGradient(),
| |
cAXplusX.getOutput(), cAX.getGradient(),
| |
cNormAXplusX.getGradient(), units, 2 * window, 1))
| |
ReturnFalse;
| |
if(!SparseMatMulGrad(SparseAttent.GetIndexes(), cNormAttention.getOutput(),
| |
cNormAttention.getGradient(), cInpAndHidden.getOutput(),
| |
cInpAndHidden.getGradient(), cAX.getGradient(),
| |
units, sparse, units, 2 * window))
| |
ReturnFalse;
| |
if(!SumAndNormalize(cAX.getGradient(), cInpAndHidden.getGradient(),
| |
cInpAndHidden.getGradient(), 2 * window, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(!DeConcat(cAXplusX.getGradient(), cAXplusX.getPrevOutput(),
| |
cInpAndHidden.getGradient(), window, window, units))
| |
ReturnFalse;
| |
if(!SumAndNormalize(SourceData.getGradient(), cAXplusX.getGradient(),
| |
SourceData.getGradient(), window, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
if(SourceData.Activation() != None)
| |
if(!DeActivation(SourceData.getOutput(), SourceData.getGradient(),
| |
SourceData.getGradient(), SourceData.Activation()))
| |
ReturnFalse;
| |
if(!DiagMatMul(cInvDiag.getOutput(), cNormAttention.getGradient(),
| |
SparseAttent.getGradient(), units, sparse, 1, None))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronFastGConv trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
//---
| |
if(!cZ_R.UpdateInputWeights(cNormAXplusX.AsObject()))
| |
ReturnFalse;
| |
if(!cHC.UpdateInputWeights(cCandidate.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronFastGConv weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronFastGConv *Source = source;
| |
if(!cZ_R.WeightsUpdate(Source.cZ_R.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cHC.WeightsUpdate(Source.cHC.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronFastGConv state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cZ_R.Save(file_handle))
| |
ReturnFalse;
| |
if(!cHC.Save(file_handle))
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, (int)GetSparseDimension()) < INT_VALUE)
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronFastGConv state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronFastGConv::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cZ_R.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cHC.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
uint sparse_dimension = FileReadInteger(file_handle);
| |
uint units = cZ_R.GetUnits();
| |
uint window_out = Neurons() / units;
| |
uint window = cZ_R.GetWindow() - window_out;
| |
//---
| |
int index = 0;
| |
if(!cInpAndHidden.Init(0, index, OpenCL, units * (window + window_out), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
| |
ReturnFalse;
| |
index += 2;
| |
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cCandidate.Init(0, index, OpenCL, (window + window_out) * units, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronFastGConv GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronFastGConv::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
//---
| |
cZ_R.SetOpenCL(OpenCL);
| |
cHC.SetOpenCL(OpenCL);
| |
cInpAndHidden.SetOpenCL(OpenCL);
| |
cNormAttention.SetOpenCL(OpenCL);
| |
cInvDiag.SetOpenCL(OpenCL);
| |
cAX.SetOpenCL(OpenCL);
| |
cAXplusX.SetOpenCL(OpenCL);
| |
cNormAXplusX.SetOpenCL(OpenCL);
| |
cZ.SetOpenCL(OpenCL);
| |
cR.SetOpenCL(OpenCL);
| |
cCandidate.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSAGDFN layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint time_steps, uint variables, uint embedding_dim,
| |
uint emb_layers, uint sparse_dimension, uint heads,
| |
float sparse, uint gcru_layers,
| |
uint forecast, uint forec_layers,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(emb_layers <= 0 || gcru_layers <= 0 || forec_layers <= 0)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables, forecast, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
uint index = 0;
| |
if(!cTranspose.Init(0, index, OpenCL, time_steps, variables, optimization, iBatch))
| |
ReturnFalse;
| |
//--- Embedding
| |
cEmbedding.Clear();
| |
cEmbedding.SetOpenCL(OpenCL);
| |
index++;
| |
CNeuronConvOCL *conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, time_steps, time_steps, embedding_dim, variables, 1, optimization, iBatch) ||
| |
!cEmbedding.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
for(uint i = 1; i < emb_layers; i++)
| |
{
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, embedding_dim, embedding_dim,
| |
embedding_dim, variables, 1, optimization, iBatch) ||
| |
!cEmbedding.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
}
| |
CNeuronBatchNormOCL *norm = new CNeuronBatchNormOCL();
| |
index++;
| |
if(!norm ||
| |
!norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) ||
| |
!cEmbedding.Add(norm))
| |
{
| |
DeleteObj(norm);
| |
ReturnFalse;
| |
}
| |
norm.SetActivationFunction(None);
| |
//--- GCRUs
| |
cGCRU.Clear();
| |
cGCRU.SetOpenCL(OpenCL);
| |
index++;
| |
if(!cAttention.Init(0, index, OpenCL, variables, embedding_dim, heads,
| |
sparse_dimension, sparse, optimization, iBatch))
| |
ReturnFalse;
| |
CNeuronFastGConv *gcru = NULL;
| |
for(uint i = 0; i < gcru_layers; i++)
| |
{
| |
index++;
| |
gcru = new CNeuronFastGConv();
| |
if(!gcru ||
| |
!gcru.Init(0, index, OpenCL, variables, embedding_dim, sparse_dimension, optimization, iBatch) ||
| |
!cGCRU.Add(gcru))
| |
{
| |
DeleteObj(gcru);
| |
ReturnFalse;
| |
}
| |
}
| |
//--- Forecast
| |
cProjection.Clear();
| |
cProjection.SetOpenCL(OpenCL);
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, embedding_dim, embedding_dim,
| |
forecast, variables, 1, optimization, iBatch) ||
| |
!cProjection.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
for(uint i = 1; i < forec_layers; i++)
| |
{
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, forecast, forecast,
| |
forecast, variables, 1, optimization, iBatch) ||
| |
!cProjection.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
}
| |
norm = new CNeuronBatchNormOCL();
| |
index++;
| |
if(!norm ||
| |
!norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) ||
| |
!cProjection.Add(norm))
| |
{
| |
DeleteObj(norm);
| |
ReturnFalse;
| |
}
| |
norm.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSAGDFN feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cTranspose.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
CNeuronBaseOCL *inputs = cTranspose.AsObject();
| |
CNeuronBaseOCL *current = NULL;
| |
//--- Embedding
| |
for(int i = 0; i < cEmbedding.Total(); i++)
| |
{
| |
current = cEmbedding[i];
| |
if(!current ||
| |
!current.FeedForward(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//--- GCRUs
| |
if(!cAttention.FeedForward(inputs))
| |
ReturnFalse;
| |
CNeuronFastGConv *gcru = NULL;
| |
for(int i = 0; i < cGCRU.Total(); i++)
| |
{
| |
gcru = cGCRU[i];
| |
if(!gcru ||
| |
!gcru.FeedForward(inputs, cAttention.AsObject()))
| |
ReturnFalse;
| |
inputs = gcru;
| |
}
| |
//--- Forecast
| |
for(int i = 0; i < cProjection.Total(); i++)
| |
{
| |
current = cProjection[i];
| |
if(!current ||
| |
!current.FeedForward(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//--- result
| |
return(CNeuronTransposeOCL::feedForward(inputs));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronSAGDFN |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronTransposeOCL::calcInputGradients(cProjection[-1]))
| |
ReturnFalse;
| |
//--- Forecast
| |
CNeuronBaseOCL *inputs = NULL;
| |
for(int i = cProjection.Total() - 1; i >= 0; i--)
| |
{
| |
inputs = (i > 0 ? cProjection[i - 1] : cGCRU[-1]);
| |
if(!inputs ||
| |
!inputs.CalcHiddenGradients(cProjection[i]))
| |
ReturnFalse;
| |
}
| |
//--- GCRU
| |
CNeuronFastGConv *gcru = cGCRU[-1];
| |
int layers = cGCRU.Total();
| |
inputs = (layers > 1 ? cGCRU[-2] : cEmbedding[-1]);
| |
if(!gcru ||
| |
!gcru.CalcInputGradients(inputs, cAttention.AsObject()))
| |
ReturnFalse;
| |
if(layers > 1)
| |
{
| |
CBufferFloat *temp = cAttention.getGradient();
| |
if(!cAttention.SetGradient(cAttention.getPrevOutput(), false))
| |
ReturnFalse;
| |
for(int i = layers - 2; i >= 0; i--)
| |
{
| |
inputs = (i > 0 ? cGCRU[i - 1] : cEmbedding[-1]);
| |
gcru = cGCRU[i];
| |
if(!gcru ||
| |
!gcru.CalcInputGradients(inputs, cAttention.AsObject()) ||
| |
!SumAndNormalize(temp, cAttention.getGradient(), temp, 1, false, 0, 0, 0, 1))
| |
ReturnFalse;
| |
}
| |
if(!cAttention.SetGradient(temp, false))
| |
ReturnFalse;
| |
}
| |
CBufferFloat *temp = inputs.getGradient();
| |
if(!inputs.SetGradient(inputs.getPrevOutput(), false) ||
| |
!inputs.CalcHiddenGradients(cAttention.AsObject()) ||
| |
!SumAndNormalize(temp, inputs.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
| |
!inputs.SetGradient(temp, false))
| |
ReturnFalse;
| |
//--- Embedding
| |
for(int i = cEmbedding.Total() - 2; i >= 0; i--)
| |
{
| |
inputs = cEmbedding[i];
| |
if(!inputs ||
| |
!inputs.CalcHiddenGradients(cEmbedding[i + 1]))
| |
ReturnFalse;
| |
}
| |
//---
| |
if(!cTranspose.CalcHiddenGradients(inputs))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronSAGDFN trainable weights from the accumulated |
| |
//| gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
CNeuronBaseOCL *inputs = cTranspose.AsObject();
| |
CNeuronBaseOCL *current = NULL;
| |
//--- Embedding
| |
for(int i = 0; i < cEmbedding.Total(); i++)
| |
{
| |
current = cEmbedding[i];
| |
if(!current ||
| |
!current.UpdateInputWeights(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//--- GCRUs
| |
if(!cAttention.UpdateInputWeights(inputs))
| |
ReturnFalse;
| |
for(int i = 0; i < cGCRU.Total(); i++)
| |
{
| |
current = cGCRU[i];
| |
if(!current ||
| |
!current.UpdateInputWeights(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//--- Forecast
| |
for(int i = 0; i < cProjection.Total(); i++)
| |
{
| |
current = cProjection[i];
| |
if(!current ||
| |
!current.UpdateInputWeights(inputs))
| |
ReturnFalse;
| |
inputs = current;
| |
}
| |
//--- result
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronSAGDFN weights toward the source model with |
| |
//| factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
CNeuronSAGDFN *Source = source;
| |
//--- Embedding
| |
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
| |
ReturnFalse;
| |
//--- GCRUs
| |
if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cGCRU.WeightsUpdate(Source.cGCRU.AsObject(), tau))
| |
ReturnFalse;
| |
//--- Forecast
| |
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
| |
ReturnFalse;
| |
//--- result
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSAGDFN state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::Save(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cTranspose.Save(file_handle))
| |
ReturnFalse;
| |
//--- Embedding
| |
if(!cEmbedding.Save(file_handle))
| |
ReturnFalse;
| |
//--- GCRUs
| |
if(!cGCRU.Save(file_handle) ||
| |
!cAttention.Save(file_handle))
| |
ReturnFalse;
| |
//--- Forecast
| |
if(!cProjection.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSAGDFN state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSAGDFN::Load(const int file_handle)
| |
{
| |
if(!CNeuronTransposeOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
| |
ReturnFalse;
| |
//--- Embedding
| |
if(!cEmbedding.Load(file_handle))
| |
ReturnFalse;
| |
//--- GCRUs
| |
if(!cGCRU.Load(file_handle) ||
| |
!LoadInsideLayer(file_handle, cAttention.AsObject()))
| |
ReturnFalse;
| |
//--- Forecast
| |
if(!cProjection.Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSAGDFN GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSAGDFN::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronTransposeOCL::SetOpenCL(obj);
| |
//---
| |
cTranspose.SetOpenCL(OpenCL);
| |
cEmbedding.SetOpenCL(OpenCL);
| |
cGCRU.SetOpenCL(OpenCL);
| |
cAttention.SetOpenCL(OpenCL);
| |
cProjection.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSpatialEmbedding layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint embed_dim,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + embed_dim)*units, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
if(!cEmbedding.Init(0, 0, OpenCL, embed_dim * units, optimization, iBatch))
| |
ReturnFalse;
| |
cEmbedding.SetActivationFunction(TANH);
| |
//---
| |
iUnits = units;
| |
iWindow = window;
| |
iEmbeddingDim = embed_dim;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSpatialEmbedding feed-forward pass and writes |
| |
//| the layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(bTrain)
| |
if(!cEmbedding.FeedForward())
| |
ReturnFalse;
| |
//---
| |
if(!Concat(NeuronOCL.getOutput(), cEmbedding.getOutput(), Output, iWindow, iEmbeddingDim, iUnits))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronSpatialEmbedding layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
if(!DeConcat(NeuronOCL.getGradient(), cEmbedding.getGradient(), Gradient, iWindow, iEmbeddingDim, iUnits))
| |
ReturnFalse;
| |
Deactivation(NeuronOCL)
| |
Deactivation(cEmbedding)
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronSpatialEmbedding trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cEmbedding.UpdateInputWeights())
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronSpatialEmbedding weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronSpatialEmbedding *Source = source;
| |
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSpatialEmbedding state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cEmbedding.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE)
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE)
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, (int)iEmbeddingDim) < INT_VALUE)
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSpatialEmbedding state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSpatialEmbedding::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iUnits = (uint)FileReadInteger(file_handle);
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iWindow = (uint)FileReadInteger(file_handle);
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iEmbeddingDim = (uint)FileReadInteger(file_handle);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSpatialEmbedding GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSpatialEmbedding::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
cEmbedding.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronGraphons layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint emb_dimension,
| |
uint experts, float dropout,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * units, optimization_type, batch))
| |
ReturnFalse;
| |
activation = None;
| |
//---
| |
int index = 0;
| |
if(!cGraphs.Init(0, index, OpenCL, Neurons()*experts, optimization, iBatch))
| |
ReturnFalse;
| |
cGraphs.SetActivationFunction(SIGMOID);
| |
//---
| |
CNeuronBaseOCL *neuron = NULL;
| |
CNeuronConvOCL *conv = NULL;
| |
CNeuronTransposeOCL *transp = NULL;
| |
CNeuronDropoutOCL *dout = NULL;
| |
CNeuronSoftMaxOCL *softmax = NULL;
| |
//--- Probability
| |
acProbability.Clear();
| |
acProbability.SetOpenCL(OpenCL);
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(experts, index, OpenCL, window, window, experts, units, 1, optimization, iBatch) ||
| |
!acProbability.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
index++;
| |
neuron = new CNeuronBaseOCL();
| |
if(!neuron ||
| |
!neuron.Init(0, index, OpenCL, experts, optimization, iBatch) ||
| |
!acProbability.Add(neuron))
| |
{
| |
DeleteObj(neuron);
| |
ReturnFalse;
| |
}
| |
index++;
| |
dout = new CNeuronDropoutOCL();
| |
if(!dout ||
| |
!dout.Init(0, index, OpenCL, experts, dropout, optimization, iBatch) ||
| |
!acProbability.Add(dout))
| |
{
| |
DeleteObj(dout);
| |
ReturnFalse;
| |
}
| |
index++;
| |
softmax = new CNeuronSoftMaxOCL();
| |
if(!softmax ||
| |
!softmax.Init(0, index, OpenCL, experts, optimization, iBatch) ||
| |
!acProbability.Add(softmax))
| |
{
| |
DeleteObj(softmax);
| |
ReturnFalse;
| |
}
| |
softmax.SetHeads(1);
| |
//--- Data Embedding
| |
acDataEmb.Clear();
| |
acDataEmb.SetOpenCL(OpenCL);
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, window, window, 2 * emb_dimension, units, 1, optimization, iBatch) ||
| |
!acDataEmb.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(SoftPlus);
| |
index++;
| |
conv = new CNeuronConvOCL();
| |
if(!conv ||
| |
!conv.Init(0, index, OpenCL, 2 * emb_dimension, 2 * emb_dimension, emb_dimension, units, 1, optimization, iBatch) ||
| |
!acDataEmb.Add(conv))
| |
{
| |
DeleteObj(conv);
| |
ReturnFalse;
| |
}
| |
conv.SetActivationFunction(None);
| |
index++;
| |
transp = new CNeuronTransposeOCL();
| |
if(!transp ||
| |
!transp.Init(0, index, OpenCL, units, emb_dimension, optimization, iBatch) ||
| |
!acDataEmb.Add(transp))
| |
{
| |
DeleteObj(transp);
| |
ReturnFalse;
| |
}
| |
transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
| |
//---
| |
index++;
| |
if(!cExpertsEmb.Init(0, index, OpenCL, units * emb_dimension * experts, optimization, iBatch))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronGraphons feed-forward pass and writes the layer |
| |
//| output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
CNeuronBaseOCL* prev = NeuronOCL;
| |
CNeuronBaseOCL* current = NULL;
| |
//--- Probability
| |
for(int i = 0; i < acProbability.Total(); i++)
| |
{
| |
current = acProbability[i];
| |
if(!current ||
| |
!current.FeedForward(prev))
| |
ReturnFalse;
| |
prev = current;
| |
}
| |
//--- Data Embedding
| |
prev = NeuronOCL;
| |
for(int i = 0; i < acDataEmb.Total(); i++)
| |
{
| |
current = acDataEmb[i];
| |
if(!current ||
| |
!current.FeedForward(prev))
| |
ReturnFalse;
| |
prev = current;
| |
}
| |
//--- Experts
| |
if(bTrain)
| |
if(!cExpertsEmb.FeedForward())
| |
ReturnFalse;
| |
//--- Graphs
| |
uint units = (uint)MathSqrt((double)Neurons());
| |
uint emb_dim = acDataEmb[-1].Neurons() / units;
| |
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
| |
if(!MatMul(cExpertsEmb.getOutput(), current.getOutput(), cGraphs.getOutput(),
| |
units, emb_dim, units, experts, false))
| |
ReturnFalse;
| |
if(cGraphs.Activation() != None)
| |
if(!Activation(cGraphs.getOutput(), cGraphs.getOutput(), cGraphs.Activation()))
| |
ReturnFalse;
| |
//--- Result
| |
if(!MatMul(acProbability[-1].getOutput(), cGraphs.getOutput(), Output, 1, experts, units * units, 1, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the CNeuronGraphons |
| |
//| layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!acDataEmb[-1] || !acProbability[-1])
| |
ReturnFalse;
| |
//---
| |
uint units = (uint)MathSqrt((double)Neurons());
| |
uint emb_dim = acDataEmb[-1].Neurons() / units;
| |
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
| |
//---
| |
if(!MatMulGrad(acProbability[-1].getOutput(), acProbability[-1].getGradient(),
| |
cGraphs.getOutput(), cGraphs.getGradient(), Gradient,
| |
1, experts, units * units, 1, false))
| |
ReturnFalse;
| |
//--- Graphs
| |
if(cGraphs.Activation() != None)
| |
if(!DeActivation(cGraphs.getOutput(), cGraphs.getGradient(),
| |
cGraphs.getGradient(), cGraphs.Activation()))
| |
ReturnFalse;
| |
if(!MatMulGrad(cExpertsEmb.getOutput(), cExpertsEmb.getGradient(),
| |
acDataEmb[-1].getOutput(), acDataEmb[-1].getGradient(),
| |
cGraphs.getGradient(), units, emb_dim, units, experts, false))
| |
ReturnFalse;
| |
//---
| |
CNeuronBaseOCL* next = NULL;
| |
CNeuronBaseOCL* current = NULL;
| |
//--- Probability
| |
for(int i = acProbability.Total() - 1; i >= 0; i--)
| |
{
| |
next = acProbability[i];
| |
current = (i == 0 ? NeuronOCL : acProbability[i - 1]);
| |
if(!current ||
| |
!current.CalcHiddenGradients(next))
| |
ReturnFalse;
| |
}
| |
//--- Data Embedding
| |
CBufferFloat *temp = NeuronOCL.getGradient();
| |
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false))
| |
ReturnFalse;
| |
for(int i = acDataEmb.Total() - 1; i >= 0; i--)
| |
{
| |
next = acDataEmb[i];
| |
current = (i == 0 ? NeuronOCL : acDataEmb[i - 1]);
| |
if(!current ||
| |
!current.CalcHiddenGradients(next))
| |
ReturnFalse;
| |
}
| |
if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
| |
!NeuronOCL.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronGraphons trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
CNeuronBaseOCL* prev = NeuronOCL;
| |
CNeuronBaseOCL* current = NULL;
| |
//--- Probability
| |
for(int i = 0; i < acProbability.Total(); i++)
| |
{
| |
current = acProbability[i];
| |
if(!current ||
| |
!current.UpdateInputWeights(prev))
| |
ReturnFalse;
| |
prev = current;
| |
}
| |
//--- Data Embedding
| |
prev = NeuronOCL;
| |
for(int i = 0; i < acDataEmb.Total(); i++)
| |
{
| |
current = acDataEmb[i];
| |
if(!current ||
| |
!current.UpdateInputWeights(prev))
| |
ReturnFalse;
| |
prev = current;
| |
}
| |
//--- Experts
| |
if(!cExpertsEmb.UpdateInputWeights())
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronGraphons weights toward the source model |
| |
//| with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronGraphons *Source = source;
| |
if(!acProbability.WeightsUpdate(Source.acProbability.AsObject(), tau))
| |
ReturnFalse;
| |
if(!acDataEmb.WeightsUpdate(Source.acDataEmb.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cExpertsEmb.WeightsUpdate(Source.cExpertsEmb.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronGraphons state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::Save(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!acProbability.Save(file_handle))
| |
ReturnFalse;
| |
if(!acDataEmb.Save(file_handle))
| |
ReturnFalse;
| |
if(!cExpertsEmb.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronGraphons state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGraphons::Load(const int file_handle)
| |
{
| |
if(!CNeuronBaseOCL::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!acProbability.Load(file_handle))
| |
ReturnFalse;
| |
if(!acDataEmb.Load(file_handle))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cExpertsEmb.AsObject()))
| |
ReturnFalse;
| |
//---
| |
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
| |
if(!cGraphs.Init(0, 0, OpenCL, Neurons()*experts, optimization, iBatch))
| |
ReturnFalse;
| |
cGraphs.SetActivationFunction(SIGMOID);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronGraphons GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGraphons::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronBaseOCL::SetOpenCL(obj);
| |
acProbability.SetOpenCL(OpenCL);
| |
acDataEmb.SetOpenCL(OpenCL);
| |
cExpertsEmb.SetOpenCL(OpenCL);
| |
cGraphs.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronGraphons layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGraphons::TrainMode(bool flag)
| |
{
| |
CNeuronBaseOCL::TrainMode(flag);
| |
acProbability.TrainMode(bTrain);
| |
acDataEmb.TrainMode(bTrain);
| |
cExpertsEmb.TrainMode(bTrain);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronSparseSoftMax layer and allocates its GPU |
| |
//| buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint dimension_in, uint dimension_out,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(dimension_in < dimension_out)
| |
ReturnFalse;
| |
if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, units * dimension_out, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
SetHeads(units);
| |
iDimensionIn = dimension_in;
| |
if(!cIndexes.BufferInit(Neurons(), -1) ||
| |
!cIndexes.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronSparseSoftMax feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!OpenCL || !NeuronOCL ||
| |
NeuronOCL.Neurons() < int(iHeads * iDimensionIn))
| |
ReturnFalse;
| |
uint global_work_offset[2] = {0, 0};
| |
uint global_work_size[2] = { iHeads, iDimensionIn };
| |
uint local_work_size[2] = { 1, global_work_size[1] };
| |
uint kernel = def_k_SparseSoftMax;
| |
setBuffer(kernel, def_k_ssoftmax_data, NeuronOCL.getOutputIndex())
| |
setBuffer(kernel, def_k_ssoftmax_outputs, getOutputIndex())
| |
setBuffer(kernel, def_k_ssoftmax_indexes, cIndexes.GetIndex())
| |
setArgument(kernel, def_k_ssoftmax_out_dimension, int(DimensionOut()))
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!Output.BufferRead())
| |
ReturnFalse;
| |
#endif
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronSparseSoftMax layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!OpenCL || !NeuronOCL ||
| |
NeuronOCL.Neurons() < int(iHeads * iDimensionIn))
| |
ReturnFalse;
| |
uint global_work_offset[2] = {0, 0};
| |
uint global_work_size[2] = { iHeads, iDimensionIn };
| |
uint local_work_size[2] = { 1, global_work_size[1] };
| |
uint kernel = def_k_SparseSoftMaxGrad;
| |
setBuffer(kernel, def_k_ssoftmax_gr_data_gr, NeuronOCL.getGradientIndex())
| |
setBuffer(kernel, def_k_ssoftmax_gr_outputs, getOutputIndex())
| |
setBuffer(kernel, def_k_ssoftmax_gr_outputs_gr, getGradientIndex())
| |
setBuffer(kernel, def_k_ssoftmax_gr_indexes, cIndexes.GetIndex())
| |
setArgument(kernel, def_k_ssoftmax_gr_out_dimension, int(DimensionOut()))
| |
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
| |
#ifdef _DEBUG
| |
if(!NeuronOCL.getGradient().BufferRead())
| |
ReturnFalse;
| |
#endif
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronSparseSoftMax state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::Save(const int file_handle)
| |
{
| |
if(!CNeuronSoftMaxOCL::Save(file_handle))
| |
ReturnFalse;
| |
if(FileWriteInteger(file_handle, iDimensionIn) < INT_VALUE)
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronSparseSoftMax state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::Load(const int file_handle)
| |
{
| |
if(!CNeuronSoftMaxOCL::Load(file_handle))
| |
ReturnFalse;
| |
if(FileIsEnding(file_handle))
| |
ReturnFalse;
| |
iDimensionIn = (uint)FileReadInteger(file_handle);
| |
//---
| |
cIndexes.BufferFree();
| |
if(!cIndexes.BufferInit(Neurons(), -1) ||
| |
!cIndexes.BufferCreate(OpenCL))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronSparseSoftMax GPU buffers to the given OpenCL |
| |
//| context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronSparseSoftMax::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronSoftMaxOCL::SetOpenCL(obj);
| |
cIndexes.BufferCreate(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Reports checked-transfer support for SparseSoftMax. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::SupportsOpenCLChecked(void) const
| |
{
| |
//--- Reject subtypes with an unverified ownership tree.
| |
return(Type() == defNeuronSparseSoftMax);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Transfers sparse indexes before publishing the base context. |
| |
//+------------------------------------------------------------------+
| |
//| Includes the sparse selector's only additional buffer |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::OwnsBufferOrDeviceSlot(CBufferFloat *candidate)
| |
{
| |
return(CNeuronBaseOCL::OwnsBufferOrDeviceSlot(candidate) ||
| |
BufferAliases(candidate, GetPointer(cIndexes)));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Function CNeuronSparseSoftMax::SetOpenCLChecked |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronSparseSoftMax::SetOpenCLChecked(COpenCLMy *obj)
| |
{
| |
//--- Validate the target and exact ownership contract before mutation.
| |
if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked())
| |
ReturnFalseEx("checked context target or sparse ownership");
| |
//--- Preserve allocated index data from its current device owner.
| |
if(cIndexes.Total() > 0 && cIndexes.GetIndex() >= 0 &&
| |
cIndexes.GetOpenCL() != obj && !cIndexes.BufferRead())
| |
ReturnFalseEx("sparse indexes read");
| |
//--- Do not overwrite already transferred indexes from a stale host cache.
| |
if(cIndexes.Total() > 0 &&
| |
(cIndexes.GetOpenCL() != obj || cIndexes.GetIndex() < 0) &&
| |
!cIndexes.BufferCreate(obj))
| |
ReturnFalseEx("sparse indexes create");
| |
//--- Publish the inherited context only after index transfer succeeds.
| |
if(!CNeuronBaseOCL::SetOpenCLChecked(obj))
| |
ReturnFalseEx("base context transfer");
| |
//--- Finalize without resetting the sparse configuration.
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Initializes the CNeuronGlobLocGraphAtt layer and allocates its |
| |
//| GPU buffers. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
| |
uint units, uint window, uint experts, float dropout,
| |
uint emb_dimension, uint sparse_dimension,
| |
ENUM_OPTIMIZATION optimization_type, uint batch)
| |
{
| |
if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, window, units,
| |
2, optimization_type, batch))
| |
ReturnFalse;
| |
//---
| |
int index = 0;
| |
if(!cGlobal.Init(0, index, OpenCL, iUnits, iWindow, emb_dimension, experts, dropout,
| |
optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cLocal.Init(0, index, OpenCL, iUnits, iWindow, experts, dropout, emb_dimension,
| |
sparse_dimension, optimization, iBatch))
| |
ReturnFalse;
| |
index++;
| |
if(!cConcat.Init(0, index, OpenCL, 2 * iWindow * iUnits, optimization, iBatch))
| |
ReturnFalse;
| |
cConcat.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Runs the CNeuronGlobLocGraphAtt feed-forward pass and writes the |
| |
//| layer output. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::feedForward(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cGlobal.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!cLocal.FeedForward(NeuronOCL))
| |
ReturnFalse;
| |
if(!Concat(cGlobal.getOutput(), cLocal.getOutput(), cConcat.getOutput(),
| |
iWindow, iWindow, iUnits))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronMHAttentionPooling::feedForward(cConcat.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Back-propagates the input gradients through the |
| |
//| CNeuronGlobLocGraphAtt layer. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!NeuronOCL)
| |
ReturnFalse;
| |
//---
| |
if(!CNeuronMHAttentionPooling::calcInputGradients(cConcat.AsObject()))
| |
ReturnFalse;
| |
if(!DeConcat(cGlobal.getGradient(), cLocal.getGradient(), cConcat.getGradient(),
| |
iWindow, iWindow, iUnits))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cGlobal.AsObject()))
| |
ReturnFalse;
| |
CBufferFloat *temp = NeuronOCL.getGradient();
| |
if(!NeuronOCL.SetGradient(PrevOutput, false))
| |
ReturnFalse;
| |
if(!NeuronOCL.CalcHiddenGradients(cLocal.AsObject()))
| |
ReturnFalse;
| |
if(!SumAndNormalize(temp, PrevOutput, temp, iWindow, false, 0, 0, 0, 1) ||
| |
!NeuronOCL.SetGradient(temp, false))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Updates the CNeuronGlobLocGraphAtt trainable weights from the |
| |
//| accumulated gradients. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
| |
{
| |
if(!cGlobal.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
if(!cLocal.UpdateInputWeights(NeuronOCL))
| |
ReturnFalse;
| |
//---
| |
return(CNeuronMHAttentionPooling::updateInputWeights(cConcat.AsObject()));
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Averages the CNeuronGlobLocGraphAtt weights toward the source |
| |
//| model with factor tau. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::WeightsUpdate(CNeuronBaseOCL *source, float tau)
| |
{
| |
if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau))
| |
ReturnFalse;
| |
//---
| |
CNeuronGlobLocGraphAtt* Source = source;
| |
if(!cGlobal.WeightsUpdate(Source.cGlobal.AsObject(), tau))
| |
ReturnFalse;
| |
if(!cLocal.WeightsUpdate(Source.cLocal.AsObject(), tau))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Saves the CNeuronGlobLocGraphAtt state to the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::Save(const int file_handle)
| |
{
| |
if(!CNeuronMHAttentionPooling::Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!cGlobal.Save(file_handle))
| |
ReturnFalse;
| |
if(!cLocal.Save(file_handle))
| |
ReturnFalse;
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Restores the CNeuronGlobLocGraphAtt state from the file handle. |
| |
//+------------------------------------------------------------------+
| |
bool CNeuronGlobLocGraphAtt::Load(const int file_handle)
| |
{
| |
if(!CNeuronMHAttentionPooling::Load(file_handle))
| |
ReturnFalse;
| |
//---
| |
if(!LoadInsideLayer(file_handle, cGlobal.AsObject()))
| |
ReturnFalse;
| |
if(!LoadInsideLayer(file_handle, cLocal.AsObject()))
| |
ReturnFalse;
| |
//---
| |
if(!cConcat.Init(0, 2, OpenCL, 2 * iWindow * iUnits, optimization, iBatch))
| |
ReturnFalse;
| |
cConcat.SetActivationFunction(None);
| |
//---
| |
return(true);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Rebinds the CNeuronGlobLocGraphAtt GPU buffers to the given |
| |
//| OpenCL context. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGlobLocGraphAtt::SetOpenCL(COpenCLMy *obj)
| |
{
| |
CNeuronMHAttentionPooling::SetOpenCL(obj);
| |
//---
| |
cGlobal.SetOpenCL(OpenCL);
| |
cLocal.SetOpenCL(OpenCL);
| |
cConcat.SetOpenCL(OpenCL);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| Sets the training flag of the CNeuronGlobLocGraphAtt layer. |
| |
//+------------------------------------------------------------------+
| |
void CNeuronGlobLocGraphAtt::TrainMode(bool flag)
| |
{
| |
CNeuronMHAttentionPooling::TrainMode(flag);
| |
//---
| |
cGlobal.TrainMode(flag);
| |
cLocal.TrainMode(flag);
| |
cConcat.TrainMode(flag);
| |
}
| |
//+------------------------------------------------------------------+ |