5671 lines
213 KiB
MQL5
5671 lines
213 KiB
MQL5
#ifndef NEURONET_BUILDING_FACADE
|
|
#include "NeuroNet.mqh"
|
|
#endif // NEURONET_BUILDING_FACADE
|
|
//+------------------------------------------------------------------+
|
|
//| Logical method implementations generated from NeuroNet.mqh: GraphSparse
|
|
//+------------------------------------------------------------------+
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
if(!cInputF.Init(numNeurons, 0, OpenCL, window, optimization, batch))
|
|
ReturnFalse;
|
|
if(!cInputS.Init(numNeurons, 1, OpenCL, window, optimization, batch))
|
|
ReturnFalse;
|
|
cInputF.SetActivationFunction(None);
|
|
cInputS.SetActivationFunction(None);
|
|
//---
|
|
if(!cF.Init(0, 2, OpenCL, numNeurons, optimization, batch))
|
|
ReturnFalse;
|
|
cF.SetActivationFunction(SIGMOID);
|
|
if(!cS.Init(0, 3, OpenCL, numNeurons, optimization, batch))
|
|
ReturnFalse;
|
|
cS.SetActivationFunction(LReLU);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL)
|
|
{
|
|
if(!NeuronOCL || !NeuronOCL.getOutput() || NeuronOCL.getOutputIndex() < 0)
|
|
ReturnFalse;
|
|
//---
|
|
if(cInputF.getOutputIndex() != NeuronOCL.getOutputIndex())
|
|
{
|
|
if(!cInputF.getOutput().BufferSet(NeuronOCL.getOutputIndex()))
|
|
ReturnFalse;
|
|
cInputF.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation());
|
|
}
|
|
if(cInputS.getOutputIndex() != NeuronOCL.getOutputIndex())
|
|
{
|
|
if(!cInputS.getOutput().BufferSet(NeuronOCL.getOutputIndex()))
|
|
ReturnFalse;
|
|
cInputS.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation());
|
|
}
|
|
//---
|
|
if(!cF.FeedForward(GetPointer(cInputF)))
|
|
ReturnFalse;
|
|
if(!cS.FeedForward(GetPointer(cInputS)))
|
|
ReturnFalse;
|
|
//---
|
|
if(!ElementMult(cF.getOutput(), cS.getOutput(), Output))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::calcInputGradients(CNeuronBaseOCL* prevLayer)
|
|
{
|
|
if(!prevLayer || !prevLayer.getGradient() || prevLayer.getGradientIndex() < 0)
|
|
ReturnFalse;
|
|
//---
|
|
if(!ElementMultGrad(cF.getOutput(), cF.getGradient(), cS.getOutput(), cS.getGradient(), Gradient, cF.Activation(), cS.Activation()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cInputF.CalcHiddenGradients((CObject *)GetPointer(cF)))
|
|
ReturnFalse;
|
|
if(!cInputS.CalcHiddenGradients((CObject *)GetPointer(cS)))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cF.getOutput(), cS.getOutput(), prevLayer.getOutput(), 1, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
|
|
{
|
|
if(!cF.UpdateInputWeights(cInputF.AsObject()))
|
|
ReturnFalse;
|
|
if(!cS.UpdateInputWeights(cInputS.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cInputF.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cInputS.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cF.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cS.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileReadInteger(file_handle) != cInputF.Type() || !cInputF.Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileReadInteger(file_handle) != cInputS.Type() || !cInputS.Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileReadInteger(file_handle) != cF.Type() || !cF.Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileReadInteger(file_handle) != cS.Type() || !cS.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronCGConvOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
if(!cInputF.WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
if(!cInputS.WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
if(!cF.WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
if(!cS.WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronCGConvOCL::SetOpenCL(COpenCLMy * obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cInputF.SetOpenCL(OpenCL);
|
|
cInputS.SetOpenCL(OpenCL);
|
|
cF.SetOpenCL(OpenCL);
|
|
cS.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint dimension,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
|
|
units_count * dimension, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cEa.SetActivationFunction(None);
|
|
index++;
|
|
if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cWconcat_ex.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEn.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(GELU);
|
|
index++;
|
|
if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cAadapt.SetHeads(units_count);
|
|
index++;
|
|
if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cAadaptX.SetActivationFunction(None);
|
|
index++;
|
|
if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cApreX.SetActivationFunction(None);
|
|
index++;
|
|
if(!cWadapt.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cWadapt.SetActivationFunction(None);
|
|
index++;
|
|
if(!cWpre.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cWpre.SetActivationFunction(None);
|
|
SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
|
|
{
|
|
if(!SecondInput || SecondInput.Total() < cAadapt.Neurons())
|
|
ReturnFalse;
|
|
//---
|
|
if(bTrain)
|
|
{
|
|
if(!cEa.FeedForward())
|
|
ReturnFalse;
|
|
if(!cWe.FeedForward(cEa.AsObject()))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
if(!cWx.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
|
|
ReturnFalse;
|
|
if(!cEn.FeedForward(cWconcat_ex.AsObject()))
|
|
ReturnFalse;
|
|
if(!cEnT.FeedForward(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount()))
|
|
ReturnFalse;
|
|
if(cEnEnT.Activation() != None)
|
|
if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation()))
|
|
ReturnFalse;
|
|
if(!cAadapt.FeedForward(cEnEnT.AsObject()))
|
|
ReturnFalse;
|
|
if(!IdentSum(cAadapt.getOutput(), cAadapt.getOutput(), cAadapt.Heads()))
|
|
ReturnFalse;
|
|
if(!MatMul(SecondInput, NeuronOCL.getOutput(), cApreX.getOutput(),
|
|
cWpre.GetUnits(), cWpre.GetUnits(), cWpre.GetWindow()))
|
|
ReturnFalse;
|
|
if(!MatMul(cAadapt.getOutput(), NeuronOCL.getOutput(), cAadaptX.getOutput(),
|
|
cWadapt.GetUnits(), cWadapt.GetUnits(), cWadapt.GetWindow()))
|
|
ReturnFalse;
|
|
if(!cWpre.FeedForward(cApreX.AsObject()))
|
|
ReturnFalse;
|
|
if(!cWadapt.FeedForward(cAadaptX.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cWadapt.getOutput(), cWpre.getOutput(), Output, cWadapt.GetWindowOut(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
|
|
CBufferFloat *SecondInput,
|
|
CBufferFloat *SecondGradient,
|
|
ENUM_ACTIVATION SecondActivation = None)
|
|
{
|
|
if(!NeuronOCL || !SecondInput || !SecondGradient ||
|
|
SecondInput.Total() < cAadapt.Neurons() ||
|
|
SecondGradient.Total() < cAadapt.Neurons())
|
|
ReturnFalse;
|
|
//---
|
|
if(!DeActivation(cWadapt.getOutput(), cWadapt.getGradient(), Gradient, cWadapt.Activation()))
|
|
ReturnFalse;
|
|
if(!DeActivation(cWpre.getOutput(), cWpre.getGradient(), Gradient, cWpre.Activation()))
|
|
ReturnFalse;
|
|
if(!cApreX.CalcHiddenGradients(cWpre.AsObject()))
|
|
ReturnFalse;
|
|
if(!cAadaptX.CalcHiddenGradients(cWadapt.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(SecondInput, SecondGradient,
|
|
NeuronOCL.getOutput(), NeuronOCL.getGradient(),
|
|
cApreX.getGradient(), cWpre.GetUnits(),
|
|
cWpre.GetUnits(), cWpre.GetWindow()))
|
|
ReturnFalse;
|
|
if(SecondActivation != None)
|
|
if(!DeActivation(SecondInput, SecondGradient,
|
|
SecondGradient, SecondActivation))
|
|
ReturnFalse;
|
|
//---
|
|
if(!MatMulGrad(cAadapt.getOutput(), cAadapt.getGradient(),
|
|
NeuronOCL.getOutput(), PrevOutput,
|
|
cAadaptX.getGradient(), cWadapt.GetUnits(),
|
|
cWadapt.GetUnits(), cWadapt.GetWindow()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
|
|
PrevOutput, cWx.GetWindow(), false))
|
|
ReturnFalse;
|
|
if(NeuronOCL.Activation() != None)
|
|
if(!DeActivation(NeuronOCL.getOutput(), PrevOutput,
|
|
PrevOutput, NeuronOCL.Activation()))
|
|
ReturnFalse;
|
|
if(!cEnEnT.CalcHiddenGradients(cAadapt.AsObject()))
|
|
ReturnFalse;
|
|
if(cEnEnT.Activation() != None)
|
|
if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(),
|
|
cEnEnT.getGradient(), cEnEnT.Activation()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
|
|
cEnT.getOutput(), cEnT.getGradient(),
|
|
cEnEnT.getGradient(), cEnT.GetCount(),
|
|
cEnT.GetWindow(), cEnT.GetCount()))
|
|
ReturnFalse;
|
|
if(!cEn.CalcHiddenGradients(cEnT.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(),
|
|
cEn.getGradient(), cEnT.GetWindow(), false))
|
|
ReturnFalse;
|
|
if(cEn.Activation() != None)
|
|
if(!DeActivation(cEn.getOutput(), cEn.getGradient(),
|
|
cEn.getGradient(), cEn.Activation()))
|
|
ReturnFalse;
|
|
if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(),
|
|
cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
|
|
ReturnFalse;
|
|
if(cWx.Activation() != None)
|
|
if(!DeActivation(cWx.getOutput(), cWx.getGradient(),
|
|
cWx.getGradient(), cWx.Activation()))
|
|
ReturnFalse;
|
|
if(cWe.Activation() != None)
|
|
if(!DeActivation(cWe.getOutput(), cWe.getGradient(),
|
|
cWe.getGradient(), cWe.Activation()))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
|
|
NeuronOCL.getGradient(), cWx.GetWindow(), false))
|
|
ReturnFalse;
|
|
if(!cEa.CalcHiddenGradients(cWe.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)
|
|
{
|
|
if(!cEa.UpdateInputWeights())
|
|
ReturnFalse;
|
|
if(!cWe.UpdateInputWeights(cEa.AsObject()))
|
|
ReturnFalse;
|
|
if(!cWx.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject()))
|
|
ReturnFalse;
|
|
if(!cWpre.UpdateInputWeights(cApreX.AsObject()))
|
|
ReturnFalse;
|
|
if(!cWadapt.UpdateInputWeights(cAadaptX.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronAGCN* Source = source;
|
|
if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWpre.WeightsUpdate(Source.cWpre.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWadapt.WeightsUpdate(Source.cWadapt.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEa.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWe.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWx.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEn.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWpre.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWadapt.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAGCN::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEa.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWe.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWx.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWpre.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWadapt.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
uint units_count = GetUnits();
|
|
uint dimension = GetWindow();
|
|
//---
|
|
int index = 3;
|
|
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cWconcat_ex.SetActivationFunction(None);
|
|
index += 2;
|
|
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(GELU);
|
|
index++;
|
|
if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cAadapt.SetHeads(units_count);
|
|
index++;
|
|
if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cAadaptX.SetActivationFunction(None);
|
|
index++;
|
|
if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cApreX.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronAGCN::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cEa.SetOpenCL(OpenCL);
|
|
cWe.SetOpenCL(OpenCL);
|
|
cWx.SetOpenCL(OpenCL);
|
|
cEn.SetOpenCL(OpenCL);
|
|
cWpre.SetOpenCL(OpenCL);
|
|
cWadapt.SetOpenCL(OpenCL);
|
|
cWconcat_ex.SetOpenCL(OpenCL);
|
|
cEnT.SetOpenCL(OpenCL);
|
|
cEnEnT.SetOpenCL(OpenCL);
|
|
cAadapt.SetOpenCL(OpenCL);
|
|
cAadaptX.SetOpenCL(OpenCL);
|
|
cApreX.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronAGCN::TrainMode(bool flag)
|
|
{
|
|
CNeuronBaseOCL::TrainMode(flag);
|
|
cEa.TrainMode(bTrain);
|
|
cWe.TrainMode(bTrain);
|
|
cWx.TrainMode(bTrain);
|
|
cEn.TrainMode(bTrain);
|
|
cWpre.TrainMode(bTrain);
|
|
cWadapt.TrainMode(bTrain);
|
|
cWconcat_ex.TrainMode(bTrain);
|
|
cEnT.TrainMode(bTrain);
|
|
cEnEnT.TrainMode(bTrain);
|
|
cAadapt.TrainMode(bTrain);
|
|
cAadaptX.TrainMode(bTrain);
|
|
cApreX.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint dimension,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
|
|
units_count * dimension, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cX_IA.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cX_AGCN.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cForgetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cResetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cContext.Init(0, index, OpenCL, units_count * dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
if(!cContext.getPrevOutput().Fill(0))
|
|
ReturnFalse;
|
|
cContext.SetActivationFunction(None);
|
|
bTemp.BufferFree();
|
|
if(!bTemp.BufferInit(units_count * units_count, 0) ||
|
|
!bTemp.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
|
|
{
|
|
if(!cX_IA.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cX_AGCN.FeedForward(cX_IA.AsObject(), SecondInput))
|
|
ReturnFalse;
|
|
if(!cForgetGate.FeedForward(cX_IA.AsObject(), SecondInput))
|
|
ReturnFalse;
|
|
if(!cResetGate.FeedForward(cX_IA.AsObject(), SecondInput))
|
|
ReturnFalse;
|
|
if(!Activation(cForgetGate.getOutput(), cForgetGate.getOutput(), GELU))
|
|
ReturnFalse;
|
|
if(!Activation(cResetGate.getOutput(), cResetGate.getOutput(), GELU))
|
|
ReturnFalse;
|
|
//--- Context
|
|
if(!cContext.SwapOutputs())
|
|
ReturnFalse;
|
|
if(!GateElementMult(cContext.getPrevOutput(), cX_AGCN.getOutput(), cForgetGate.getOutput(), cContext.getOutput()))
|
|
ReturnFalse;
|
|
//--- Output
|
|
if(!Activation(cContext.getOutput(), cContext.getPrevOutput(), ELU))
|
|
ReturnFalse;
|
|
if(!GateElementMult(cContext.getPrevOutput(), cX_IA.getOutput(), cResetGate.getOutput(), Output))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
|
|
CBufferFloat *SecondInput,
|
|
CBufferFloat *SecondGradient,
|
|
ENUM_ACTIVATION SecondActivation = None)
|
|
{
|
|
if(!NeuronOCL || !SecondInput || !SecondGradient)
|
|
ReturnFalse;
|
|
//--- Output
|
|
if(!GateElementMultGrad(cContext.getPrevOutput(), cContext.getGradient(),
|
|
cX_IA.getOutput(), cX_IA.getPrevOutput(),
|
|
cResetGate.getOutput(), cResetGate.getGradient(),
|
|
Gradient, ELU, cX_IA.Activation(), GELU))
|
|
ReturnFalse;
|
|
//--- Context
|
|
if(!GateElementMultGrad(cContext.getOutput(), cContext.getPrevOutput(),
|
|
cX_AGCN.getOutput(), cX_AGCN.getGradient(),
|
|
cForgetGate.getOutput(), cForgetGate.getGradient(),
|
|
cContext.getGradient(), None,
|
|
cX_AGCN.Activation(), GELU))
|
|
ReturnFalse;
|
|
//--- Gradient to Interposition Attention
|
|
if(!cX_IA.CalcHiddenGradients(cX_AGCN.AsObject(), SecondInput, SecondGradient, SecondActivation) ||
|
|
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!cX_IA.CalcHiddenGradients(cForgetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) ||
|
|
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) ||
|
|
!SumAndNormalize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!cX_IA.CalcHiddenGradients(cResetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) ||
|
|
!SumAndNormalize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) ||
|
|
!SumAndNormalize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
if(!NeuronOCL.CalcHiddenGradients(cX_IA.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)
|
|
{
|
|
if(!cX_IA.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cX_AGCN.UpdateInputWeights(cX_IA.AsObject(), second))
|
|
ReturnFalse;
|
|
if(!cForgetGate.UpdateInputWeights(cX_IA.AsObject(), second))
|
|
ReturnFalse;
|
|
if(!cResetGate.UpdateInputWeights(cX_IA.AsObject(), second))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronGinARCell* Source = source;
|
|
if(!cX_IA.WeightsUpdate(Source.cX_IA.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cX_AGCN.WeightsUpdate(Source.cX_AGCN.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cForgetGate.WeightsUpdate(Source.cForgetGate.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cResetGate.WeightsUpdate(Source.cResetGate.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cX_IA.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cX_AGCN.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cForgetGate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cResetGate.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cX_IA.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cX_AGCN.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cForgetGate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cResetGate.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 4;
|
|
if(!cContext.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
if(!cContext.getPrevOutput().Fill(0))
|
|
ReturnFalse;
|
|
cContext.SetActivationFunction(None);
|
|
bTemp.BufferFree();
|
|
if(!bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0) ||
|
|
!bTemp.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGinARCell::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cX_IA.SetOpenCL(OpenCL);
|
|
cX_AGCN.SetOpenCL(OpenCL);
|
|
cForgetGate.SetOpenCL(OpenCL);
|
|
cResetGate.SetOpenCL(OpenCL);
|
|
cContext.SetOpenCL(OpenCL);
|
|
bTemp.BufferFree();
|
|
bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0);
|
|
bTemp.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGinARCell::TrainMode(bool flag)
|
|
{
|
|
CNeuronBaseOCL::TrainMode(flag);
|
|
cX_IA.TrainMode(bTrain);
|
|
cX_AGCN.TrainMode(bTrain);
|
|
cForgetGate.TrainMode(bTrain);
|
|
cResetGate.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinARCell::Clear(void)
|
|
{
|
|
if(!CNeuronBaseOCL::Clear())
|
|
ReturnFalse;
|
|
if(!cContext.getPrevOutput() ||
|
|
!cContext.getPrevOutput().Fill(0))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint dimension,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronSwiGLUOCL::Init(numOutputs, myIndex, open_cl, caCells.Size()*dimension,
|
|
caCells.Size()*dimension, dimension, units_count, 1,
|
|
optimization_type, batch))
|
|
ReturnFalse;
|
|
int index = 0;
|
|
if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cEa.SetActivationFunction(None);
|
|
index++;
|
|
if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cWconcat_ex.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEn.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(GELU);
|
|
index++;
|
|
if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cApre.SetHeads(units_count);
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
{
|
|
index++;
|
|
if(!caCells[i].Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
}
|
|
index++;
|
|
if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
//--- Calculate Apre
|
|
if(bTrain)
|
|
{
|
|
if(!cEa.FeedForward())
|
|
ReturnFalse;
|
|
if(!cWe.FeedForward(cEa.AsObject()))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
if(!cWx.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
|
|
ReturnFalse;
|
|
if(!cEn.FeedForward(cWconcat_ex.AsObject()))
|
|
ReturnFalse;
|
|
if(!cEnT.FeedForward(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount()))
|
|
ReturnFalse;
|
|
if(cEnEnT.Activation() != None)
|
|
if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation()))
|
|
ReturnFalse;
|
|
if(!cApre.FeedForward(cEnEnT.AsObject()))
|
|
ReturnFalse;
|
|
if(!IdentSum(cApre.getOutput(), cApre.getOutput(), cApre.Heads()))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
CNeuronBaseOCL *temp = NeuronOCL;
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
{
|
|
if(!caCells[i].FeedForward(temp, cApre.getOutput()))
|
|
ReturnFalse;
|
|
temp = caCells[i].AsObject();
|
|
}
|
|
//---
|
|
if(!Concat(caCells[0].getOutput(), caCells[1].getOutput(), caCells[2].getOutput(), caCells[3].getOutput(),
|
|
cConcat.getOutput(), GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetUnits()))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronSwiGLUOCL::feedForward(cConcat.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronSwiGLUOCL::calcInputGradients(cConcat.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(caCells[0].getPrevOutput(), caCells[1].getPrevOutput(),
|
|
caCells[2].getPrevOutput(), caCells[3].getGradient(),
|
|
cConcat.getGradient(), GetWindow() / 4,
|
|
GetWindow() / 4, GetWindow() / 4,
|
|
GetWindow() / 4, GetUnits()))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
cApre.getGradient().Fill(0);
|
|
for(uint i = caCells.Size() - 1; i > 0; i--)
|
|
if(!caCells[i - 1].CalcHiddenGradients(caCells[i].AsObject(), cApre.getOutput(),
|
|
cApre.getPrevOutput(),
|
|
(ENUM_ACTIVATION)cApre.Activation()) ||
|
|
!SumAndNormalize(caCells[i - 1].getGradient(), caCells[i - 1].getPrevOutput(),
|
|
caCells[i - 1].getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1) ||
|
|
!SumAndNormalize(cApre.getGradient(), cApre.getPrevOutput(),
|
|
cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(caCells[0].AsObject(), cApre.getOutput(),
|
|
cApre.getPrevOutput(), (ENUM_ACTIVATION)cApre.Activation()) ||
|
|
!SumAndNormalize(cApre.getGradient(), cApre.getPrevOutput(),
|
|
cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEnEnT.CalcHiddenGradients(cApre.AsObject()))
|
|
ReturnFalse;
|
|
if(cEnEnT.Activation() != None)
|
|
if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(),
|
|
cEnEnT.getGradient(), cEnEnT.Activation()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
|
|
cEnT.getOutput(), cEnT.getGradient(),
|
|
cEnEnT.getGradient(), cEnT.GetCount(),
|
|
cEnT.GetWindow(), cEnT.GetCount()))
|
|
ReturnFalse;
|
|
if(!cEn.CalcHiddenGradients(cEnT.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(),
|
|
cEn.getGradient(), cEnT.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(cEn.Activation() != None)
|
|
if(!DeActivation(cEn.getOutput(), cEn.getGradient(),
|
|
cEn.getGradient(), cEn.Activation()))
|
|
ReturnFalse;
|
|
if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(),
|
|
cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits()))
|
|
ReturnFalse;
|
|
if(cWx.Activation() != None)
|
|
if(!DeActivation(cWx.getOutput(), cWx.getGradient(),
|
|
cWx.getGradient(), cWx.Activation()))
|
|
ReturnFalse;
|
|
if(cWe.Activation() != None)
|
|
if(!DeActivation(cWe.getOutput(), cWe.getGradient(),
|
|
cWe.getGradient(), cWe.Activation()))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = NeuronOCL.getGradient();
|
|
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), temp,
|
|
NeuronOCL.getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
if(!cEa.CalcHiddenGradients(cWe.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
//--- Apre
|
|
if(!cEa.UpdateInputWeights())
|
|
ReturnFalse;
|
|
if(!cWe.UpdateInputWeights(cEa.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cWx.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject()))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
CNeuronBaseOCL *temp = NeuronOCL;
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
{
|
|
if(!caCells[i].UpdateInputWeights(temp, cApre.getOutput()))
|
|
ReturnFalse;
|
|
temp = caCells[i].AsObject();
|
|
}
|
|
//---
|
|
return CNeuronSwiGLUOCL::updateInputWeights(cConcat.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronSwiGLUOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronGinAR* Source = source;
|
|
//--- Apre
|
|
if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
if(!caCells[i].WeightsUpdate(Source.caCells[i].AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronSwiGLUOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//--- Apre
|
|
if(!cEa.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWe.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cWx.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEn.Save(file_handle))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
if(!caCells[i].Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronSwiGLUOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//--- Apre
|
|
if(!LoadInsideLayer(file_handle, cEa.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWe.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cWx.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
|
|
ReturnFalse;
|
|
//--- GimAR Cells
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
if(!LoadInsideLayer(file_handle, caCells[i].AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
uint units_count = GetUnits();
|
|
uint dimension = cWx.GetWindow();
|
|
uint index = 3;
|
|
if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cWconcat_ex.SetActivationFunction(None);
|
|
index += 2;
|
|
if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(GELU);
|
|
index++;
|
|
if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
cApre.SetHeads(units_count);
|
|
index += caCells.Size();
|
|
if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGinAR::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronSwiGLUOCL::SetOpenCL(obj);
|
|
//--- Apre
|
|
cEa.SetOpenCL(OpenCL);
|
|
cWe.SetOpenCL(OpenCL);
|
|
cWx.SetOpenCL(OpenCL);
|
|
cEn.SetOpenCL(OpenCL);
|
|
//--- GimAR Cells
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
caCells[i].SetOpenCL(OpenCL);
|
|
//---
|
|
cWconcat_ex.SetOpenCL(OpenCL);
|
|
cEnT.SetOpenCL(OpenCL);
|
|
cEnEnT.SetOpenCL(OpenCL);
|
|
cApre.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGinAR::TrainMode(bool flag)
|
|
{
|
|
CNeuronSwiGLUOCL::TrainMode(flag);
|
|
//--- Apre
|
|
cEa.TrainMode(bTrain);
|
|
cWe.TrainMode(bTrain);
|
|
cWx.TrainMode(bTrain);
|
|
cEn.TrainMode(bTrain);
|
|
//--- GimAR Cells
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
caCells[i].TrainMode(bTrain);
|
|
//---
|
|
cWconcat_ex.TrainMode(bTrain);
|
|
cEnT.TrainMode(bTrain);
|
|
cEnEnT.TrainMode(bTrain);
|
|
cApre.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGinAR::Clear(void)
|
|
{
|
|
if(!CNeuronSwiGLUOCL::Clear())
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caCells.Size(); i++)
|
|
if(!caCells[i].Clear())
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint variables,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * variables, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
iVariables = variables;
|
|
iCount = units_count;
|
|
//---
|
|
uint dimension = (iVariables + 1) / 2;
|
|
uint index = 0;
|
|
if(!cEn.Init(0, index, OpenCL, iVariables * dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEn.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnT.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(None);
|
|
index++;
|
|
if(!cAttan.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cAttan.SetHeads(iVariables);
|
|
//---
|
|
index++;
|
|
if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cMeanSTDevs.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::AdaptSpatialNorm(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!OpenCL || !NeuronOCL)
|
|
ReturnFalse;
|
|
uint global_work_offset[3] = { 0 };
|
|
uint global_work_size[] = { iCount, MathMin(iVariables, uint(OpenCL.GetMaxLocalSize(1))), iVariables};
|
|
uint local_work_size[] = { 1, global_work_size[1], 1};
|
|
//---
|
|
uint kernel = def_k_AdaptSpatialNorm;
|
|
setBuffer(kernel, def_k_asn_inputs, NeuronOCL.getOutputIndex())
|
|
setBuffer(kernel, def_k_asn_mean_stdevs, cMeanSTDevs.getOutputIndex())
|
|
setBuffer(kernel, def_k_asn_attention, cAttan.getOutputIndex())
|
|
setBuffer(kernel, def_k_asn_outputs, getOutputIndex())
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!Output.BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::AdaptSpatialNormGrad(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!OpenCL || !NeuronOCL)
|
|
ReturnFalse;
|
|
uint global_work_offset[3] = { 0 };
|
|
uint global_work_size[] = { MathMax(iVariables, iCount), MathMin(MathMax(iVariables, iCount), uint(OpenCL.GetMaxLocalSize(1))), iVariables};
|
|
uint local_work_size[] = { 1, global_work_size[1], 1};
|
|
//---
|
|
uint kernel = def_k_AdaptSpatialNormGrad;
|
|
setBuffer(kernel, def_k_asng_inputs, NeuronOCL.getOutputIndex())
|
|
setBuffer(kernel, def_k_asng_inputs_gr, NeuronOCL.getGradientIndex())
|
|
setBuffer(kernel, def_k_asng_mean_stdevs, cMeanSTDevs.getOutputIndex())
|
|
setBuffer(kernel, def_k_asng_mean_stdevs_gr, cMeanSTDevs.getGradientIndex())
|
|
setBuffer(kernel, def_k_asng_attention, cAttan.getOutputIndex())
|
|
setBuffer(kernel, def_k_asng_attention_gr, cAttan.getGradientIndex())
|
|
setBuffer(kernel, def_k_asng_outputs_gr, getGradientIndex())
|
|
setArgument(kernel, def_k_asng_total_inputs, iCount)
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!NeuronOCL.getGradient().BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(bTrain)
|
|
{
|
|
if(!cEn.FeedForward())
|
|
ReturnFalse;
|
|
if(!cEnT.FeedForward(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(),
|
|
iVariables, cEnT.GetWindow(), iVariables, 1, false))
|
|
ReturnFalse;
|
|
if(!cAttan.FeedForward(cEnEnT.AsObject()))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return AdaptSpatialNorm(NeuronOCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!AdaptSpatialNormGrad(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cEnEnT.CalcHiddenGradients(cAttan.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(),
|
|
cEnT.getOutput(), cEnT.getGradient(),
|
|
cEnEnT.getGradient(), iVariables,
|
|
cEnT.GetWindow(), iVariables, 1, false))
|
|
ReturnFalse;
|
|
if(!cEn.CalcHiddenGradients(cEnT.AsObject()) ||
|
|
!SumAndNormalize(cEn.getGradient(), cEn.getPrevOutput(), cEn.getGradient(),
|
|
cEnT.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(cEn.Activation() != None)
|
|
if(!DeActivation(cEn.getOutput(), cEn.getGradient(), cEn.getGradient(), cEn.Activation()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
return cEn.UpdateInputWeights();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEn.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cAttan.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cAttan.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
iVariables = cAttan.Heads();
|
|
iCount = Neurons() / iVariables;
|
|
uint dimension = cEn.Neurons() / iVariables;
|
|
//---
|
|
uint index = 1;
|
|
if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnT.SetActivationFunction(None);
|
|
index++;
|
|
if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEnEnT.SetActivationFunction(None);
|
|
index += 2;
|
|
if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cMeanSTDevs.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronAdaptSpatialNorm::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cEn.SetOpenCL(OpenCL);
|
|
cEnT.SetOpenCL(OpenCL);
|
|
cEnEnT.SetOpenCL(OpenCL);
|
|
cAttan.SetOpenCL(OpenCL);
|
|
cMeanSTDevs.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronAdaptSpatialNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronAdaptSpatialNorm* Source = source;
|
|
if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEn.FeedForward())
|
|
ReturnFalse;
|
|
if(!cEnT.FeedForward(cEn.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(),
|
|
iVariables, cEnT.GetWindow(), iVariables, 1, false))
|
|
ReturnFalse;
|
|
if(!cAttan.FeedForward(cEnEnT.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint variables, uint forecast,
|
|
uint season_period, uint short_period,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast, variables, optimization_type, batch))
|
|
ReturnFalse;
|
|
SetActivationFunction(None);
|
|
//---
|
|
uint index = 0;
|
|
if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cLongNorm.SetActivationFunction(None);
|
|
index++;
|
|
if(!cSeasonTransp.Init(0, index, OpenCL, variables, units_count / season_period, season_period, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSeasonTransp.SetActivationFunction(None);
|
|
index++;
|
|
if(!cSeasonNorm.Init(0, index, OpenCL, cSeasonTransp.GetCount(), season_period, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSeasonNorm.SetActivationFunction(None);
|
|
index++;
|
|
if(!cUnSeasonTransp.Init(0, index, OpenCL, variables, season_period, cSeasonTransp.GetCount(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSeasonNorm.SetActivationFunction(None);
|
|
index++;
|
|
if(!cAdaptSpatNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
cAdaptSpatNorm.SetActivationFunction(None);
|
|
index++;
|
|
uint concatSize = units_count * variables; //inputs
|
|
concatSize += cLongNorm.Neurons() + cLongNorm.GetMeanSTDevs().Neurons(); // long term
|
|
concatSize += cSeasonNorm.Neurons() + cSeasonNorm.GetMeanSTDevs().Neurons(); // seasons
|
|
concatSize += cShortNorm.Neurons() + cShortNorm.GetMeanSTDevs().Neurons(); // short term
|
|
concatSize += cAdaptSpatNorm.Neurons() + cAdaptSpatNorm.GetMeanSTDevs().Neurons(); // spatial
|
|
if(!cConcatenated.Init(0, index, OpenCL, concatSize, optimization, iBatch))
|
|
ReturnFalse;
|
|
cConcatenated.SetActivationFunction(None);
|
|
index++;
|
|
if(!cProjection.Init(0, index, OpenCL, concatSize / variables, concatSize / variables, units_count + forecast, 1, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cTranspose.Init(0, index, OpenCL, variables, units_count + forecast, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!caFusion[0].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch))
|
|
ReturnFalse;
|
|
caFusion[0].SetActivationFunction(TANH);
|
|
index++;
|
|
if(!caFusion[1].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch))
|
|
ReturnFalse;
|
|
caFusion[1].SetActivationFunction(SIGMOID);
|
|
index++;
|
|
if(!cFusionOut.Init(0, index, OpenCL, caFusion[0].Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cLongNorm.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cSeasonTransp.FeedForward(cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cAdaptSpatNorm.FeedForward(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
uint windows[3] = {NeuronOCL.Neurons() / iWindow,
|
|
cLongNorm.GetPeriod()*cLongNorm.GetUnits(),
|
|
2 * cLongNorm.GetUnits()
|
|
};
|
|
if(!Concat(NeuronOCL.getOutput(), cLongNorm.getOutput(), cLongNorm.GetMeanSTDevs().getOutput(),
|
|
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[0] = windows[0] + windows[1] + windows[2];
|
|
windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits();
|
|
windows[2] = 2 * cSeasonNorm.GetUnits();
|
|
if(!cConcatenated.SwapOutputs() ||
|
|
!Concat(cConcatenated.getPrevOutput(), cSeasonNorm.getOutput(), cSeasonNorm.GetMeanSTDevs().getOutput(),
|
|
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[0] = windows[0] + windows[1] + windows[2];
|
|
windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits();
|
|
windows[2] = 2 * cShortNorm.GetUnits();
|
|
if(!cConcatenated.SwapOutputs() ||
|
|
!Concat(cConcatenated.getPrevOutput(), cShortNorm.getOutput(), cShortNorm.GetMeanSTDevs().getOutput(),
|
|
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[0] = windows[0] + windows[1] + windows[2];
|
|
windows[1] = cAdaptSpatNorm.GetUnits();
|
|
windows[2] = 2 * cAdaptSpatNorm.GetUnits();
|
|
if(!cConcatenated.SwapOutputs() ||
|
|
!Concat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getOutput(), cAdaptSpatNorm.GetMeanSTDevs().getOutput(),
|
|
cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
if(!cProjection.FeedForward(cConcatenated.AsObject()))
|
|
ReturnFalse;
|
|
if(!cTranspose.FeedForward(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
if(!caFusion[i].FeedForward(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
if(!ElementMult(caFusion[0].getOutput(), caFusion[1].getOutput(), cFusionOut.getOutput()))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronTransposeOCL::feedForward(cFusionOut.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronTransposeOCL::calcInputGradients(cFusionOut.AsObject()))
|
|
ReturnFalse;
|
|
if(!ElementMultGrad(caFusion[0].getOutput(), caFusion[0].getGradient(),
|
|
caFusion[1].getOutput(), caFusion[1].getGradient(),
|
|
cFusionOut.getGradient(), caFusion[0].Activation(), caFusion[1].Activation()))
|
|
ReturnFalse;
|
|
if(!cTranspose.CalcHiddenGradients(caFusion[0].AsObject()))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = cTranspose.getGradient();
|
|
if(!cTranspose.SetGradient(cTranspose.getPrevOutput(), false) ||
|
|
!cTranspose.CalcHiddenGradients(caFusion[1].AsObject()) ||
|
|
!SumAndNormalize(temp, cTranspose.getGradient(), temp, cTranspose.GetCount(), false, 0, 0, 0, 1) ||
|
|
!cTranspose.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
if(!cProjection.CalcHiddenGradients(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
if(!cConcatenated.CalcHiddenGradients(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
uint windows[3] = {0};
|
|
windows[1] = cAdaptSpatNorm.GetUnits();
|
|
windows[2] = 2 * cAdaptSpatNorm.GetUnits();
|
|
windows[0] = cConcatenated.Neurons() / iWindow - windows[1] - windows[2];
|
|
if(!DeConcat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getGradient(),
|
|
cAdaptSpatNorm.GetMeanSTDevs().getGradient(),
|
|
cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits();
|
|
windows[2] = 2 * cShortNorm.GetUnits();
|
|
windows[0] = windows[0] - windows[1] - windows[2];
|
|
if(!DeConcat(cConcatenated.getGradient(), cShortNorm.getPrevOutput(),
|
|
cShortNorm.GetMeanSTDevs().getGradient(),
|
|
cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits();
|
|
windows[2] = 2 * cSeasonNorm.GetUnits();
|
|
windows[0] = windows[0] - windows[1] - windows[2];
|
|
if(!DeConcat(cConcatenated.getPrevOutput(), cSeasonNorm.getPrevOutput(),
|
|
cSeasonNorm.GetMeanSTDevs().getGradient(),
|
|
cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
windows[1] = cLongNorm.GetPeriod() * cLongNorm.GetUnits();
|
|
windows[2] = 2 * cLongNorm.GetUnits();
|
|
windows[0] = windows[0] - windows[1] - windows[2];
|
|
if(!DeConcat(NeuronOCL.getPrevOutput(), cLongNorm.getPrevOutput(),
|
|
cLongNorm.GetMeanSTDevs().getGradient(),
|
|
cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cShortNorm.CalcHiddenGradients(cAdaptSpatNorm.AsObject()) ||
|
|
!SumAndNormalize(cShortNorm.getGradient(), cShortNorm.getPrevOutput(),
|
|
cShortNorm.getGradient(), cShortNorm.GetPeriod(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject()) ||
|
|
!SumAndNormalize(cSeasonNorm.getGradient(), cSeasonNorm.getPrevOutput(),
|
|
cSeasonNorm.getGradient(), cSeasonNorm.GetPeriod(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject()) ||
|
|
!SumAndNormalize(cLongNorm.getGradient(), cLongNorm.getPrevOutput(),
|
|
cLongNorm.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject()) ||
|
|
!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(),
|
|
NeuronOCL.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cAdaptSpatNorm.UpdateInputWeights(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cProjection.UpdateInputWeights(cConcatenated.AsObject()))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
if(!caFusion[i].UpdateInputWeights(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cLongNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonTransp.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cUnSeasonTransp.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cShortNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cAdaptSpatNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cConcatenated.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cProjection.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cTranspose.Save(file_handle))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
if(!caFusion[i].Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cFusionOut.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cAdaptSpatNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cConcatenated.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
if(!LoadInsideLayer(file_handle, caFusion[i].AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cFusionOut.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSCNNEncoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronTransposeOCL::SetOpenCL(obj);
|
|
//---
|
|
cLongNorm.SetOpenCL(OpenCL);
|
|
cSeasonTransp.SetOpenCL(OpenCL);
|
|
cSeasonNorm.SetOpenCL(OpenCL);
|
|
cUnSeasonTransp.SetOpenCL(OpenCL);
|
|
cShortNorm.SetOpenCL(OpenCL);
|
|
cAdaptSpatNorm.SetOpenCL(OpenCL);
|
|
cConcatenated.SetOpenCL(OpenCL);
|
|
cProjection.SetOpenCL(OpenCL);
|
|
cTranspose.SetOpenCL(OpenCL);
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
caFusion[i].SetOpenCL(OpenCL);
|
|
cFusionOut.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSCNNEncoder::TrainMode(bool flag)
|
|
{
|
|
CNeuronTransposeOCL::TrainMode(flag);
|
|
//---
|
|
cLongNorm.TrainMode(bTrain);
|
|
cSeasonTransp.TrainMode(bTrain);
|
|
cSeasonNorm.TrainMode(bTrain);
|
|
cUnSeasonTransp.TrainMode(bTrain);
|
|
cShortNorm.TrainMode(bTrain);
|
|
cAdaptSpatNorm.TrainMode(bTrain);
|
|
cConcatenated.TrainMode(bTrain);
|
|
cProjection.TrainMode(bTrain);
|
|
cTranspose.TrainMode(bTrain);
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
caFusion[i].TrainMode(bTrain);
|
|
cFusionOut.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronSCNNEncoder* Source = source;
|
|
if(!cAdaptSpatNorm.WeightsUpdate(Source.cAdaptSpatNorm.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caFusion.Size(); i++)
|
|
if(!caFusion[i].WeightsUpdate(Source.caFusion[i].AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint variables, uint forecast,
|
|
uint season_period, uint short_period, uint layers,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables, optimization_type, batch))
|
|
ReturnFalse;
|
|
SetActivationFunction(None);
|
|
//---
|
|
cLayers.Clear();
|
|
cLayers.SetOpenCL(OpenCL);
|
|
CNeuronSCNNEncoder* encoder = NULL;
|
|
CNeuronBaseOCL* residual = NULL;
|
|
for(uint l = 0; l < layers; l++)
|
|
{
|
|
encoder = new CNeuronSCNNEncoder();
|
|
if(!encoder)
|
|
ReturnFalse;
|
|
if(!encoder.Init(0, l, OpenCL, units_count, variables, forecast, season_period, short_period, optimization, iBatch) ||
|
|
!cLayers.Add(encoder))
|
|
ReturnFalse;
|
|
encoder.SetActivationFunction(None);
|
|
if((l + 1) == layers)
|
|
break;
|
|
//---
|
|
residual = new CNeuronBaseOCL();
|
|
if(!residual)
|
|
ReturnFalse;
|
|
if(!residual.Init(0, l, OpenCL, units_count * variables, optimization, iBatch) ||
|
|
!cLayers.Add(residual))
|
|
ReturnFalse;
|
|
residual.SetActivationFunction(None);
|
|
}
|
|
if(!SetGradient(encoder.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
if(!Output.Fill(0))
|
|
ReturnFalse;
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronSCNNEncoder* current = NULL;
|
|
CNeuronBaseOCL* residual = NULL;
|
|
CNeuronBaseOCL* temp = NULL;
|
|
int layers = cLayers.Total();
|
|
//---
|
|
for(int l = 0; l < layers; l += 2)
|
|
{
|
|
current = cLayers[l];
|
|
if(!current ||
|
|
!current.FeedForward(inputs) ||
|
|
!SumAndNormalize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if((l + 1) == layers)
|
|
break;
|
|
uint variables = current.GetWindow();
|
|
uint dimension = inputs.Neurons() / variables;
|
|
uint forecast = current.GetCount() - dimension;
|
|
residual = cLayers[l + 1];
|
|
if(!residual)
|
|
ReturnFalse;
|
|
if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(),
|
|
dimension, forecast, variables) ||
|
|
!SumAndNormalize(residual.getOutput(), inputs.getOutput(), residual.getOutput(),
|
|
dimension, true, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
inputs = residual;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
if(!PrevOutput.Fill(0))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* inputs = NULL;
|
|
CNeuronSCNNEncoder* current = cLayers[-1];
|
|
CNeuronBaseOCL* residual = NULL;
|
|
int layers = cLayers.Total() - 2;
|
|
//---
|
|
for(int l = layers; l >= 0; l--)
|
|
switch(cLayers[l].Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
inputs = cLayers[l];
|
|
if(!inputs ||
|
|
!inputs.CalcHiddenGradients(current))
|
|
ReturnFalse;
|
|
if(!!residual)
|
|
if(!SumAndNormalize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
residual = inputs;
|
|
break;
|
|
case defNeuronSCNNEncoder:
|
|
current = cLayers[l];
|
|
if(!residual)
|
|
break;
|
|
inputs = cLayers[l + 2];
|
|
if(!inputs)
|
|
ReturnFalse;
|
|
if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(),
|
|
residual.Neurons() / current.GetWindow(),
|
|
current.GetCount() - residual.Neurons() / current.GetWindow(),
|
|
current.GetWindow()) ||
|
|
!SumAndNormalize(current.getGradient(), inputs.getGradient(), current.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
break;
|
|
default:
|
|
ReturnFalse;
|
|
break;
|
|
}
|
|
//---
|
|
if(!NeuronOCL.CalcHiddenGradients(current))
|
|
ReturnFalse;
|
|
if(!!residual)
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronBaseOCL* current = NULL;
|
|
//---
|
|
for(int l = 0; l < cLayers.Total(); l++)
|
|
{
|
|
current = cLayers[l];
|
|
if(!current)
|
|
ReturnFalse;
|
|
if(current.Type() == defNeuronSCNNEncoder ||
|
|
current.Type() == defNeuronSSCNNEncoder)
|
|
if(!current.UpdateInputWeights(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronSCNN* Source = source;
|
|
if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cLayers.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSCNN::Load(const int file_handle)
|
|
{
|
|
cLayers.Clear();
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
cLayers.SetOpenCL(OpenCL);
|
|
if(!cLayers.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* last = cLayers[-1];
|
|
if(!SetGradient(last.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSCNN::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cLayers.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSCNN::TrainMode(bool flag)
|
|
{
|
|
CNeuronBaseOCL::TrainMode(flag);
|
|
cLayers.TrainMode(flag);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint window, uint window_out,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window_out, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
if(!cProjection.Init(0, 0, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
if(!cConvolution.Init(0, 1, OpenCL, window_out, window_out, window_out, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cConvolution.SetActivationFunction(None);
|
|
if(!cResidual.Init(0, 2, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cResidual.SetActivationFunction(None);
|
|
//---
|
|
if(!SetGradient(cConvolution.getGradient(), true))
|
|
ReturnFalse;
|
|
if(!cResidual.SetGradient(getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cProjection.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cConvolution.FeedForward(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!cResidual.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cConvolution.getOutput(), cResidual.getOutput(),
|
|
Output, GetWindowOut(), true, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!cProjection.CalcHiddenGradients(cConvolution.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = NeuronOCL.getGradient();
|
|
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) ||
|
|
!NeuronOCL.CalcHiddenGradients(cResidual.AsObject()) ||
|
|
!SumAndNormalize(temp, NeuronOCL.getGradient(), temp,
|
|
GetWindowIn(), false, 0, 0, 0, 1) ||
|
|
!NeuronOCL.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cProjection.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cConvolution.UpdateInputWeights(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!cResidual.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronPolynomialRegression* Source = source;
|
|
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cConvolution.WeightsUpdate(Source.cConvolution.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cProjection.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cConvolution.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cResidual.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPolynomialRegression::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cConvolution.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cResidual.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronPolynomialRegression::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
//---
|
|
cProjection.SetOpenCL(OpenCL);
|
|
cConvolution.SetOpenCL(OpenCL);
|
|
cResidual.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint variables, uint forecast,
|
|
uint season_period, uint short_period,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast,
|
|
variables, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
int index = 0;
|
|
if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, iWindow, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cLongExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cLongExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cLongMeanSTDevTransp.Init(0, index, OpenCL, iWindow, 2, optimization, iBatch))
|
|
ReturnFalse;
|
|
if(!cLongMeanSTDevTransp.getGradient().Fill(0))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cLongMeanExtrapolate.Init(0, index, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cLongExtrapolate.SetActivationFunction(None);
|
|
if(!cLongMeanExtrapolate.getPrevOutput().Fill(1))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSeasonTransp.Init(0, index, OpenCL, iWindow, units_count / season_period, season_period, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSeasonNorm.Init(0, index, OpenCL, season_period, cSeasonTransp.GetCount(), iWindow, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cUnSeasonTransp.Init(0, index, OpenCL, iWindow, season_period, cSeasonTransp.GetCount(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSeasonExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSeasonExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cSeasonMeanExtrapolate.Init(0, index, OpenCL, season_period, season_period, iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSeasonMeanExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, iWindow, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cShortExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cShortExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cShortMeanExtrapolate.Init(0, index, OpenCL, cShortNorm.GetUnits(), cShortNorm.GetUnits(), iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cShortMeanExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cSpatialNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSpatialExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSpatialExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cSpatialMeanExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, 1, iWindow, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSpatialMeanExtrapolate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cConcatenated.Init(0, index, OpenCL, 8 * Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cTranspose.Init(0, index, OpenCL, 8 * iWindow, iCount, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cFusion.Init(0, index, OpenCL, iCount, 8 * iWindow, iWindow, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
//--- Long
|
|
if(!cLongNorm.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cLongExtrapolate.FeedForward(cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cLongMeanSTDevTransp.FeedForward(cLongNorm.GetMeanSTDevs()))
|
|
ReturnFalse;
|
|
if(!MatMul(cLongMeanSTDevTransp.getOutput(), cLongMeanExtrapolate.getPrevOutput(),
|
|
cLongMeanExtrapolate.getOutput(), 1, 1, iCount, iWindow, true))
|
|
ReturnFalse;
|
|
//--- Season
|
|
if(!cSeasonTransp.FeedForward(cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonExtrapolate.FeedForward(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonMeanExtrapolate.FeedForward(cSeasonNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Short
|
|
if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cShortExtrapolate.FeedForward(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cShortMeanExtrapolate.FeedForward(cShortNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Spatial
|
|
if(!cSpatialNorm.FeedForward(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatialExtrapolate.FeedForward(cSpatialNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatialMeanExtrapolate.FeedForward(cSpatialNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Concat
|
|
if(!Concat(cLongExtrapolate.getOutput(), cLongMeanExtrapolate.getOutput(),
|
|
cSeasonExtrapolate.getOutput(), cSeasonMeanExtrapolate.getOutput(),
|
|
cConcatenated.getOutput(), iCount, iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
if(!Concat(cConcatenated.getOutput(), cShortExtrapolate.getOutput(),
|
|
cShortMeanExtrapolate.getOutput(), cConcatenated.getPrevOutput(),
|
|
4 * iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
if(!Concat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getOutput(),
|
|
cSpatialMeanExtrapolate.getOutput(), cConcatenated.getOutput(),
|
|
6 * iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
//--- Fusion
|
|
if(!cTranspose.FeedForward(cConcatenated.AsObject()))
|
|
ReturnFalse;
|
|
if(!cFusion.FeedForward(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronTransposeOCL::feedForward(cFusion.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//--- Fusion
|
|
if(!CNeuronTransposeOCL::calcInputGradients(cFusion.AsObject()))
|
|
ReturnFalse;
|
|
if(!cTranspose.CalcHiddenGradients(cFusion.AsObject()))
|
|
ReturnFalse;
|
|
if(!cConcatenated.CalcHiddenGradients(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//--- DeConcat
|
|
if(!DeConcat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getGradient(),
|
|
cSpatialMeanExtrapolate.getGradient(), cConcatenated.getGradient(),
|
|
6 * iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
if(!DeConcat(cConcatenated.getGradient(), cShortExtrapolate.getGradient(),
|
|
cShortMeanExtrapolate.getGradient(), cConcatenated.getPrevOutput(),
|
|
4 * iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
if(!DeConcat(cLongExtrapolate.getGradient(), cLongMeanExtrapolate.getGradient(),
|
|
cSeasonExtrapolate.getGradient(), cSeasonMeanExtrapolate.getGradient(),
|
|
cConcatenated.getGradient(), iCount, iCount, iCount, iCount, iWindow))
|
|
ReturnFalse;
|
|
//--- Spatial
|
|
if(!cSpatialNorm.GetMeans().CalcHiddenGradients(cSpatialMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatialNorm.CalcHiddenGradients(cSpatialExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
//--- Short
|
|
if(!cShortNorm.CalcHiddenGradients(cSpatialNorm.AsObject()))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = cShortNorm.getGradient();
|
|
if(!cShortNorm.SetGradient(cShortNorm.getPrevOutput(), false) ||
|
|
!cShortNorm.CalcHiddenGradients(cShortExtrapolate.AsObject()) ||
|
|
!SumAndNormalize(temp, cShortNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
|
|
!cShortNorm.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
if(!cShortNorm.GetMeans().CalcHiddenGradients(cShortMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
//--- Season
|
|
if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
temp = cUnSeasonTransp.getGradient();
|
|
if(!cUnSeasonTransp.SetGradient(cUnSeasonTransp.getPrevOutput(), false) ||
|
|
!cUnSeasonTransp.CalcHiddenGradients(cSeasonExtrapolate.AsObject()) ||
|
|
!SumAndNormalize(temp, cUnSeasonTransp.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
|
|
!cUnSeasonTransp.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.GetMeans().CalcHiddenGradients(cSeasonMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
//--- Long
|
|
if(!MatMulGrad(cLongMeanSTDevTransp.getOutput(), cLongMeanSTDevTransp.getGradient(),
|
|
cLongMeanExtrapolate.getPrevOutput(), cLongMeanExtrapolate.getGradient(),
|
|
cLongMeanExtrapolate.getGradient(), 1, 1, iCount, iWindow, true))
|
|
ReturnFalse;
|
|
if(!cLongNorm.GetMeanSTDevs().CalcHiddenGradients(cLongMeanSTDevTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
temp = cLongNorm.getGradient();
|
|
if(!cLongNorm.SetGradient(cLongNorm.getPrevOutput(), false) ||
|
|
!cLongNorm.CalcHiddenGradients(cLongExtrapolate.AsObject()) ||
|
|
!SumAndNormalize(temp, cLongNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) ||
|
|
!cLongNorm.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
//--- Long
|
|
if(!cLongNorm.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cLongExtrapolate.UpdateInputWeights(cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
//--- Season
|
|
if(!cSeasonNorm.UpdateInputWeights(cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonExtrapolate.UpdateInputWeights(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSeasonMeanExtrapolate.UpdateInputWeights(cSeasonNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Short
|
|
if(!cShortNorm.UpdateInputWeights(cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!cShortExtrapolate.UpdateInputWeights(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cShortMeanExtrapolate.UpdateInputWeights(cShortNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Spatial
|
|
if(!cSpatialNorm.UpdateInputWeights(cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatialExtrapolate.UpdateInputWeights(cSpatialNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatialMeanExtrapolate.UpdateInputWeights(cSpatialNorm.GetMeans()))
|
|
ReturnFalse;
|
|
//--- Fusion
|
|
if(!cFusion.UpdateInputWeights(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronSSCNNEncoder* Source = source;
|
|
//--- Long
|
|
if(!cLongNorm.WeightsUpdate(Source.cLongNorm.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cLongExtrapolate.WeightsUpdate(Source.cLongExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- Season
|
|
if(!cSeasonNorm.WeightsUpdate(Source.cSeasonNorm.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cSeasonExtrapolate.WeightsUpdate(Source.cSeasonExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cSeasonMeanExtrapolate.WeightsUpdate(Source.cSeasonMeanExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- Short
|
|
if(!cShortNorm.WeightsUpdate(Source.cShortNorm.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cShortExtrapolate.WeightsUpdate(Source.cShortExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cShortMeanExtrapolate.WeightsUpdate(Source.cShortMeanExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- Spatial
|
|
if(!cSpatialNorm.WeightsUpdate(Source.cSpatialNorm.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cSpatialExtrapolate.WeightsUpdate(Source.cSpatialExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cSpatialMeanExtrapolate.WeightsUpdate(Source.cSpatialMeanExtrapolate.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- Fusion
|
|
if(!cFusion.WeightsUpdate(Source.cFusion.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cLongNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cLongExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cLongMeanSTDevTransp.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cLongMeanExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonTransp.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cUnSeasonTransp.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSeasonMeanExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cShortNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cShortExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cShortMeanExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSpatialNorm.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSpatialExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSpatialMeanExtrapolate.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cConcatenated.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cTranspose.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cFusion.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNNEncoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cLongNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cLongExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cLongMeanSTDevTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cLongMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSeasonMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cShortNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cShortExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cShortMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSpatialNorm.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSpatialExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSpatialMeanExtrapolate.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cConcatenated.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cFusion.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSSCNNEncoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronTransposeOCL::SetOpenCL(obj);
|
|
//---
|
|
cLongNorm.SetOpenCL(OpenCL);
|
|
cLongExtrapolate.SetOpenCL(OpenCL);
|
|
cLongMeanSTDevTransp.SetOpenCL(OpenCL);
|
|
cLongMeanExtrapolate.SetOpenCL(OpenCL);
|
|
cSeasonTransp.SetOpenCL(OpenCL);
|
|
cSeasonNorm.SetOpenCL(OpenCL);
|
|
cUnSeasonTransp.SetOpenCL(OpenCL);
|
|
cSeasonExtrapolate.SetOpenCL(OpenCL);
|
|
cSeasonMeanExtrapolate.SetOpenCL(OpenCL);
|
|
cShortNorm.SetOpenCL(OpenCL);
|
|
cShortExtrapolate.SetOpenCL(OpenCL);
|
|
cShortMeanExtrapolate.SetOpenCL(OpenCL);
|
|
cSpatialNorm.SetOpenCL(OpenCL);
|
|
cSpatialExtrapolate.SetOpenCL(OpenCL);
|
|
cSpatialMeanExtrapolate.SetOpenCL(OpenCL);
|
|
cConcatenated.SetOpenCL(OpenCL);
|
|
cTranspose.SetOpenCL(OpenCL);
|
|
cFusion.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSSCNNEncoder::TrainMode(bool flag)
|
|
{
|
|
CNeuronTransposeOCL::TrainMode(flag);
|
|
//---
|
|
cLongNorm.TrainMode(bTrain);
|
|
cLongExtrapolate.TrainMode(bTrain);
|
|
cLongMeanSTDevTransp.TrainMode(bTrain);
|
|
cLongMeanExtrapolate.TrainMode(bTrain);
|
|
cSeasonTransp.TrainMode(bTrain);
|
|
cSeasonNorm.TrainMode(bTrain);
|
|
cUnSeasonTransp.TrainMode(bTrain);
|
|
cSeasonExtrapolate.TrainMode(bTrain);
|
|
cSeasonMeanExtrapolate.TrainMode(bTrain);
|
|
cShortNorm.TrainMode(bTrain);
|
|
cShortExtrapolate.TrainMode(bTrain);
|
|
cShortMeanExtrapolate.TrainMode(bTrain);
|
|
cSpatialNorm.TrainMode(bTrain);
|
|
cSpatialExtrapolate.TrainMode(bTrain);
|
|
cSpatialMeanExtrapolate.TrainMode(bTrain);
|
|
cConcatenated.TrainMode(bTrain);
|
|
cTranspose.TrainMode(bTrain);
|
|
cFusion.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units_count, uint variables, uint forecast,
|
|
uint season_period, uint short_period, uint layers,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables,
|
|
optimization_type, batch))
|
|
ReturnFalse;
|
|
SetActivationFunction(None);
|
|
//---
|
|
cLayers.Clear();
|
|
cLayers.SetOpenCL(OpenCL);
|
|
CNeuronSSCNNEncoder* encoder = NULL;
|
|
CNeuronBaseOCL* residual = NULL;
|
|
for(uint l = 0; l < layers; l++)
|
|
{
|
|
encoder = new CNeuronSSCNNEncoder();
|
|
if(!encoder)
|
|
ReturnFalse;
|
|
if(!encoder.Init(0, 2 * l, OpenCL, units_count, variables, forecast, season_period,
|
|
short_period, optimization, iBatch) ||
|
|
!cLayers.Add(encoder))
|
|
ReturnFalse;
|
|
encoder.SetActivationFunction(None);
|
|
if((l + 1) == layers)
|
|
break;
|
|
//---
|
|
residual = new CNeuronBaseOCL();
|
|
if(!residual)
|
|
ReturnFalse;
|
|
if(!residual.Init(0, 2 * l + 1, OpenCL, units_count * variables, optimization, iBatch) ||
|
|
!cLayers.Add(residual))
|
|
ReturnFalse;
|
|
residual.SetActivationFunction(None);
|
|
}
|
|
if(!SetGradient(encoder.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNN::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
if(!Output.Fill(0))
|
|
ReturnFalse;
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronSSCNNEncoder* current = NULL;
|
|
CNeuronBaseOCL* residual = NULL;
|
|
CNeuronBaseOCL* temp = NULL;
|
|
int layers = cLayers.Total();
|
|
//---
|
|
for(int l = 0; l < layers; l += 2)
|
|
{
|
|
current = cLayers[l];
|
|
if(!current ||
|
|
!current.FeedForward(inputs) ||
|
|
!SumAndNormalize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if((l + 1) == layers)
|
|
break;
|
|
uint variables = current.GetWindow();
|
|
uint dimension = inputs.Neurons() / variables;
|
|
uint forecast = current.GetCount() - dimension;
|
|
residual = cLayers[l + 1];
|
|
if(!residual)
|
|
ReturnFalse;
|
|
if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(),
|
|
dimension, forecast, variables) ||
|
|
!SumAndNormalize(residual.getOutput(), inputs.getOutput(), residual.getOutput(),
|
|
dimension, true, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
inputs = residual;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
if(!PrevOutput.Fill(0))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* inputs = NULL;
|
|
CNeuronSSCNNEncoder* current = cLayers[-1];
|
|
CNeuronBaseOCL* residual = NULL;
|
|
int layers = cLayers.Total() - 2;
|
|
//---
|
|
for(int l = layers; l >= 0; l--)
|
|
switch(cLayers[l].Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
inputs = cLayers[l];
|
|
if(!inputs ||
|
|
!inputs.CalcHiddenGradients(current))
|
|
ReturnFalse;
|
|
if(!!residual)
|
|
if(!SumAndNormalize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
residual = inputs;
|
|
break;
|
|
case defNeuronSSCNNEncoder:
|
|
current = cLayers[l];
|
|
if(!residual)
|
|
break;
|
|
inputs = cLayers[l + 2];
|
|
if(!inputs)
|
|
ReturnFalse;
|
|
if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(),
|
|
residual.Neurons() / current.GetWindow(),
|
|
current.GetCount() - residual.Neurons() / current.GetWindow(),
|
|
current.GetWindow()) ||
|
|
!SumAndNormalize(current.getGradient(), inputs.getGradient(), current.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
break;
|
|
default:
|
|
ReturnFalse;
|
|
break;
|
|
}
|
|
//---
|
|
if(!NeuronOCL.CalcHiddenGradients(current))
|
|
ReturnFalse;
|
|
if(!!residual)
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(),
|
|
current.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CChebPolinom::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension,
|
|
uint steps, ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * dimension * steps, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
activation = None;
|
|
iDimension = dimension;
|
|
iSteps = steps;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CChebPolinom::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension))
|
|
ReturnFalse;
|
|
//---
|
|
uint global_work_offset[3] = { 0 };
|
|
uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))),
|
|
iDimension, iDimension
|
|
};
|
|
uint local_work_size[3] = { global_work_size[0], 1, 1 };
|
|
//---
|
|
uint kernel = def_k_ChebStep;
|
|
setBuffer(kernel, def_k_cheb_support, NeuronOCL.getOutputIndex())
|
|
setBuffer(kernel, def_k_cheb_outputs, getOutputIndex())
|
|
for(int step = MathMin(2, MathMax(int(iSteps) - 1, 0)); step < int(iSteps); step++)
|
|
{
|
|
setArgument(kernel, def_k_cheb_step, step + 1)
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!Output.BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CChebPolinom::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension))
|
|
ReturnFalse;
|
|
//---
|
|
uint global_work_offset[3] = { 0 };
|
|
uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))),
|
|
iDimension, iDimension
|
|
};
|
|
uint local_work_size[3] = { global_work_size[0], 1, 1 };
|
|
//---
|
|
uint kernel = def_k_ChebStepGrad;
|
|
setBuffer(kernel, def_k_chebgr_support, NeuronOCL.getOutputIndex())
|
|
setBuffer(kernel, def_k_chebgr_support_g, NeuronOCL.getGradientIndex())
|
|
setBuffer(kernel, def_k_chebgr_outputs, getOutputIndex())
|
|
setBuffer(kernel, def_k_chebgr_outputs_g, getGradientIndex())
|
|
for(int step = (int(iSteps) - 1); step >= MathMax(MathMin(2, int(iSteps) - 1), 1); step--)
|
|
{
|
|
setArgument(kernel, def_k_chebgr_step, step)
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!NeuronOCL.getGradient().BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CChebPolinom::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE)
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, int(iSteps)) < INT_VALUE)
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CChebPolinom::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iDimension = (uint)FileReadInteger(file_handle);
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iSteps = (uint)FileReadInteger(file_handle);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGrapConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint count, uint window, uint cheb_k,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronTransposeRCDOCL::Init(numOutputs, myIndex, open_cl, cheb_k,
|
|
count, window, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cX_G.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGrapConv::FeedForward(CNeuronBaseOCL *NeuronOCL,
|
|
CChebPolinom *Support)
|
|
{
|
|
if(!NeuronOCL || !Support)
|
|
ReturnFalse;
|
|
if(Support.GetDimension() != iWindow ||
|
|
Support.GetSteps() != iCount)
|
|
ReturnFalse;
|
|
if(NeuronOCL.Neurons() != (Neurons() / iCount))
|
|
ReturnFalse;
|
|
//---
|
|
if(!MatMul(Support.getOutput(), NeuronOCL.getOutput(), cX_G.getOutput(),
|
|
iWindow, iWindow, GetDimension(), iCount, false))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronTransposeRCDOCL::feedForward(cX_G.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGrapConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
|
|
CChebPolinom *Support)
|
|
{
|
|
if(!NeuronOCL || !Support)
|
|
ReturnFalse;
|
|
if(Support.GetDimension() != iWindow ||
|
|
Support.GetSteps() != iCount)
|
|
ReturnFalse;
|
|
if(NeuronOCL.Neurons() != (Neurons() / iCount))
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronTransposeRCDOCL::calcInputGradients(cX_G.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(Support.getOutput(), Support.getGradient(),
|
|
NeuronOCL.getOutput(), NeuronOCL.getGradient(),
|
|
cX_G.getGradient(), iWindow, iWindow,
|
|
GetDimension(), iCount, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGrapConv::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeRCDOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetGrapConv::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronTransposeRCDOCL::SetOpenCL(obj);
|
|
cX_G.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint window_out,
|
|
uint cheb_k, uint embed_dim,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
int index = 0;
|
|
if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cInpAndHidden.SetActivationFunction(None);
|
|
index++;
|
|
if(!cZ_R.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZ_R.SetActivationFunction(None);
|
|
index++;
|
|
if(!cZ_R_emb.Init(0, index, OpenCL, embed_dim, embed_dim,
|
|
2 * cheb_k * (window + window_out) * window_out,
|
|
units, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZ_R_emb.SetActivationFunction(None);
|
|
index++;
|
|
if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZe_Re.SetActivationFunction(None);
|
|
index++;
|
|
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZ.SetActivationFunction(None);
|
|
index++;
|
|
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cR.SetActivationFunction(None);
|
|
index++;
|
|
if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cCandidate.SetActivationFunction(None);
|
|
index++;
|
|
if(!cHC.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch))
|
|
ReturnFalse;
|
|
cHC.SetActivationFunction(None);
|
|
index++;
|
|
if(!cHC_emb.Init(0, index, OpenCL, embed_dim, embed_dim,
|
|
(window + window_out) * window_out * cheb_k,
|
|
units, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cHC_emb.SetActivationFunction(None);
|
|
index++;
|
|
if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cHCe.SetActivationFunction(None);
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
if(!Output.Fill(0))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::feedForward(CNeuronBaseOCL *NeuronOCL,
|
|
CChebPolinom *Support,
|
|
CNeuronBaseOCL *Embedding)
|
|
{
|
|
if(!NeuronOCL || !Support || !Embedding)
|
|
ReturnFalse;
|
|
if(!SwapOutputs())
|
|
ReturnFalse;
|
|
//---
|
|
uint cheb_k = cZ_R.GetChebK();
|
|
uint units = cZ_R.GetCount();
|
|
uint window_out = Neurons() / units;
|
|
uint window = cZ_R.GetWindow() - window_out;
|
|
//---
|
|
if(!Concat(NeuronOCL.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window_out, units))
|
|
ReturnFalse;
|
|
if(!cZ_R.FeedForward(cInpAndHidden.AsObject(), Support))
|
|
ReturnFalse;
|
|
if(!cZ_R_emb.FeedForward(Embedding))
|
|
ReturnFalse;
|
|
if(!MatMul(cZ_R.getOutput(), cZ_R_emb.getOutput(), cZe_Re.getOutput(), 1, (window + window_out)*cheb_k,
|
|
2 * window_out, units, true))
|
|
ReturnFalse;
|
|
if(!Activation(cZe_Re.getOutput(), cZe_Re.getOutput(), SIGMOID))
|
|
ReturnFalse;
|
|
if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZe_Re.getOutput(), window_out, window_out, units))
|
|
ReturnFalse;
|
|
if(!ElementMult(cZ.getOutput(), PrevOutput, cZ.getPrevOutput()))
|
|
ReturnFalse;
|
|
if(!Concat(NeuronOCL.getOutput(), cZ.getPrevOutput(), cCandidate.getOutput(), window, window_out, units))
|
|
ReturnFalse;
|
|
if(!cHC.FeedForward(cCandidate.AsObject(), Support))
|
|
ReturnFalse;
|
|
if(!cHC_emb.FeedForward(Embedding))
|
|
ReturnFalse;
|
|
if(!MatMul(cHC.getOutput(), cHC_emb.getOutput(), cHCe.getOutput(), 1, (window + window_out)*cheb_k,
|
|
window_out, units, true))
|
|
ReturnFalse;
|
|
if(!Activation(cHCe.getOutput(), cHCe.getOutput(), TANH))
|
|
ReturnFalse;
|
|
if(!GateElementMult(PrevOutput, cHCe.getOutput(), cR.getOutput(), Output))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
|
|
CChebPolinom *Support,
|
|
CNeuronBaseOCL *Embedding)
|
|
{
|
|
if(!NeuronOCL || !Support || !Embedding)
|
|
ReturnFalse;
|
|
//---
|
|
uint cheb_k = cZ_R.GetChebK();
|
|
uint units = cZ_R.GetCount();
|
|
uint window_out = Neurons() / units;
|
|
uint window = cZ_R.GetWindow() - window_out;
|
|
//---
|
|
if(!GateElementMultGrad(PrevOutput, cR.getPrevOutput(),
|
|
cHCe.getOutput(), cHCe.getGradient(),
|
|
cR.getOutput(), cR.getGradient(),
|
|
Gradient, None, TANH, cR.Activation()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cHC.getOutput(), cHC.getGradient(),
|
|
cHC_emb.getOutput(), cHC_emb.getGradient(),
|
|
cHCe.getGradient(), 1, (window + window_out)*cheb_k,
|
|
window_out, units, true))
|
|
ReturnFalse;
|
|
if(!cHC.calcInputGradients(cCandidate.AsObject(), Support))
|
|
ReturnFalse;
|
|
if(!DeConcat(NeuronOCL.getGradient(), cZ.getPrevOutput(), cCandidate.getGradient(),
|
|
window, window_out, units))
|
|
ReturnFalse;
|
|
if(!ElementMultGrad(cZ.getOutput(), cZ.getGradient(),
|
|
PrevOutput, cCandidate.getPrevOutput(),
|
|
cZ.getPrevOutput(), None, None))
|
|
ReturnFalse;
|
|
if(!Concat(cZ.getGradient(), cR.getGradient(), cZe_Re.getGradient(),
|
|
window_out, window_out, units))
|
|
ReturnFalse;
|
|
if(!DeActivation(cZe_Re.getOutput(), cZe_Re.getGradient(), cZe_Re.getGradient(), SIGMOID))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cZ_R.getOutput(), cZ_R.getGradient(),
|
|
cZ_R_emb.getOutput(), cZ_R_emb.getGradient(),
|
|
cZe_Re.getGradient(), 1, (window + window_out)*cheb_k,
|
|
2 * window_out, units, true))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = Support.getGradient();
|
|
if(!Support.SetGradient(GetPointer(bSupportAccum), false) ||
|
|
!cZ_R.calcInputGradients(cInpAndHidden.AsObject(), Support) ||
|
|
!SumAndNormalize(temp, Support.getGradient(), temp, Support.GetDimension(), false, 0, 0, 0, 1) ||
|
|
!Support.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
if(!DeConcat(cInpAndHidden.getPrevOutput(), cCandidate.getPrevOutput(), cInpAndHidden.getGradient(), window, window_out, units))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(NeuronOCL.getGradient(), cInpAndHidden.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(NeuronOCL.Activation() != None)
|
|
if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!Embedding.CalcHiddenGradients(cHC_emb.AsObject()))
|
|
ReturnFalse;
|
|
temp = Embedding.getGradient();
|
|
if(!Embedding.SetGradient(Embedding.getPrevOutput(), false) ||
|
|
!Embedding.CalcHiddenGradients(cZ_R_emb.AsObject()) ||
|
|
!SumAndNormalize(temp, Embedding.getGradient(), temp, cZ_R_emb.GetWindow(), false, 0, 0, 0, 1) ||
|
|
!Embedding.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::updateInputWeights(CNeuronBaseOCL *NeuronOCL,
|
|
CChebPolinom *Support,
|
|
CNeuronBaseOCL *Embedding)
|
|
{
|
|
if(!Embedding)
|
|
ReturnFalse;
|
|
//---
|
|
if(!cZ_R_emb.UpdateInputWeights(Embedding))
|
|
ReturnFalse;
|
|
if(!cHC_emb.UpdateInputWeights(Embedding))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetGCRU* Source = source;
|
|
if(!cZ_R_emb.WeightsUpdate(Source.cZ_R_emb.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cHC_emb.WeightsUpdate(Source.cHC_emb.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cZ_R.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cZ_R_emb.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cHC.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cHC_emb.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetGCRU::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cZ_R.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cZ_R_emb.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cHC.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cHC_emb.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
uint cheb_k = cZ_R.GetChebK();
|
|
uint units = cZ_R.GetCount();
|
|
uint window_out = Neurons() / units;
|
|
uint window = cZ_R.GetWindow() - window_out;
|
|
//---
|
|
int index = 0;
|
|
if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cInpAndHidden.SetActivationFunction(None);
|
|
index += 3;
|
|
if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZ.SetActivationFunction(None);
|
|
index++;
|
|
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cR.SetActivationFunction(None);
|
|
index++;
|
|
if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cCandidate.SetActivationFunction(None);
|
|
index += 3;
|
|
if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cHCe.SetActivationFunction(None);
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
if(!Output.Fill(0))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetGCRU::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
//---
|
|
cZ_R.SetOpenCL(OpenCL);
|
|
cZ_R_emb.SetOpenCL(OpenCL);
|
|
cHC.SetOpenCL(OpenCL);
|
|
cHC_emb.SetOpenCL(OpenCL);
|
|
cInpAndHidden.SetOpenCL(OpenCL);
|
|
cZe_Re.SetOpenCL(OpenCL);
|
|
cZ.SetOpenCL(OpenCL);
|
|
cR.SetOpenCL(OpenCL);
|
|
cCandidate.SetOpenCL(OpenCL);
|
|
cHCe.SetOpenCL(OpenCL);
|
|
//---
|
|
bSupportAccum.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint window_out, uint cheb_k,
|
|
uint layers, uint embed_dim, uint period1,
|
|
uint timeframe1, uint period2, uint timeframe2,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(layers <= 0)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!caEmbeddings[0].Init(0, index, OpenCL, embed_dim, period1, optimization, iBatch))
|
|
ReturnFalse;
|
|
aTimeframes[0] = MathMax(1, timeframe1);
|
|
index++;
|
|
if(!caEmbeddings[1].Init(0, index, OpenCL, embed_dim, period2, optimization, iBatch))
|
|
ReturnFalse;
|
|
aTimeframes[1] = MathMax(1, timeframe2);
|
|
if(!cConcatEmbeddings.Init(numOutputs, myIndex, open_cl, 2 * embed_dim, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
cGRCUs.Clear();
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
index++;
|
|
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
ReturnFalse;
|
|
for(uint i = 1; i < layers; i++)
|
|
{
|
|
index++;
|
|
temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, int label)
|
|
{
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
{
|
|
int position = (label / int(aTimeframes[i])) % caEmbeddings[i].GetPeriod();
|
|
if(!caEmbeddings[i].SetPosition(position) ||
|
|
!caEmbeddings[i].FeedForward())
|
|
ReturnFalse;
|
|
}
|
|
if(!Concat(caEmbeddings[0].getOutput(), caEmbeddings[1].getOutput(),
|
|
cConcatEmbeddings.getOutput(), 1, 1, caEmbeddings[0].Neurons()))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL *inputs = NeuronOCL;
|
|
CNeuronHimNetGCRU *current = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current ||
|
|
!current.feedForward(inputs, Support, cConcatEmbeddings.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
|
|
{
|
|
if(!NeuronOCL || !Support)
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL);
|
|
CNeuronHimNetGCRU* current = cGRCUs[-1];
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(caEmbeddings[0].getGradient(), caEmbeddings[1].getGradient(),
|
|
cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons()))
|
|
ReturnFalse;
|
|
if(cGRCUs.Total() > 1)
|
|
{
|
|
CBufferFloat *temp = Support.getGradient();
|
|
if(!Support.SetGradient(GetPointer(bSupportAccum), false))
|
|
ReturnFalse;
|
|
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
|
|
{
|
|
current = cGRCUs[i];
|
|
inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL);
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(temp, Support.getGradient(), temp, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!DeConcat(caEmbeddings[0].getPrevOutput(), caEmbeddings[1].getPrevOutput(),
|
|
cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons()))
|
|
ReturnFalse;
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
if(!SumAndNormalize(caEmbeddings[i].getGradient(), caEmbeddings[i].getPrevOutput(),
|
|
caEmbeddings[i].getGradient(), 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
}
|
|
if(!Support.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
|
|
{
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
if(!caEmbeddings[i].UpdateInputWeights())
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronHimNetGCRU* current = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current.updateInputWeights(inputs, Support, cConcatEmbeddings.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetTempEncoder* Source = source;
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
{
|
|
if(!caEmbeddings[i].Save(file_handle))
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, int(aTimeframes[i])) < INT_VALUE)
|
|
ReturnFalse;
|
|
}
|
|
if(!cGRCUs.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
{
|
|
if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject()))
|
|
ReturnFalse;
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
aTimeframes[i] = (uint)FileReadInteger(file_handle);
|
|
}
|
|
if(!cGRCUs.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
CNeuronHimNetGCRU* temp = cGRCUs[-1];
|
|
uint units = temp.GetCount();
|
|
uint cheb_k = temp.GetChebK();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetTempEncoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
for(uint i = 0; i < caEmbeddings.Size(); i++)
|
|
caEmbeddings[i].SetOpenCL(OpenCL);
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::Clear(void)
|
|
{
|
|
CNeuronBaseOCL* temp = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
temp = cGRCUs[i];
|
|
if(!temp ||
|
|
!temp.Clear())
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetTempEncoder::GetCount(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetCount();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetTempEncoder::GetChebK(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetChebK();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetTempEncoder::GetWindowIn(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetWindowIn();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetTempEncoder::GetWindowOut(void) const
|
|
{
|
|
uint count = GetCount();
|
|
if(count <= 0)
|
|
return 0;
|
|
//---
|
|
return Neurons() / count;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint window_out, uint cheb_k,
|
|
uint layers, uint embed_dim,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(layers <= 0)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cEmbedding.Init(0, index, OpenCL, embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
cGRCUs.Clear();
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
index++;
|
|
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
{
|
|
DeleteObj(temp);
|
|
ReturnFalse;
|
|
}
|
|
for(uint i = 1; i < layers; i++)
|
|
{
|
|
index++;
|
|
temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
{
|
|
DeleteObj(temp);
|
|
ReturnFalse;
|
|
}
|
|
}
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.Clear();
|
|
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
|
|
!bEmbeddingAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
|
|
{
|
|
if(bTrain)
|
|
if(!cEmbedding.FeedForward())
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL *inputs = NeuronOCL;
|
|
CNeuronHimNetGCRU *current = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current ||
|
|
!current.feedForward(inputs, Support, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
|
|
{
|
|
if(!NeuronOCL || !Support)
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL);
|
|
CNeuronHimNetGCRU* current = cGRCUs[-1];
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, Support, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(cGRCUs.Total() > 1)
|
|
{
|
|
CBufferFloat *sup = Support.getGradient();
|
|
if(!Support.SetGradient(GetPointer(bSupportAccum), false))
|
|
ReturnFalse;
|
|
CBufferFloat *emb = cEmbedding.getGradient();
|
|
if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false))
|
|
ReturnFalse;
|
|
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
|
|
{
|
|
current = cGRCUs[i];
|
|
inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL);
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, Support, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(sup, Support.getGradient(), sup, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
}
|
|
if(!Support.SetGradient(sup, false))
|
|
ReturnFalse;
|
|
if(!cEmbedding.SetGradient(emb, false))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support)
|
|
{
|
|
if(!cEmbedding.UpdateInputWeights())
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronHimNetGCRU* current = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current.updateInputWeights(inputs, Support, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetSpatEncoder* Source = source;
|
|
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEmbedding.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cGRCUs.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!cGRCUs.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
CNeuronHimNetGCRU* temp = cGRCUs[-1];
|
|
uint units = temp.GetCount();
|
|
uint cheb_k = temp.GetChebK();
|
|
if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.Clear();
|
|
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
|
|
!bEmbeddingAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetSpatEncoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.BufferCreate(OpenCL);
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::Clear(void)
|
|
{
|
|
CNeuronBaseOCL* temp = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
temp = cGRCUs[i];
|
|
if(!temp ||
|
|
!temp.Clear())
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetSpatEncoder::GetCount(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetCount();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetSpatEncoder::GetChebK(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetChebK();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetSpatEncoder::GetWindowIn(void) const
|
|
{
|
|
if(cGRCUs.Total() <= 0)
|
|
return 0;
|
|
CNeuronHimNetGCRU* temp = cGRCUs[0];
|
|
if(!temp)
|
|
return 0;
|
|
//---
|
|
return temp.GetWindowIn();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
uint CNeuronHimNetSpatEncoder::GetWindowOut(void) const
|
|
{
|
|
uint count = GetCount();
|
|
if(count <= 0)
|
|
return 0;
|
|
//---
|
|
return Neurons() / count;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetSpatEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
|
|
{
|
|
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* temp = cGRCUs[-1];
|
|
if(!temp ||
|
|
!temp.SetGradient(Gradient, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetTempEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
|
|
{
|
|
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* temp = cGRCUs[-1];
|
|
if(!temp ||
|
|
!temp.SetGradient(Gradient, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint window_out, uint cheb_k,
|
|
uint layers, uint embed_dim, uint period1,
|
|
uint timeframe1, uint period2, uint timeframe2,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cEmbedding.Init(0, index, OpenCL, units * embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
SetActivationFunction(TANH);
|
|
index++;
|
|
if(!cEmbeddingT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSupport.SetActivationFunction(None);
|
|
index++;
|
|
if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cNormSupport.SetHeads(units);
|
|
cNormSupport.SetActivationFunction(None);
|
|
index++;
|
|
if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cTempEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, (embed_dim + 1) / 2, period1, timeframe1, period2, timeframe2, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cSpatEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
if(!SetGradient(cTempEncoder.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
|
|
{
|
|
if(!SecondInput)
|
|
ReturnFalse;
|
|
//---
|
|
if(bTrain)
|
|
{
|
|
if(!cEmbedding.FeedForward())
|
|
ReturnFalse;
|
|
if(!cEmbeddingT.FeedForward(cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cEmbedding.getOutput(), cEmbeddingT.getOutput(), cSupport.getOutput(),
|
|
cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false))
|
|
ReturnFalse;
|
|
if(!cNormSupport.FeedForward(cSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.FeedForward(cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
if(!cTempEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject(), int(SecondInput[0])))
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cTempEncoder.getOutput(), cSpatEncoder.getOutput(), Output,
|
|
cTempEncoder.GetWindowOut(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::calcInputGradients(CNeuronBaseOCL *prevLayer)
|
|
{
|
|
if(!cTempEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
CBufferFloat* temp = cPolinomSupport.getGradient();
|
|
CBufferFloat* prev = prevLayer.getGradient();
|
|
if(!cPolinomSupport.SetGradient(cPolinomSupport.getPrevOutput(), false) ||
|
|
!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
|
|
!cSpatEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject()) ||
|
|
!SumAndNormalize(temp, cPolinomSupport.getGradient(), temp,
|
|
cSpatEncoder.GetCount(), false, 0, 0, 0, 1) ||
|
|
!SumAndNormalize(prev, prevLayer.getGradient(), prev,
|
|
cSpatEncoder.GetCount(), false, 0, 0, 0, 1) ||
|
|
!cPolinomSupport.SetGradient(temp, false) ||
|
|
!prevLayer.SetGradient(prev, false))
|
|
ReturnFalse;
|
|
if(prevLayer.Activation() != None)
|
|
if(!DeActivation(prevLayer.getOutput(), prev, prev, prevLayer.Activation()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cEmbedding.getOutput(), cEmbedding.getPrevOutput(),
|
|
cEmbeddingT.getOutput(), cEmbeddingT.getGradient(),
|
|
cSupport.getGradient(),
|
|
cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false))
|
|
ReturnFalse;
|
|
if(!cEmbedding.CalcHiddenGradients(cEmbeddingT.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cEmbedding.getGradient(), cEmbedding.getPrevOutput(), cEmbedding.getGradient(),
|
|
cEmbeddingT.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(cEmbedding.Activation() != None)
|
|
if(!DeActivation(cEmbedding.getOutput(), cEmbedding.getGradient(),
|
|
cEmbedding.getGradient(), cEmbedding.Activation()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cEmbedding.UpdateInputWeights())
|
|
ReturnFalse;
|
|
if(!cTempEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetEncoder* Source = source;
|
|
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cTempEncoder.WeightsUpdate(Source.cTempEncoder.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.WeightsUpdate(Source.cSpatEncoder.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEmbedding.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEmbeddingT.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cNormSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cTempEncoder.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEmbeddingT.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cTempEncoder.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSpatEncoder.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!SetGradient(cTempEncoder.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetEncoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
//---
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
cEmbeddingT.SetOpenCL(OpenCL);
|
|
cSupport.SetOpenCL(OpenCL);
|
|
cNormSupport.SetOpenCL(OpenCL);
|
|
cPolinomSupport.SetOpenCL(OpenCL);
|
|
cTempEncoder.SetOpenCL(OpenCL);
|
|
cSpatEncoder.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetEncoder::TrainMode(bool flag)
|
|
{
|
|
CNeuronBaseOCL::TrainMode(flag);
|
|
//---
|
|
cEmbedding.TrainMode(bTrain);
|
|
cEmbeddingT.TrainMode(bTrain);
|
|
cSupport.TrainMode(bTrain);
|
|
cNormSupport.TrainMode(bTrain);
|
|
cPolinomSupport.TrainMode(bTrain);
|
|
cTempEncoder.TrainMode(bTrain);
|
|
cSpatEncoder.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::Clear(void)
|
|
{
|
|
if(!CNeuronBaseOCL::Clear())
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEmbedding.Clear())
|
|
ReturnFalse;
|
|
if(!cEmbeddingT.Clear())
|
|
ReturnFalse;
|
|
if(!cSupport.Clear())
|
|
ReturnFalse;
|
|
if(!cNormSupport.Clear())
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.Clear())
|
|
ReturnFalse;
|
|
if(!cTempEncoder.Clear())
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.Clear())
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true)
|
|
{
|
|
if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cTempEncoder.SetGradient(Gradient, delete_prev))
|
|
ReturnFalse;
|
|
if(!cSpatEncoder.SetGradient(Gradient, delete_prev))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint window_out,
|
|
uint cheb_k, uint layers, uint embed_dim,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cProjection.Init(0, index, OpenCL, window, window, embed_dim, units, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
SetActivationFunction(TANH);
|
|
index++;
|
|
if(!cProjectionT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cEmbedding.Init(0, index, OpenCL, units, units, 1, 1, embed_dim, optimization, iBatch))
|
|
ReturnFalse;
|
|
SetActivationFunction(SIGMOID);
|
|
index++;
|
|
if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cSupport.SetActivationFunction(None);
|
|
index++;
|
|
if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
cNormSupport.SetHeads(units);
|
|
cNormSupport.SetActivationFunction(None);
|
|
index++;
|
|
if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
cGRCUs.Clear();
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
index++;
|
|
CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
{
|
|
DeleteObj(temp);
|
|
ReturnFalse;
|
|
}
|
|
for(uint i = 1; i < layers; i++)
|
|
{
|
|
index++;
|
|
temp = new CNeuronHimNetGCRU();
|
|
if(!temp ||
|
|
!temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) ||
|
|
!cGRCUs.Add(temp))
|
|
{
|
|
DeleteObj(temp);
|
|
ReturnFalse;
|
|
}
|
|
}
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.Clear();
|
|
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
|
|
!bEmbeddingAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cProjection.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cProjectionT.FeedForward(cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMul(cProjection.getOutput(), cProjectionT.getOutput(), cSupport.getOutput(),
|
|
cProjectionT.GetCount(), cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false))
|
|
ReturnFalse;
|
|
if(!cNormSupport.FeedForward(cSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.FeedForward(cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEmbedding.FeedForward(cProjectionT.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetGCRU* current = NULL;
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current ||
|
|
!current.feedForward(NeuronOCL, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::calcInputGradients(CNeuronBaseOCL *prevLayer)
|
|
{
|
|
if(!prevLayer)
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : prevLayer);
|
|
CNeuronHimNetGCRU* current = cGRCUs[-1];
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(cGRCUs.Total() > 1)
|
|
{
|
|
CBufferFloat *sup = cPolinomSupport.getGradient();
|
|
if(!cPolinomSupport.SetGradient(GetPointer(bSupportAccum), false))
|
|
ReturnFalse;
|
|
CBufferFloat *emb = cEmbedding.getGradient();
|
|
if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false))
|
|
ReturnFalse;
|
|
for(int i = cGRCUs.Total() - 2; i >= 0; i--)
|
|
{
|
|
current = cGRCUs[i];
|
|
inputs = (i > 0 ? cGRCUs[i - 1] : prevLayer);
|
|
if(!current ||
|
|
!current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cPolinomSupport.getGradient(), sup, sup, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
}
|
|
if(!cPolinomSupport.SetGradient(sup, false))
|
|
ReturnFalse;
|
|
if(!cEmbedding.SetGradient(emb, false))
|
|
ReturnFalse;
|
|
}
|
|
if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cProjection.getOutput(), cProjection.getPrevOutput(),
|
|
cProjectionT.getOutput(), cProjectionT.getPrevOutput(),
|
|
cSupport.getGradient(), cProjectionT.GetCount(),
|
|
cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false))
|
|
ReturnFalse;
|
|
if(!cProjectionT.CalcHiddenGradients(cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(cProjectionT.Activation() != None)
|
|
if(!DeActivation(cProjectionT.getOutput(), cProjectionT.getPrevOutput(),
|
|
cProjectionT.getPrevOutput(), cProjectionT.Activation()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cProjectionT.getGradient(), cProjectionT.getPrevOutput(),
|
|
cProjectionT.getGradient(), cProjectionT.GetCount(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!cProjection.CalcHiddenGradients(cProjectionT.AsObject()))
|
|
ReturnFalse;
|
|
if(cProjection.Activation() != None)
|
|
if(!DeActivation(cProjection.getOutput(), cProjection.getPrevOutput(),
|
|
cProjection.getPrevOutput(), cProjection.Activation()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cProjection.getGradient(), cProjection.getPrevOutput(),
|
|
cProjection.getGradient(), cProjectionT.GetWindow(), false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
//---
|
|
CBufferFloat* temp = prevLayer.getGradient();
|
|
if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
|
|
!prevLayer.CalcHiddenGradients(cProjection.AsObject()) ||
|
|
!SumAndNormalize(temp, prevLayer.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
|
|
!prevLayer.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cProjection.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cEmbedding.UpdateInputWeights(cProjectionT.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* inputs = NeuronOCL;
|
|
CNeuronHimNetGCRU* current = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
current = cGRCUs[i];
|
|
if(!current.updateInputWeights(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronHimNetDecoder* Source = source;
|
|
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cProjection.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cProjectionT.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cEmbedding.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cNormSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cGRCUs.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cProjection.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cProjectionT.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cNormSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject()))
|
|
ReturnFalse;
|
|
if(!cGRCUs.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.Clear();
|
|
CNeuronHimNetGCRU* temp = cGRCUs[-1];
|
|
if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) ||
|
|
!bSupportAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.Clear();
|
|
if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) ||
|
|
!bEmbeddingAccum.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
SetActivationFunction((ENUM_ACTIVATION)temp.Activation());
|
|
if(!SetOutput(temp.getOutput(), true) ||
|
|
!SetGradient(temp.getGradient(), true))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronHimNetDecoder::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
//---
|
|
cProjection.SetOpenCL(OpenCL);
|
|
cProjectionT.SetOpenCL(OpenCL);
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
cSupport.SetOpenCL(OpenCL);
|
|
cNormSupport.SetOpenCL(OpenCL);
|
|
cPolinomSupport.SetOpenCL(OpenCL);
|
|
cGRCUs.SetOpenCL(OpenCL);
|
|
//---
|
|
bSupportAccum.BufferFree();
|
|
bSupportAccum.BufferCreate(OpenCL);
|
|
bEmbeddingAccum.BufferFree();
|
|
bEmbeddingAccum.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronHimNetDecoder::Clear(void)
|
|
{
|
|
if(!CNeuronBaseOCL::Clear())
|
|
ReturnFalse;
|
|
//---
|
|
if(!cProjection.Clear())
|
|
ReturnFalse;
|
|
if(!cProjectionT.Clear())
|
|
ReturnFalse;
|
|
if(!cEmbedding.Clear())
|
|
ReturnFalse;
|
|
if(!cSupport.Clear())
|
|
ReturnFalse;
|
|
if(!cNormSupport.Clear())
|
|
ReturnFalse;
|
|
if(!cPolinomSupport.Clear())
|
|
ReturnFalse;
|
|
CNeuronBaseOCL* temp = NULL;
|
|
for(int i = 0; i < cGRCUs.Total(); i++)
|
|
{
|
|
temp = cGRCUs[i];
|
|
if(!temp || !temp.Clear())
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint sparse_dimension,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
int index = 0;
|
|
if(!cInpAndHidden.Init(0, index, OpenCL, 2 * units * window, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cZ_R.Init(0, index, OpenCL, 2 * window, 2 * window, 2 * window, units, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cZ_R.SetActivationFunction(SIGMOID);
|
|
index++;
|
|
if(!cZ.Init(0, index, OpenCL, window * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cR.Init(0, index, OpenCL, window * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cCandidate.Init(0, index, OpenCL, 2 * window * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cHC.Init(0, index, OpenCL, 2 * window, 2 * window, window, units, 1, optimization, iBatch))
|
|
ReturnFalse;
|
|
cHC.SetActivationFunction(TANH);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::RandomWalk(CBufferFloat *data,
|
|
CBufferFloat *normal,
|
|
CBufferFloat *inv_diag,
|
|
const int rows,
|
|
const int cols)
|
|
{
|
|
if(!OpenCL)
|
|
ReturnFalse;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2] = {(uint)rows, MathMin((uint)cols, (uint)OpenCL.GetMaxLocalSize(1))};
|
|
uint local_work_size[2] = {1, global_work_size[1]};
|
|
uint kernel = def_k_RandomWalk;
|
|
setBuffer(kernel, def_k_rw_data, data.GetIndex())
|
|
setBuffer(kernel, def_k_rw_norm, normal.GetIndex())
|
|
setBuffer(kernel, def_k_rw_inv_diag, inv_diag.GetIndex())
|
|
setArgument(kernel, def_k_rw_total_cols, cols)
|
|
//---
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!inv_diag.BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::FeedForward(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent)
|
|
{
|
|
if(!SourceData || !SparseAttent)
|
|
ReturnFalse;
|
|
//---
|
|
const uint units = GetCount();
|
|
const uint window = GetWindow();
|
|
const uint sparse = GetSparseDimension();
|
|
//---
|
|
if(!RandomWalk(SparseAttent.getOutput(), cNormAttention.getOutput(), cInvDiag.getOutput(), window, sparse))
|
|
ReturnFalse;
|
|
if(!SwapOutputs())
|
|
ReturnFalse;
|
|
if(!Concat(SourceData.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window, units))
|
|
ReturnFalse;
|
|
//---
|
|
if(!SparseMatMul(SparseAttent.GetIndexes(), cNormAttention.getOutput(), cInpAndHidden.getOutput(),
|
|
cAX.getOutput(), units, sparse, units, 2 * window))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cAX.getOutput(), cInpAndHidden.getOutput(), cAXplusX.getOutput(), 2 * window, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!DiagMatMul(cInvDiag.getOutput(), cAXplusX.getOutput(), cNormAXplusX.getOutput(), units, 2 * window, 1, None))
|
|
ReturnFalse;
|
|
if(!cZ_R.FeedForward(cNormAXplusX.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZ_R.getOutput(), window, window, units))
|
|
ReturnFalse;
|
|
if(!ElementMult(cR.getOutput(), PrevOutput, cR.getPrevOutput()))
|
|
ReturnFalse;
|
|
if(!Concat(SourceData.getOutput(), cR.getPrevOutput(), cCandidate.getOutput(), window, window, units))
|
|
ReturnFalse;
|
|
if(!cHC.FeedForward(cCandidate.AsObject()))
|
|
ReturnFalse;
|
|
if(!GateElementMult(PrevOutput, cHC.getOutput(), cZ.getOutput(), Output))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::CalcInputGradients(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent)
|
|
{
|
|
if(!SourceData || !SparseAttent)
|
|
ReturnFalse;
|
|
//---
|
|
const uint units = GetCount();
|
|
const uint window = GetWindow();
|
|
const uint sparse = GetSparseDimension();
|
|
//---
|
|
if(!GateElementMultGrad(PrevOutput, cInpAndHidden.getGradient(),
|
|
cHC.getOutput(), cHC.getGradient(),
|
|
cZ.getOutput(), cZ.getGradient(),
|
|
Gradient, None, cHC.Activation(), cZ_R.Activation()))
|
|
ReturnFalse;
|
|
if(!cCandidate.CalcHiddenGradients(cHC.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(SourceData.getGradient(), cR.getPrevOutput(),
|
|
cCandidate.getGradient(), window, window, units))
|
|
ReturnFalse;
|
|
if(!ElementMultGrad(cR.getOutput(), cR.getGradient(),
|
|
PrevOutput, cInpAndHidden.getGradient(),
|
|
cR.getPrevOutput(), cZ_R.Activation(), None))
|
|
ReturnFalse;
|
|
if(!Concat(cZ.getGradient(), cR.getGradient(),
|
|
cZ_R.getGradient(), window, window, units))
|
|
ReturnFalse;
|
|
if(!cNormAXplusX.CalcHiddenGradients(cZ_R.AsObject()))
|
|
ReturnFalse;
|
|
if(!DiagMatMulGrad(cInvDiag.getOutput(), cInvDiag.getGradient(),
|
|
cAXplusX.getOutput(), cAX.getGradient(),
|
|
cNormAXplusX.getGradient(), units, 2 * window, 1))
|
|
ReturnFalse;
|
|
if(!SparseMatMulGrad(SparseAttent.GetIndexes(), cNormAttention.getOutput(),
|
|
cNormAttention.getGradient(), cInpAndHidden.getOutput(),
|
|
cInpAndHidden.getGradient(), cAX.getGradient(),
|
|
units, sparse, units, 2 * window))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(cAX.getGradient(), cInpAndHidden.getGradient(),
|
|
cInpAndHidden.getGradient(), 2 * window, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(!DeConcat(cAXplusX.getGradient(), cAXplusX.getPrevOutput(),
|
|
cInpAndHidden.getGradient(), window, window, units))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(SourceData.getGradient(), cAXplusX.getGradient(),
|
|
SourceData.getGradient(), window, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
if(SourceData.Activation() != None)
|
|
if(!DeActivation(SourceData.getOutput(), SourceData.getGradient(),
|
|
SourceData.getGradient(), SourceData.Activation()))
|
|
ReturnFalse;
|
|
if(!DiagMatMul(cInvDiag.getOutput(), cNormAttention.getGradient(),
|
|
SparseAttent.getGradient(), units, sparse, 1, None))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
//---
|
|
if(!cZ_R.UpdateInputWeights(cNormAXplusX.AsObject()))
|
|
ReturnFalse;
|
|
if(!cHC.UpdateInputWeights(cCandidate.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronFastGConv *Source = source;
|
|
if(!cZ_R.WeightsUpdate(Source.cZ_R.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cHC.WeightsUpdate(Source.cHC.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cZ_R.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cHC.Save(file_handle))
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, (int)GetSparseDimension()) < INT_VALUE)
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronFastGConv::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cZ_R.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cHC.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
uint sparse_dimension = FileReadInteger(file_handle);
|
|
uint units = cZ_R.GetUnits();
|
|
uint window_out = Neurons() / units;
|
|
uint window = cZ_R.GetWindow() - window_out;
|
|
//---
|
|
int index = 0;
|
|
if(!cInpAndHidden.Init(0, index, OpenCL, units * (window + window_out), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch))
|
|
ReturnFalse;
|
|
index += 2;
|
|
if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cCandidate.Init(0, index, OpenCL, (window + window_out) * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronFastGConv::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
//---
|
|
cZ_R.SetOpenCL(OpenCL);
|
|
cHC.SetOpenCL(OpenCL);
|
|
cInpAndHidden.SetOpenCL(OpenCL);
|
|
cNormAttention.SetOpenCL(OpenCL);
|
|
cInvDiag.SetOpenCL(OpenCL);
|
|
cAX.SetOpenCL(OpenCL);
|
|
cAXplusX.SetOpenCL(OpenCL);
|
|
cNormAXplusX.SetOpenCL(OpenCL);
|
|
cZ.SetOpenCL(OpenCL);
|
|
cR.SetOpenCL(OpenCL);
|
|
cCandidate.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint time_steps, uint variables, uint embedding_dim,
|
|
uint emb_layers, uint sparse_dimension, uint heads,
|
|
float sparse, uint gcru_layers,
|
|
uint forecast, uint forec_layers,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(emb_layers <= 0 || gcru_layers <= 0 || forec_layers <= 0)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables, forecast, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
uint index = 0;
|
|
if(!cTranspose.Init(0, index, OpenCL, time_steps, variables, optimization, iBatch))
|
|
ReturnFalse;
|
|
//--- Embedding
|
|
cEmbedding.Clear();
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
index++;
|
|
CNeuronConvOCL *conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, time_steps, time_steps, embedding_dim, variables, 1, optimization, iBatch) ||
|
|
!cEmbedding.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
for(uint i = 1; i < emb_layers; i++)
|
|
{
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, embedding_dim, embedding_dim,
|
|
embedding_dim, variables, 1, optimization, iBatch) ||
|
|
!cEmbedding.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
}
|
|
CNeuronBatchNormOCL *norm = new CNeuronBatchNormOCL();
|
|
index++;
|
|
if(!norm ||
|
|
!norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) ||
|
|
!cEmbedding.Add(norm))
|
|
{
|
|
DeleteObj(norm);
|
|
ReturnFalse;
|
|
}
|
|
norm.SetActivationFunction(None);
|
|
//--- GCRUs
|
|
cGCRU.Clear();
|
|
cGCRU.SetOpenCL(OpenCL);
|
|
index++;
|
|
if(!cAttention.Init(0, index, OpenCL, variables, embedding_dim, heads,
|
|
sparse_dimension, sparse, optimization, iBatch))
|
|
ReturnFalse;
|
|
CNeuronFastGConv *gcru = NULL;
|
|
for(uint i = 0; i < gcru_layers; i++)
|
|
{
|
|
index++;
|
|
gcru = new CNeuronFastGConv();
|
|
if(!gcru ||
|
|
!gcru.Init(0, index, OpenCL, variables, embedding_dim, sparse_dimension, optimization, iBatch) ||
|
|
!cGCRU.Add(gcru))
|
|
{
|
|
DeleteObj(gcru);
|
|
ReturnFalse;
|
|
}
|
|
}
|
|
//--- Forecast
|
|
cProjection.Clear();
|
|
cProjection.SetOpenCL(OpenCL);
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, embedding_dim, embedding_dim,
|
|
forecast, variables, 1, optimization, iBatch) ||
|
|
!cProjection.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
for(uint i = 1; i < forec_layers; i++)
|
|
{
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, forecast, forecast,
|
|
forecast, variables, 1, optimization, iBatch) ||
|
|
!cProjection.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
}
|
|
norm = new CNeuronBatchNormOCL();
|
|
index++;
|
|
if(!norm ||
|
|
!norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) ||
|
|
!cProjection.Add(norm))
|
|
{
|
|
DeleteObj(norm);
|
|
ReturnFalse;
|
|
}
|
|
norm.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cTranspose.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
CNeuronBaseOCL *inputs = cTranspose.AsObject();
|
|
CNeuronBaseOCL *current = NULL;
|
|
//--- Embedding
|
|
for(int i = 0; i < cEmbedding.Total(); i++)
|
|
{
|
|
current = cEmbedding[i];
|
|
if(!current ||
|
|
!current.FeedForward(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//--- GCRUs
|
|
if(!cAttention.FeedForward(inputs))
|
|
ReturnFalse;
|
|
CNeuronFastGConv *gcru = NULL;
|
|
for(int i = 0; i < cGCRU.Total(); i++)
|
|
{
|
|
gcru = cGCRU[i];
|
|
if(!gcru ||
|
|
!gcru.FeedForward(inputs, cAttention.AsObject()))
|
|
ReturnFalse;
|
|
inputs = gcru;
|
|
}
|
|
//--- Forecast
|
|
for(int i = 0; i < cProjection.Total(); i++)
|
|
{
|
|
current = cProjection[i];
|
|
if(!current ||
|
|
!current.FeedForward(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//--- result
|
|
return CNeuronTransposeOCL::feedForward(inputs);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronTransposeOCL::calcInputGradients(cProjection[-1]))
|
|
ReturnFalse;
|
|
//--- Forecast
|
|
CNeuronBaseOCL *inputs = NULL;
|
|
for(int i = cProjection.Total() - 1; i >= 0; i--)
|
|
{
|
|
inputs = (i > 0 ? cProjection[i - 1] : cGCRU[-1]);
|
|
if(!inputs ||
|
|
!inputs.CalcHiddenGradients(cProjection[i]))
|
|
ReturnFalse;
|
|
}
|
|
//--- GCRU
|
|
CNeuronFastGConv *gcru = cGCRU[-1];
|
|
int layers = cGCRU.Total();
|
|
inputs = (layers > 1 ? cGCRU[-2] : cEmbedding[-1]);
|
|
if(!gcru ||
|
|
!gcru.CalcInputGradients(inputs, cAttention.AsObject()))
|
|
ReturnFalse;
|
|
if(layers > 1)
|
|
{
|
|
CBufferFloat *temp = cAttention.getGradient();
|
|
if(!cAttention.SetGradient(cAttention.getPrevOutput(), false))
|
|
ReturnFalse;
|
|
for(int i = layers - 2; i >= 0; i--)
|
|
{
|
|
inputs = (i > 0 ? cGCRU[i - 1] : cEmbedding[-1]);
|
|
gcru = cGCRU[i];
|
|
if(!gcru ||
|
|
!gcru.CalcInputGradients(inputs, cAttention.AsObject()) ||
|
|
!SumAndNormalize(temp, cAttention.getGradient(), temp, 1, false, 0, 0, 0, 1))
|
|
ReturnFalse;
|
|
}
|
|
if(!cAttention.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
}
|
|
CBufferFloat *temp = inputs.getGradient();
|
|
if(!inputs.SetGradient(inputs.getPrevOutput(), false) ||
|
|
!inputs.CalcHiddenGradients(cAttention.AsObject()) ||
|
|
!SumAndNormalize(temp, inputs.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
|
|
!inputs.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//--- Embedding
|
|
for(int i = cEmbedding.Total() - 2; i >= 0; i--)
|
|
{
|
|
inputs = cEmbedding[i];
|
|
if(!inputs ||
|
|
!inputs.CalcHiddenGradients(cEmbedding[i + 1]))
|
|
ReturnFalse;
|
|
}
|
|
//---
|
|
if(!cTranspose.CalcHiddenGradients(inputs))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
CNeuronBaseOCL *inputs = cTranspose.AsObject();
|
|
CNeuronBaseOCL *current = NULL;
|
|
//--- Embedding
|
|
for(int i = 0; i < cEmbedding.Total(); i++)
|
|
{
|
|
current = cEmbedding[i];
|
|
if(!current ||
|
|
!current.UpdateInputWeights(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//--- GCRUs
|
|
if(!cAttention.UpdateInputWeights(inputs))
|
|
ReturnFalse;
|
|
for(int i = 0; i < cGCRU.Total(); i++)
|
|
{
|
|
current = cGCRU[i];
|
|
if(!current ||
|
|
!current.UpdateInputWeights(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//--- Forecast
|
|
for(int i = 0; i < cProjection.Total(); i++)
|
|
{
|
|
current = cProjection[i];
|
|
if(!current ||
|
|
!current.UpdateInputWeights(inputs))
|
|
ReturnFalse;
|
|
inputs = current;
|
|
}
|
|
//--- result
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronTransposeOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
CNeuronSAGDFN *Source = source;
|
|
//--- Embedding
|
|
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- GCRUs
|
|
if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cGCRU.WeightsUpdate(Source.cGCRU.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- Forecast
|
|
if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau))
|
|
ReturnFalse;
|
|
//--- result
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cTranspose.Save(file_handle))
|
|
ReturnFalse;
|
|
//--- Embedding
|
|
if(!cEmbedding.Save(file_handle))
|
|
ReturnFalse;
|
|
//--- GCRUs
|
|
if(!cGCRU.Save(file_handle) ||
|
|
!cAttention.Save(file_handle))
|
|
ReturnFalse;
|
|
//--- Forecast
|
|
if(!cProjection.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSAGDFN::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronTransposeOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cTranspose.AsObject()))
|
|
ReturnFalse;
|
|
//--- Embedding
|
|
if(!cEmbedding.Load(file_handle))
|
|
ReturnFalse;
|
|
//--- GCRUs
|
|
if(!cGCRU.Load(file_handle) ||
|
|
!LoadInsideLayer(file_handle, cAttention.AsObject()))
|
|
ReturnFalse;
|
|
//--- Forecast
|
|
if(!cProjection.Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSAGDFN::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronTransposeOCL::SetOpenCL(obj);
|
|
//---
|
|
cTranspose.SetOpenCL(OpenCL);
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
cGCRU.SetOpenCL(OpenCL);
|
|
cAttention.SetOpenCL(OpenCL);
|
|
cProjection.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint embed_dim,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + embed_dim)*units, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
if(!cEmbedding.Init(0, 0, OpenCL, embed_dim * units, optimization, iBatch))
|
|
ReturnFalse;
|
|
cEmbedding.SetActivationFunction(TANH);
|
|
//---
|
|
iUnits = units;
|
|
iWindow = window;
|
|
iEmbeddingDim = embed_dim;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(bTrain)
|
|
if(!cEmbedding.FeedForward())
|
|
ReturnFalse;
|
|
//---
|
|
if(!Concat(NeuronOCL.getOutput(), cEmbedding.getOutput(), Output, iWindow, iEmbeddingDim, iUnits))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
if(!DeConcat(NeuronOCL.getGradient(), cEmbedding.getGradient(), Gradient, iWindow, iEmbeddingDim, iUnits))
|
|
ReturnFalse;
|
|
Deactivation(NeuronOCL)
|
|
Deactivation(cEmbedding)
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cEmbedding.UpdateInputWeights())
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronSpatialEmbedding *Source = source;
|
|
if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cEmbedding.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE)
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE)
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, (int)iEmbeddingDim) < INT_VALUE)
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSpatialEmbedding::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cEmbedding.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iUnits = (uint)FileReadInteger(file_handle);
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iWindow = (uint)FileReadInteger(file_handle);
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iEmbeddingDim = (uint)FileReadInteger(file_handle);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSpatialEmbedding::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
cEmbedding.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint emb_dimension,
|
|
uint experts, float dropout,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * units, optimization_type, batch))
|
|
ReturnFalse;
|
|
activation = None;
|
|
//---
|
|
int index = 0;
|
|
if(!cGraphs.Init(0, index, OpenCL, Neurons()*experts, optimization, iBatch))
|
|
ReturnFalse;
|
|
cGraphs.SetActivationFunction(SIGMOID);
|
|
//---
|
|
CNeuronBaseOCL *neuron = NULL;
|
|
CNeuronConvOCL *conv = NULL;
|
|
CNeuronTransposeOCL *transp = NULL;
|
|
CNeuronDropoutOCL *dout = NULL;
|
|
CNeuronSoftMaxOCL *softmax = NULL;
|
|
//--- Probability
|
|
acProbability.Clear();
|
|
acProbability.SetOpenCL(OpenCL);
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(experts, index, OpenCL, window, window, experts, units, 1, optimization, iBatch) ||
|
|
!acProbability.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
index++;
|
|
neuron = new CNeuronBaseOCL();
|
|
if(!neuron ||
|
|
!neuron.Init(0, index, OpenCL, experts, optimization, iBatch) ||
|
|
!acProbability.Add(neuron))
|
|
{
|
|
DeleteObj(neuron);
|
|
ReturnFalse;
|
|
}
|
|
index++;
|
|
dout = new CNeuronDropoutOCL();
|
|
if(!dout ||
|
|
!dout.Init(0, index, OpenCL, experts, dropout, optimization, iBatch) ||
|
|
!acProbability.Add(dout))
|
|
{
|
|
DeleteObj(dout);
|
|
ReturnFalse;
|
|
}
|
|
index++;
|
|
softmax = new CNeuronSoftMaxOCL();
|
|
if(!softmax ||
|
|
!softmax.Init(0, index, OpenCL, experts, optimization, iBatch) ||
|
|
!acProbability.Add(softmax))
|
|
{
|
|
DeleteObj(softmax);
|
|
ReturnFalse;
|
|
}
|
|
softmax.SetHeads(1);
|
|
//--- Data Embedding
|
|
acDataEmb.Clear();
|
|
acDataEmb.SetOpenCL(OpenCL);
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, window, window, 2 * emb_dimension, units, 1, optimization, iBatch) ||
|
|
!acDataEmb.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(SoftPlus);
|
|
index++;
|
|
conv = new CNeuronConvOCL();
|
|
if(!conv ||
|
|
!conv.Init(0, index, OpenCL, 2 * emb_dimension, 2 * emb_dimension, emb_dimension, units, 1, optimization, iBatch) ||
|
|
!acDataEmb.Add(conv))
|
|
{
|
|
DeleteObj(conv);
|
|
ReturnFalse;
|
|
}
|
|
conv.SetActivationFunction(None);
|
|
index++;
|
|
transp = new CNeuronTransposeOCL();
|
|
if(!transp ||
|
|
!transp.Init(0, index, OpenCL, units, emb_dimension, optimization, iBatch) ||
|
|
!acDataEmb.Add(transp))
|
|
{
|
|
DeleteObj(transp);
|
|
ReturnFalse;
|
|
}
|
|
transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
|
|
//---
|
|
index++;
|
|
if(!cExpertsEmb.Init(0, index, OpenCL, units * emb_dimension * experts, optimization, iBatch))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
CNeuronBaseOCL* prev = NeuronOCL;
|
|
CNeuronBaseOCL* current = NULL;
|
|
//--- Probability
|
|
for(int i = 0; i < acProbability.Total(); i++)
|
|
{
|
|
current = acProbability[i];
|
|
if(!current ||
|
|
!current.FeedForward(prev))
|
|
ReturnFalse;
|
|
prev = current;
|
|
}
|
|
//--- Data Embedding
|
|
prev = NeuronOCL;
|
|
for(int i = 0; i < acDataEmb.Total(); i++)
|
|
{
|
|
current = acDataEmb[i];
|
|
if(!current ||
|
|
!current.FeedForward(prev))
|
|
ReturnFalse;
|
|
prev = current;
|
|
}
|
|
//--- Experts
|
|
if(bTrain)
|
|
if(!cExpertsEmb.FeedForward())
|
|
ReturnFalse;
|
|
//--- Graphs
|
|
uint units = (uint)MathSqrt((double)Neurons());
|
|
uint emb_dim = acDataEmb[-1].Neurons() / units;
|
|
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
|
|
if(!MatMul(cExpertsEmb.getOutput(), current.getOutput(), cGraphs.getOutput(),
|
|
units, emb_dim, units, experts, false))
|
|
ReturnFalse;
|
|
if(cGraphs.Activation() != None)
|
|
if(!Activation(cGraphs.getOutput(), cGraphs.getOutput(), cGraphs.Activation()))
|
|
ReturnFalse;
|
|
//--- Result
|
|
if(!MatMul(acProbability[-1].getOutput(), cGraphs.getOutput(), Output, 1, experts, units * units, 1, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!acDataEmb[-1] || !acProbability[-1])
|
|
ReturnFalse;
|
|
//---
|
|
uint units = (uint)MathSqrt((double)Neurons());
|
|
uint emb_dim = acDataEmb[-1].Neurons() / units;
|
|
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
|
|
//---
|
|
if(!MatMulGrad(acProbability[-1].getOutput(), acProbability[-1].getGradient(),
|
|
cGraphs.getOutput(), cGraphs.getGradient(), Gradient,
|
|
1, experts, units * units, 1, false))
|
|
ReturnFalse;
|
|
//--- Graphs
|
|
if(cGraphs.Activation() != None)
|
|
if(!DeActivation(cGraphs.getOutput(), cGraphs.getGradient(),
|
|
cGraphs.getGradient(), cGraphs.Activation()))
|
|
ReturnFalse;
|
|
if(!MatMulGrad(cExpertsEmb.getOutput(), cExpertsEmb.getGradient(),
|
|
acDataEmb[-1].getOutput(), acDataEmb[-1].getGradient(),
|
|
cGraphs.getGradient(), units, emb_dim, units, experts, false))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronBaseOCL* next = NULL;
|
|
CNeuronBaseOCL* current = NULL;
|
|
//--- Probability
|
|
for(int i = acProbability.Total() - 1; i >= 0; i--)
|
|
{
|
|
next = acProbability[i];
|
|
current = (i == 0 ? NeuronOCL : acProbability[i - 1]);
|
|
if(!current ||
|
|
!current.CalcHiddenGradients(next))
|
|
ReturnFalse;
|
|
}
|
|
//--- Data Embedding
|
|
CBufferFloat *temp = NeuronOCL.getGradient();
|
|
if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false))
|
|
ReturnFalse;
|
|
for(int i = acDataEmb.Total() - 1; i >= 0; i--)
|
|
{
|
|
next = acDataEmb[i];
|
|
current = (i == 0 ? NeuronOCL : acDataEmb[i - 1]);
|
|
if(!current ||
|
|
!current.CalcHiddenGradients(next))
|
|
ReturnFalse;
|
|
}
|
|
if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) ||
|
|
!NeuronOCL.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
CNeuronBaseOCL* prev = NeuronOCL;
|
|
CNeuronBaseOCL* current = NULL;
|
|
//--- Probability
|
|
for(int i = 0; i < acProbability.Total(); i++)
|
|
{
|
|
current = acProbability[i];
|
|
if(!current ||
|
|
!current.UpdateInputWeights(prev))
|
|
ReturnFalse;
|
|
prev = current;
|
|
}
|
|
//--- Data Embedding
|
|
prev = NeuronOCL;
|
|
for(int i = 0; i < acDataEmb.Total(); i++)
|
|
{
|
|
current = acDataEmb[i];
|
|
if(!current ||
|
|
!current.UpdateInputWeights(prev))
|
|
ReturnFalse;
|
|
prev = current;
|
|
}
|
|
//--- Experts
|
|
if(!cExpertsEmb.UpdateInputWeights())
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronGraphons *Source = source;
|
|
if(!acProbability.WeightsUpdate(Source.acProbability.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!acDataEmb.WeightsUpdate(Source.acDataEmb.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cExpertsEmb.WeightsUpdate(Source.cExpertsEmb.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!acProbability.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!acDataEmb.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cExpertsEmb.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGraphons::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!acProbability.Load(file_handle))
|
|
ReturnFalse;
|
|
if(!acDataEmb.Load(file_handle))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cExpertsEmb.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons();
|
|
if(!cGraphs.Init(0, 0, OpenCL, Neurons()*experts, optimization, iBatch))
|
|
ReturnFalse;
|
|
cGraphs.SetActivationFunction(SIGMOID);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGraphons::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronBaseOCL::SetOpenCL(obj);
|
|
acProbability.SetOpenCL(OpenCL);
|
|
acDataEmb.SetOpenCL(OpenCL);
|
|
cExpertsEmb.SetOpenCL(OpenCL);
|
|
cGraphs.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGraphons::TrainMode(bool flag)
|
|
{
|
|
CNeuronBaseOCL::TrainMode(flag);
|
|
acProbability.TrainMode(bTrain);
|
|
acDataEmb.TrainMode(bTrain);
|
|
cExpertsEmb.TrainMode(bTrain);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSparseSoftMax::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint dimension_in, uint dimension_out,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(dimension_in < dimension_out)
|
|
ReturnFalse;
|
|
if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, units * dimension_out, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
SetHeads(units);
|
|
iDimensionIn = dimension_in;
|
|
if(!cIndexes.BufferInit(Neurons(), -1) ||
|
|
!cIndexes.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSparseSoftMax::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!OpenCL || !NeuronOCL ||
|
|
NeuronOCL.Neurons() < int(iHeads * iDimensionIn))
|
|
ReturnFalse;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2] = { iHeads, iDimensionIn };
|
|
uint local_work_size[2] = { 1, global_work_size[1] };
|
|
uint kernel = def_k_SparseSoftMax;
|
|
setBuffer(kernel, def_k_ssoftmax_data, NeuronOCL.getOutputIndex())
|
|
setBuffer(kernel, def_k_ssoftmax_outputs, getOutputIndex())
|
|
setBuffer(kernel, def_k_ssoftmax_indexes, cIndexes.GetIndex())
|
|
setArgument(kernel, def_k_ssoftmax_out_dimension, int(DimensionOut()))
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!Output.BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSparseSoftMax::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!OpenCL || !NeuronOCL ||
|
|
NeuronOCL.Neurons() < int(iHeads * iDimensionIn))
|
|
ReturnFalse;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2] = { iHeads, iDimensionIn };
|
|
uint local_work_size[2] = { 1, global_work_size[1] };
|
|
uint kernel = def_k_SparseSoftMaxGrad;
|
|
setBuffer(kernel, def_k_ssoftmax_gr_data_gr, NeuronOCL.getGradientIndex())
|
|
setBuffer(kernel, def_k_ssoftmax_gr_outputs, getOutputIndex())
|
|
setBuffer(kernel, def_k_ssoftmax_gr_outputs_gr, getGradientIndex())
|
|
setBuffer(kernel, def_k_ssoftmax_gr_indexes, cIndexes.GetIndex())
|
|
setArgument(kernel, def_k_ssoftmax_gr_out_dimension, int(DimensionOut()))
|
|
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
|
|
#ifdef _DEBUG
|
|
if(!NeuronOCL.getGradient().BufferRead())
|
|
ReturnFalse;
|
|
#endif
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSparseSoftMax::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronSoftMaxOCL::Save(file_handle))
|
|
ReturnFalse;
|
|
if(FileWriteInteger(file_handle, iDimensionIn) < INT_VALUE)
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronSparseSoftMax::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronSoftMaxOCL::Load(file_handle))
|
|
ReturnFalse;
|
|
if(FileIsEnding(file_handle))
|
|
ReturnFalse;
|
|
iDimensionIn = (uint)FileReadInteger(file_handle);
|
|
//---
|
|
cIndexes.BufferFree();
|
|
if(!cIndexes.BufferInit(Neurons(), -1) ||
|
|
!cIndexes.BufferCreate(OpenCL))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronSparseSoftMax::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronSoftMaxOCL::SetOpenCL(obj);
|
|
cIndexes.BufferCreate(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
|
|
uint units, uint window, uint experts, float dropout,
|
|
uint emb_dimension, uint sparse_dimension,
|
|
ENUM_OPTIMIZATION optimization_type, uint batch)
|
|
{
|
|
if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, window, units,
|
|
2, optimization_type, batch))
|
|
ReturnFalse;
|
|
//---
|
|
int index = 0;
|
|
if(!cGlobal.Init(0, index, OpenCL, iUnits, iWindow, emb_dimension, experts, dropout,
|
|
optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cLocal.Init(0, index, OpenCL, iUnits, iWindow, experts, dropout, emb_dimension,
|
|
sparse_dimension, optimization, iBatch))
|
|
ReturnFalse;
|
|
index++;
|
|
if(!cConcat.Init(0, index, OpenCL, 2 * iWindow * iUnits, optimization, iBatch))
|
|
ReturnFalse;
|
|
cConcat.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cGlobal.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cLocal.FeedForward(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!Concat(cGlobal.getOutput(), cLocal.getOutput(), cConcat.getOutput(),
|
|
iWindow, iWindow, iUnits))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronMHAttentionPooling::feedForward(cConcat.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!NeuronOCL)
|
|
ReturnFalse;
|
|
//---
|
|
if(!CNeuronMHAttentionPooling::calcInputGradients(cConcat.AsObject()))
|
|
ReturnFalse;
|
|
if(!DeConcat(cGlobal.getGradient(), cLocal.getGradient(), cConcat.getGradient(),
|
|
iWindow, iWindow, iUnits))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cGlobal.AsObject()))
|
|
ReturnFalse;
|
|
CBufferFloat *temp = NeuronOCL.getGradient();
|
|
if(!NeuronOCL.SetGradient(PrevOutput, false))
|
|
ReturnFalse;
|
|
if(!NeuronOCL.CalcHiddenGradients(cLocal.AsObject()))
|
|
ReturnFalse;
|
|
if(!SumAndNormalize(temp, PrevOutput, temp, iWindow, false, 0, 0, 0, 1) ||
|
|
!NeuronOCL.SetGradient(temp, false))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(!cGlobal.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
if(!cLocal.UpdateInputWeights(NeuronOCL))
|
|
ReturnFalse;
|
|
//---
|
|
return CNeuronMHAttentionPooling::updateInputWeights(cConcat.AsObject());
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::WeightsUpdate(CNeuronBaseOCL *source, float tau)
|
|
{
|
|
if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau))
|
|
ReturnFalse;
|
|
//---
|
|
CNeuronGlobLocGraphAtt* Source = source;
|
|
if(!cGlobal.WeightsUpdate(Source.cGlobal.AsObject(), tau))
|
|
ReturnFalse;
|
|
if(!cLocal.WeightsUpdate(Source.cLocal.AsObject(), tau))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronMHAttentionPooling::Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cGlobal.Save(file_handle))
|
|
ReturnFalse;
|
|
if(!cLocal.Save(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGlobLocGraphAtt::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronMHAttentionPooling::Load(file_handle))
|
|
ReturnFalse;
|
|
//---
|
|
if(!LoadInsideLayer(file_handle, cGlobal.AsObject()))
|
|
ReturnFalse;
|
|
if(!LoadInsideLayer(file_handle, cLocal.AsObject()))
|
|
ReturnFalse;
|
|
//---
|
|
if(!cConcat.Init(0, 2, OpenCL, 2 * iWindow * iUnits, optimization, iBatch))
|
|
ReturnFalse;
|
|
cConcat.SetActivationFunction(None);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGlobLocGraphAtt::SetOpenCL(COpenCLMy *obj)
|
|
{
|
|
CNeuronMHAttentionPooling::SetOpenCL(obj);
|
|
//---
|
|
cGlobal.SetOpenCL(OpenCL);
|
|
cLocal.SetOpenCL(OpenCL);
|
|
cConcat.SetOpenCL(OpenCL);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronGlobLocGraphAtt::TrainMode(bool flag)
|
|
{
|
|
CNeuronMHAttentionPooling::TrainMode(flag);
|
|
//---
|
|
cGlobal.TrainMode(flag);
|
|
cLocal.TrainMode(flag);
|
|
cConcat.TrainMode(flag);
|
|
}
|