dng/NN_in_Trading
1
2
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
NN_in_Trading/Experts/NeuroNet_DNG/NeuroNet_Misc.mqh
DNG f56b206ae2 MomAD
2026-06-18 13:05:27 +03:00

2651 lines
92 KiB
MQL5
Raw Permalink Blame History

#ifndef NEURONET_BUILDING_FACADE
#include "NeuroNet.mqh"
#endif // NEURONET_BUILDING_FACADE
//+------------------------------------------------------------------+
//| Logical method implementations generated from NeuroNet.mqh: Misc
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Init(uint numInputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models)
{
if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
ReturnFalse;
OpenCL = open_cl;
optimization = ADAM;
iBatch = 1;
iModels = models;
//---
if(CheckPointer(Output) == POINTER_INVALID)
{
Output = new CBufferFloat();
if(CheckPointer(Output) == POINTER_INVALID)
ReturnFalse;
}
if(!Output.BufferInit(numNeurons * models, 0.0))
ReturnFalse;
if(!Output.BufferCreate(OpenCL))
ReturnFalse;
//---
if(CheckPointer(PrevOutput) == POINTER_INVALID)
{
PrevOutput = new CBufferFloat();
if(CheckPointer(PrevOutput) == POINTER_INVALID)
ReturnFalse;
}
if(!PrevOutput.BufferInit(numNeurons * models, 1.0))
ReturnFalse;
if(!PrevOutput.BufferCreate(OpenCL))
ReturnFalse;
//---
if(CheckPointer(Gradient) == POINTER_INVALID)
{
Gradient = new CBufferFloat();
if(CheckPointer(Gradient) == POINTER_INVALID)
ReturnFalse;
}
if(!Gradient.BufferInit((numNeurons + 1)*models, 0.0))
ReturnFalse;
if(!Gradient.BufferCreate(OpenCL))
ReturnFalse;
//---
if(CheckPointer(Weights) == POINTER_INVALID)
{
Weights = new CBufferFloat();
if(CheckPointer(Weights) == POINTER_INVALID)
ReturnFalse;
}
int count = (int)((numInputs + 1) * numNeurons * models);
if(!Weights.Reserve(count))
ReturnFalse;
float k = (float)(1 / sqrt(numInputs + 1));
for(int i = 0; i < count; i++)
{
if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier))
ReturnFalse;
}
if(!Weights.BufferCreate(OpenCL))
ReturnFalse;
//---
DeleteObj(DeltaWeights);
//---
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
{
FirstMomentum = new CBufferFloat();
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
ReturnFalse;
}
if(!FirstMomentum.BufferInit(count, 0))
ReturnFalse;
if(!FirstMomentum.BufferCreate(OpenCL))
ReturnFalse;
//---
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
{
SecondMomentum = new CBufferFloat();
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
ReturnFalse;
}
if(!SecondMomentum.BufferInit(count, 0))
ReturnFalse;
if(!SecondMomentum.BufferCreate(OpenCL))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::feedForward(CNeuronBaseOCL* NeuronOCL)
{
if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
ReturnFalse;
uint global_work_offset[2] = {0, 0};
uint global_work_size[2];
global_work_size[0] = Output.Total() / iModels;
global_work_size[1] = iModels;
setBuffer(def_k_FFMultiModels, def_k_ff_matrix_w, getWeightsIndex())
setBuffer(def_k_FFMultiModels, def_k_ff_matrix_i, NeuronOCL.getOutputIndex())
setBuffer(def_k_FFMultiModels, def_k_ff_matrix_o, Output.GetIndex())
setArgument(def_k_FFMultiModels, def_k_ff_inputs, NeuronOCL.Neurons() / iModels)
setArgument(def_k_FFMultiModels, def_k_ff_activation, (int)activation)
kernelExecute(def_k_FFMultiModels, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!Output.BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::calcHiddenGradients(CNeuronBaseOCL* NeuronOCL)
{
if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
ReturnFalse;
uint global_work_offset[2] = {0, 0};
uint global_work_size[2];
global_work_size[0] = NeuronOCL.Neurons() / iModels;
global_work_size[1] = iModels;
setBuffer(def_k_HGMultiModels, def_k_chg_matrix_w, getWeightsIndex())
setBuffer(def_k_HGMultiModels, def_k_chg_matrix_g, getGradientIndex())
setBuffer(def_k_HGMultiModels, def_k_chg_matrix_o, NeuronOCL.getOutputIndex())
setBuffer(def_k_HGMultiModels, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex())
setArgument(def_k_HGMultiModels, def_k_chg_outputs, Neurons() / iModels)
setArgument(def_k_HGMultiModels, def_k_chg_activation, NeuronOCL.Activation())
iUpdateModel = (int)MathRound(MathRand() / 32767.0 * (iModels - 1));
setArgument(def_k_HGMultiModels, def_k_chg_model, iUpdateModel)
//Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__);
kernelExecute(def_k_HGMultiModels, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!NeuronOCL.getGradient().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
{
if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
ReturnFalse;
uint global_work_offset[2] = {0, 0};
uint global_work_size[2];
global_work_size[0] = Neurons() / iModels;
global_work_size[1] = NeuronOCL.Neurons() / iModels + 1;
uint rest = 0;
float lt = lr;
setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_w, getWeightsIndex())
setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_g, getGradientIndex())
setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())
setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_m, getFirstMomentumIndex())
setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_v, getSecondMomentumIndex())
lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t)));
setArgument(def_k_UWMultiModels, def_k_uwa_inputs, NeuronOCL.Neurons() / iModels)
setArgument(def_k_UWMultiModels, def_k_uwa_l, lt)
setArgument(def_k_UWMultiModels, def_k_uwa_b1, b1)
setArgument(def_k_UWMultiModels, def_k_uwa_b2, b2)
setArgument(def_k_UWMultiModels, def_k_uwa_model, iUpdateModel)
global_work_size[1] = (global_work_size[1] + 3) / 4;
ResetLastError();
kernelExecute(def_k_UWMultiModels, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!Weights.BufferRead())
ReturnFalse;
#endif
t++;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
if(FileWriteInteger(file_handle, iModels) <= 0)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
iModels = FileReadInteger(file_handle);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronConcatenate::CNeuronConcatenate(void) : i_SecondInputs(0)
{
ConcWeights = new CBufferFloat();
ConcDeltaWeights = new CBufferFloat();
ConcFirstMomentum = new CBufferFloat();
ConcSecondMomentum = new CBufferFloat;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronConcatenate::~CNeuronConcatenate()
{
DeleteObj(ConcWeights);
DeleteObj(ConcDeltaWeights);
DeleteObj(ConcFirstMomentum);
DeleteObj(ConcSecondMomentum);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint numInputs1, uint numInputs2, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch))
ReturnFalse;
//---
i_SecondInputs = (int)numInputs2;
if(!ConcWeights)
{
ConcWeights = new CBufferFloat();
if(!ConcWeights)
ReturnFalse;
}
int count = (int)((numInputs1 + numInputs2 + 1) * numNeurons);
if(!ConcWeights.Reserve(count))
ReturnFalse;
float k = (float)(1 / sqrt(numNeurons + 1));
for(int i = 0; i < count; i++)
{
if(!ConcWeights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier))
ReturnFalse;
}
if(!ConcWeights.BufferCreate(OpenCL))
ReturnFalse;
//---
if(optimization == SGD)
{
if(!ConcDeltaWeights)
{
ConcDeltaWeights = new CBufferFloat();
if(!ConcDeltaWeights)
ReturnFalse;
}
if(!ConcDeltaWeights.BufferInit(count, 0))
ReturnFalse;
if(!ConcDeltaWeights.BufferCreate(OpenCL))
ReturnFalse;
DeleteObj(ConcFirstMomentum);
DeleteObj(ConcSecondMomentum);
}
else
{
DeleteObj(ConcDeltaWeights);
//---
if(!ConcFirstMomentum)
{
ConcFirstMomentum = new CBufferFloat();
if(CheckPointer(ConcFirstMomentum) == POINTER_INVALID)
ReturnFalse;
}
if(!ConcFirstMomentum.BufferInit(count, 0))
ReturnFalse;
if(!ConcFirstMomentum.BufferCreate(OpenCL))
ReturnFalse;
//---
if(!ConcSecondMomentum)
{
ConcSecondMomentum = new CBufferFloat();
if(!ConcSecondMomentum)
ReturnFalse;
}
if(!ConcSecondMomentum.BufferInit(count, 0))
ReturnFalse;
if(!ConcSecondMomentum.BufferCreate(OpenCL))
ReturnFalse;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput)
{
if(!OpenCL || !NeuronOCL || !SecondInput)
ReturnFalse;
if(SecondInput.Total() < i_SecondInputs)
ReturnFalse;
if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
ReturnFalse;
//---
setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_w, ConcWeights.GetIndex())
setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i1, NeuronOCL.getOutputIndex())
setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i2, SecondInput.GetIndex())
setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_o, Output.GetIndex())
setArgument(def_k_ConcatFeedForward, def_k_cff_inputs1, (int)NeuronOCL.Neurons())
setArgument(def_k_ConcatFeedForward, def_k_cff_inputs2, (int)i_SecondInputs)
setArgument(def_k_ConcatFeedForward, def_k_cff_activation, (int)activation)
//---
uint global_work_offset[1] = {0};
uint global_work_size[1];
global_work_size[0] = Output.Total();
kernelExecute(def_k_ConcatFeedForward, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!Output.BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None)
{
if(!OpenCL || !NeuronOCL || !SecondInput || !SecondGradient)
ReturnFalse;
if(SecondInput.Total() < i_SecondInputs || SecondGradient.Total() < i_SecondInputs)
ReturnFalse;
if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
ReturnFalse;
if(SecondGradient.GetIndex() < 0 && !SecondGradient.BufferCreate(OpenCL))
ReturnFalse;
//---
uint global_work_offset[1] = {0};
uint global_work_size[1];
global_work_size[0] = NeuronOCL.Neurons() + i_SecondInputs;
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_w, ConcWeights.GetIndex())
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_g, Gradient.GetIndex())
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig1, NeuronOCL.getGradientIndex())
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig2, SecondGradient.GetIndex())
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o1, NeuronOCL.getOutputIndex())
setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o2, SecondInput.GetIndex())
setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs1, NeuronOCL.Neurons())
setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs2, i_SecondInputs)
setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_outputs, Neurons())
setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation1, NeuronOCL.Activation())
setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation2, (int)SecondActivation)
kernelExecute(def_k_ConcatCalcHiddenGradient, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!ConcWeights.BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput)
{
if(!OpenCL || !NeuronOCL || !SecondInput)
ReturnFalse;
if(SecondInput.Total() < i_SecondInputs)
ReturnFalse;
if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
ReturnFalse;
//---
uint global_work_offset[2] = {0, 0};
uint global_work_size[2];
global_work_size[0] = Neurons();
global_work_size[1] = NeuronOCL.Neurons() + i_SecondInputs + 1;
float lt = lr;
ResetLastError();
switch(NeuronOCL.Optimization())
{
case SGD:
setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_w, ConcWeights.GetIndex())
setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_g, getGradientIndex())
setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i1, NeuronOCL.getOutputIndex())
setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i2, SecondInput.GetIndex())
setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_dw, ConcDeltaWeights.GetIndex())
setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, NeuronOCL.Neurons())
setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, i_SecondInputs)
setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_learning_rates, lr)
setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_momentum, alpha)
ResetLastError();
kernelExecute(def_k_ConcatUpdWeightsMomentum, global_work_offset, global_work_size)
break;
case ADAM:
case ADAM_MINI:
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_w, ConcWeights.GetIndex())
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_g, getGradientIndex())
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i1, NeuronOCL.getOutputIndex())
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i2, SecondInput.GetIndex())
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_m, ConcFirstMomentum.GetIndex())
setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_v, ConcSecondMomentum.GetIndex())
lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t)));
setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs1, NeuronOCL.Neurons())
setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs2, i_SecondInputs)
setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_l, lt)
setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b1, b1)
setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b2, b2)
kernelExecute(def_k_ConcatUpdWeightsAdam, global_work_offset, global_work_size)
t++;
break;
default:
ReturnFalse;
break;
}
#ifdef _DEBUG
if(!ConcWeights.BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
//---
if(FileWriteInteger(file_handle, i_SecondInputs) < INT_VALUE)
ReturnFalse;
if(!ConcWeights.Save(file_handle))
ReturnFalse;
if(optimization == SGD)
{
if(!ConcDeltaWeights.Save(file_handle))
ReturnFalse;
}
else
{
if(!ConcFirstMomentum.Save(file_handle))
ReturnFalse;
if(!ConcSecondMomentum.Save(file_handle))
ReturnFalse;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
//---
i_SecondInputs = FileReadInteger(file_handle);
if(!ConcWeights.Load(file_handle))
ReturnFalse;
ConcWeights.BufferCreate(OpenCL);
if(optimization == SGD)
{
if(!ConcDeltaWeights.Load(file_handle))
ReturnFalse;
ConcDeltaWeights.BufferCreate(OpenCL);
}
else
{
if(!ConcFirstMomentum.Load(file_handle))
ReturnFalse;
if(!ConcSecondMomentum.Load(file_handle))
ReturnFalse;
ConcFirstMomentum.BufferCreate(OpenCL);
ConcSecondMomentum.BufferCreate(OpenCL);
}
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronConcatenate::SetOpenCL(COpenCLMy * obj)
{
CNeuronBaseOCL::SetOpenCL(obj);
if(!ConcWeights)
return;
ConcWeights.BufferCreate(OpenCL);
if(optimization == SGD)
{
if(!ConcDeltaWeights)
return;
ConcDeltaWeights.BufferCreate(OpenCL);
}
else
{
if(!ConcFirstMomentum || !ConcSecondMomentum)
return;
ConcFirstMomentum.BufferCreate(OpenCL);
ConcSecondMomentum.BufferCreate(OpenCL);
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::WeightsUpdate(CNeuronBaseOCL* source, float tau)
{
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
ReturnFalse;
//---
CNeuronConcatenate *temp = source;
if(ConcWeights.Total() != temp.ConcWeights.Total())
ReturnFalse;
//---
uint global_work_offset[1] = {0};
uint global_work_size[1] = {ConcWeights.Total()};
ResetLastError();
if(tau != 1.0f && optimization == ADAM)
{
setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, ConcWeights.GetIndex())
setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.ConcWeights.GetIndex())
setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, ConcFirstMomentum.GetIndex())
setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, ConcSecondMomentum.GetIndex())
setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau)
setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1)
setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2)
kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!ConcFirstMomentum.BufferRead())
ReturnFalse;
#endif
}
else
{
setBuffer(def_k_SoftUpdate, def_k_su_target, ConcWeights.GetIndex())
setBuffer(def_k_SoftUpdate, def_k_su_source, temp.ConcWeights.GetIndex())
setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau)
kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!ConcWeights.BufferRead())
ReturnFalse;
#endif
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch)
{
const uint layers = windows.Size();
if(layers <= 0)
ReturnFalse;
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs_count * variables, optimization_type, batch))
ReturnFalse;
//---
ia_Windows.Clear();
ia_Windows.Reserve(layers);
for(uint i = 0; i < layers; i++)
if(!ia_Windows.Add((int)windows[i]))
ReturnFalse;
i_Variables = variables;
i_Count = inputs_count / ia_Windows[0];
b_NeedTranspose = need_transpose;
//---
if(b_NeedTranspose)
{
CNeuronTransposeOCL *transp = new CNeuronTransposeOCL();
if(!transp)
ReturnFalse;
if(!transp.Init(0, 0, OpenCL, inputs_count, i_Variables, optimization, iBatch))
{
DeleteObj(transp);
ReturnFalse;
}
if(!caTranspose.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
transp = new CNeuronTransposeOCL();
if(!transp)
ReturnFalse;
if(!transp.Init(0, 1, OpenCL, i_Variables, inputs_count, optimization, iBatch))
{
DeleteObj(transp);
ReturnFalse;
}
if(!caTranspose.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
if(!SetOutput(transp.getOutput()) ||
!SetGradient(transp.getGradient())
)
ReturnFalse;
}
//---
uint total = ia_Windows[0] * i_Count;
CNeuronConvOCL *conv = new CNeuronConvOCL();
if(!conv.Init(0, 0, OpenCL, inputs_count, inputs_count, total, 1, i_Variables, optimization, iBatch))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(SIGMOID);
if(!caConvolutions.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
//---
total = 0;
for(uint i = 0; i < layers; i++)
{
conv = new CNeuronConvOCL();
if(!conv.Init(0, i + 1, OpenCL, ia_Windows[i], ia_Windows[i], (i < (layers - 1) ? ia_Windows[i + 1] : 1), i_Count, i_Variables, optimization, iBatch))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(SIGMOID);
if(!caConvolutions.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
if(!caConvOutputs.Add(conv.getOutput()) ||
!caConvGradients.Add(conv.getGradient())
)
ReturnFalse;
total += conv.Neurons();
}
//---
CNeuronBaseOCL *comul = new CNeuronBaseOCL();
if(!comul.Init(0, 0, OpenCL, total, optimization, iBatch))
{
DeleteObj(comul);
ReturnFalse;
}
if(!caMLP.Add(comul))
{
DeleteObj(comul);
ReturnFalse;
}
if(layers == 1)
{
comul.SetOutput(conv.getOutput());
comul.SetGradient(conv.getGradient());
}
conv = new CNeuronConvOCL();
if(!conv.Init(0, 0, OpenCL, total / i_Variables, total / i_Variables, inputs_count, 1, i_Variables, optimization, iBatch))
{
DeleteObj(conv);
ReturnFalse;
}
if(!caMLP.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
if(!b_NeedTranspose)
{
if(!SetOutput(conv.getOutput()) ||
!SetGradient(conv.getGradient())
)
ReturnFalse;
}
//---
CBufferFloat *buf = new CBufferFloat();
if(!buf)
ReturnFalse;
if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
!caTemp.Add(buf))
{
DeleteObj(buf);
ReturnFalse;
}
buf = new CBufferFloat();
if(!buf)
ReturnFalse;
if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
!caTemp.Add(buf))
{
DeleteObj(buf);
ReturnFalse;
}
buf = new CBufferFloat();
if(!buf)
ReturnFalse;
if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
!caTemp.Add(buf))
{
DeleteObj(buf);
ReturnFalse;
}
//---
caConvOutputs.FreeMode(false);
caConvGradients.FreeMode(false);
caTranspose.FreeMode(true);
caConvolutions.FreeMode(true);
caMLP.FreeMode(true);
caTemp.FreeMode(true);
SetActivationFunction(None);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::feedForward(CNeuronBaseOCL* NeuronOCL)
{
CNeuronBaseOCL *inp = NeuronOCL;
CNeuronBaseOCL *current = NULL;
//---
if(b_NeedTranspose)
{
current = caTranspose.At(0);
if(!current ||
!current.FeedForward(inp))
ReturnFalse;
inp = current;
}
//---
int layers = caConvolutions.Total() - 1;
for(int l = 0; l <= layers; l++)
{
current = caConvolutions.At(l);
if(!current ||
!current.FeedForward(inp))
ReturnFalse;
inp = current;
}
//---
current = caMLP.At(0);
if(!current)
ReturnFalse;
switch(layers)
{
case 0:
ReturnFalse;
case 1:
break;
case 2:
if(!Concat(caConvOutputs.At(0),
caConvOutputs.At(1),
current.getOutput(),
ia_Windows[1], 1,
i_Variables * i_Count))
ReturnFalse;
break;
case 3:
if(!Concat(caConvOutputs.At(0),
caConvOutputs.At(1),
caConvOutputs.At(2),
current.getOutput(),
ia_Windows[1], ia_Windows[2], 1,
i_Variables * i_Count))
ReturnFalse;
break;
case 4:
if(!Concat(caConvOutputs.At(0),
caConvOutputs.At(1),
caConvOutputs.At(2),
caConvOutputs.At(3),
current.getOutput(),
ia_Windows[1], ia_Windows[2], ia_Windows[3], 1,
i_Variables * i_Count))
ReturnFalse;
break;
default:
if(!Concat(caConvOutputs.At(0),
caConvOutputs.At(1),
caConvOutputs.At(2),
caConvOutputs.At(3),
caTemp.At(0),
ia_Windows[1], ia_Windows[2], ia_Windows[3], ia_Windows[4],
i_Variables * i_Count))
ReturnFalse;
break;
}
uint last_buf = 0;
for(int i = 4; i < layers; i += 3)
{
uint buf_size = 0;
for(int j = 1; j <= i; j++)
buf_size += ia_Windows[j];
switch(layers - i)
{
case 1:
if(!Concat(caTemp.At(last_buf),
caConvOutputs.At(i),
current.getOutput(),
buf_size, 1,
i_Variables * i_Count))
ReturnFalse;
break;
case 2:
if(!Concat(caTemp.At(last_buf),
caConvOutputs.At(i),
caConvOutputs.At(i + 1),
current.getOutput(),
buf_size, ia_Windows[i + 1], 1,
i_Variables * i_Count))
ReturnFalse;
break;
case 3:
if(!Concat(caTemp.At(last_buf),
caConvOutputs.At(i),
caConvOutputs.At(i + 1),
caConvOutputs.At(i + 2),
current.getOutput(),
buf_size, ia_Windows[i + 1], ia_Windows[i + 2], 1,
i_Variables * i_Count))
ReturnFalse;
break;
default:
if(!Concat(caTemp.At(last_buf),
caConvOutputs.At(i),
caConvOutputs.At(i + 1),
caConvOutputs.At(i + 2),
caTemp.At((last_buf + 1) % 2),
buf_size, ia_Windows[i + 1], ia_Windows[i + 2], ia_Windows[i + 3],
i_Variables * i_Count))
ReturnFalse;
break;
}
last_buf = (last_buf + 1) % 2;
}
//---
inp = current;
current = caMLP.At(1);
if(!current ||
!current.FeedForward(inp))
ReturnFalse;
//---
if(b_NeedTranspose)
{
inp = current;
current = caTranspose.At(1);
if(!current ||
!current.FeedForward(inp))
ReturnFalse;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::calcInputGradients(CNeuronBaseOCL* prevLayer)
{
if(!prevLayer)
ReturnFalse;
//---
CNeuronBaseOCL *current = caMLP.At(0);
CNeuronBaseOCL *next = caMLP.At(1);
if(b_NeedTranspose)
{
if(!next.CalcHiddenGradients(caTranspose.At(1)))
ReturnFalse;
}
if(!current.CalcHiddenGradients(next.AsObject()))
ReturnFalse;
next = current;
//---
int layers = caConvGradients.Total();
if(layers == 1)
{
next = caConvolutions.At(1);
if(next.Activation() != None)
{
if(!DeActivation(next.getOutput(), next.getGradient(), next.getGradient(), next.Activation()))
ReturnFalse;
}
}
else
{
int prev_window = 0;
for(int i = 1; i < layers; i++)
prev_window += int(ia_Windows[i]);
if(!DeConcat(caTemp.At(0), caConvGradients.At(layers - 1), next.getGradient(), prev_window, 1, i_Variables * i_Count))
ReturnFalse;
next = caConvolutions.At(layers);
int current_buf = 0;
for(int l = layers; l > 1; l--)
{
current = caConvolutions.At(l - 1);
if(!current.CalcHiddenGradients(next.AsObject()))
ReturnFalse;
int window = int(ia_Windows[l - 1]);
prev_window -= window;
if(!DeConcat(caTemp.At((current_buf + 1) % 2), caTemp.At(2), caTemp.At(current_buf), prev_window, window, i_Variables * i_Count))
ReturnFalse;
if(current.Activation() != None)
{
if(!DeActivation(current.getOutput(), caTemp.At(2), caTemp.At(2), current.Activation()))
ReturnFalse;
}
if(!SumAndNormalize(current.getGradient(), caTemp.At(2), current.getGradient(), 1, false, 0, 0, 0, 1))
ReturnFalse;
next = current;
current_buf = (current_buf + 1) % 2;
}
}
current = caConvolutions.At(0);
if(!current.CalcHiddenGradients(next.AsObject()))
ReturnFalse;
next = current;
//---
if(b_NeedTranspose)
{
current = caTranspose.At(0);
if(!current.CalcHiddenGradients(next.AsObject()))
ReturnFalse;
next = current;
}
//---
if(!prevLayer.CalcHiddenGradients(next.AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
{
CObject *prev = (b_NeedTranspose ? caTranspose.At(0) : NeuronOCL);
CNeuronBaseOCL *current = NULL;
for(int i = 0; i < caConvolutions.Total(); i++)
{
current = caConvolutions.At(1);
if(!current ||
!current.UpdateInputWeights(prev)
)
ReturnFalse;
prev = current;
}
current = caMLP.At(1);
if(!current ||
!current.UpdateInputWeights(caMLP.At(0))
)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
//--- Save constants
if(FileWriteInteger(file_handle, int(i_Variables)) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, int(i_Count)) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, int(b_NeedTranspose)) < INT_VALUE)
ReturnFalse;
//--- Objects
if(!ia_Windows.Save(file_handle))
ReturnFalse;
if(b_NeedTranspose)
{
for(int i = 0; i < 2; i++)
if(!caTranspose.At(i).Save(file_handle))
ReturnFalse;
}
int layers = caConvolutions.Total();
if(FileWriteInteger(file_handle, layers) < INT_VALUE)
ReturnFalse;
for(int i = 0; i < layers; i++)
{
if(!caConvolutions.At(i).Save(file_handle))
ReturnFalse;
}
for(int i = 0; i < 2; i++)
if(!caMLP.At(i).Save(file_handle))
ReturnFalse;
for(int i = 0; i < 3; i++)
if(!caTemp.At(i).Save(file_handle))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Load(const int file_handle)
{
CBufferFloat *temp = new CBufferFloat();
if(!temp)
ReturnFalse;
if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) ||
!SetOutput(temp, false))
{
DeleteObj(temp);
ReturnFalse;
}
temp = new CBufferFloat();
if(!temp)
ReturnFalse;
if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) ||
!SetGradient(temp, false))
{
DeleteObj(temp);
ReturnFalse;
}
//---
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
//--- Load constants
if(FileIsEnding(file_handle))
ReturnFalse;
i_Variables = uint(FileReadInteger(file_handle));
if(FileIsEnding(file_handle))
ReturnFalse;
i_Count = uint(FileReadInteger(file_handle));
if(FileIsEnding(file_handle))
ReturnFalse;
b_NeedTranspose = bool(FileReadInteger(file_handle));
if(FileIsEnding(file_handle))
ReturnFalse;
//--- Objects
if(!ia_Windows.Load(file_handle))
ReturnFalse;
if(b_NeedTranspose)
{
caTranspose.Clear();
CNeuronTransposeOCL *transp = NULL;
for(int i = 0; i < 2; i++)
{
transp = new CNeuronTransposeOCL();
if(!transp)
ReturnFalse;
if(!transp.Init(0, i, OpenCL, 1, 1, optimization, iBatch) ||
!LoadInsideLayer(file_handle, transp) ||
!caTranspose.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
}
}
int layers = FileReadInteger(file_handle);
caConvolutions.Clear();
caConvOutputs.Clear();
caConvGradients.Clear();
CNeuronConvOCL *conv = NULL;
for(int i = 0; i < layers; i++)
{
conv = new CNeuronConvOCL();
if(!conv)
ReturnFalse;
if(!conv.Init(0, i, OpenCL, 1, 1, 1, 1, 1, optimization, iBatch) ||
!LoadInsideLayer(file_handle, conv) ||
!caConvolutions.Add(conv)
)
{
DeleteObj(conv);
ReturnFalse;
}
if(i > 0)
{
if(!caConvOutputs.Add(conv.getOutput()) ||
!caConvGradients.Add(conv.getGradient())
)
ReturnFalse;
}
}
for(int i = 0; i < 2; i++)
if(!LoadInsideLayer(file_handle, caMLP.At(i)))
ReturnFalse;
for(int i = 0; i < 3; i++)
if(!caTemp.At(i).Load(file_handle))
ReturnFalse;
//---
CNeuronBaseOCL *neuron = (b_NeedTranspose ? caTranspose.At(1) : caMLP.At(1));
if(!SetOutput(neuron.getOutput()) ||
!SetGradient(neuron.getGradient())
)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronCSCMOCL::SetOpenCL(COpenCLMy * obj)
{
CNeuronBaseOCL::SetOpenCL(obj);
CNeuronBaseOCL *current = NULL;
if(b_NeedTranspose)
for(int i = 0; i < 2; i++)
{
current = caTranspose.At(i);
current.SetOpenCL(OpenCL);
}
//---
for(int i = 0; i < caConvolutions.Total(); i++)
{
current = caConvolutions.At(i);
current.SetOpenCL(OpenCL);
}
//---
for(int i = 0; i < 2; i++)
{
current = caMLP.At(i);
current.SetOpenCL(OpenCL);
}
for(int i = 0; i < 3; i++)
{
CBufferFloat *buf = caTemp.At(i);
buf.BufferCreate(OpenCL);
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau)
{
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
ReturnFalse;
CNeuronCSCMOCL *Source = source;
if(caConvolutions.Total() != Source.caConvolutions.Total())
ReturnFalse;
CNeuronBaseOCL *current = NULL;
for(int i = 0; i < caConvolutions.Total(); i++)
{
current = caConvolutions.At(i);
if(!current ||
!current.WeightsUpdate(Source.caConvolutions.At(i), tau)
)
ReturnFalse;
}
current = caMLP.At(1);
if(!current ||
!current.WeightsUpdate(Source.caMLP.At(1), tau)
)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch))
ReturnFalse;
if(!cAddNoise.Init(0, 0, OpenCL, window * units_count, iBatch, optimization))
ReturnFalse;
cAddNoise.SetActivationFunction(None);
if(!cAttention[0].Init(0, 1, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch))
ReturnFalse;
if(!cMergeSplit[0].Init(0, 2, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch))
ReturnFalse;
if(inside_block > 0)
{
CNeuronDiffusion *temp = new CNeuronDiffusion();
if(!temp)
ReturnFalse;
if(!temp.Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, inside_block - 1, optimization, iBatch))
{
DeleteObj(temp);
ReturnFalse;
}
cNeck = temp;
}
else
{
CNeuronConvOCL *temp = new CNeuronConvOCL();
if(!temp)
ReturnFalse;
if(!temp.Init(0, 3, OpenCL, window, window, window, (units_count + 1) / 2, optimization, iBatch))
{
DeleteObj(temp);
ReturnFalse;
}
cNeck = temp;
}
if(!cAttention[1].Init(0, 4, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch))
ReturnFalse;
if(!cMergeSplit[1].Init(0, 5, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch))
ReturnFalse;
if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch))
ReturnFalse;
if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true))
ReturnFalse;
cResidual.SetActivationFunction((ENUM_ACTIVATION)cMergeSplit[1].Activation());
//---
if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject()))
ReturnFalse;
//---
if(!SetOutput(cRevIn.getOutput(), true))
ReturnFalse;
//---
cRevIn.SetActivationFunction(None);
SetActivationFunction(None);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::feedForward(CNeuronBaseOCL* NeuronOCL)
{
if(!cAddNoise.FeedForward(NeuronOCL))
ReturnFalse;
if(!cAttention[0].FeedForward(cAddNoise.AsObject()))
ReturnFalse;
if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject()))
ReturnFalse;
if(!cNeck.FeedForward(cMergeSplit[0].AsObject()))
ReturnFalse;
if(!cAttention[1].FeedForward(cNeck))
ReturnFalse;
if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject()))
ReturnFalse;
if(!SumAndNormalize(cAddNoise.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, true, 0, 0, 0, 1))
ReturnFalse;
if(!cRevIn.FeedForward(cResidual.AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::calcInputGradients(CNeuronBaseOCL* prevLayer)
{
if(!prevLayer)
ReturnFalse;
//---
float error = 1;
if(!cRevIn.calcOutputGradients(prevLayer.getOutput(), error) ||
!SumAndNormalize(cRevIn.getGradient(), Gradient, cRevIn.getGradient(), 1, false, 0, 0, 0, 1))
ReturnFalse;
if(!cResidual.CalcHiddenGradients(cRevIn.AsObject()))
ReturnFalse;
if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject()))
ReturnFalse;
if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject()))
ReturnFalse;
if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject()))
ReturnFalse;
if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject()))
ReturnFalse;
if(!cAddNoise.CalcHiddenGradients(cAttention[0].AsObject()))
ReturnFalse;
if(!SumAndNormalize(cAddNoise.getGradient(), cResidual.getGradient(), cAddNoise.getGradient(), 1, false, 0, 0, 0, 1))
ReturnFalse;
if(!prevLayer.CalcHiddenGradients(cAddNoise.AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
{
if(!cAddNoise.UpdateInputWeights(NeuronOCL))
ReturnFalse;
if(!cAttention[0].UpdateInputWeights(cAddNoise.AsObject()))
ReturnFalse;
if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject()))
ReturnFalse;
if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject()))
ReturnFalse;
if(!cAttention[1].UpdateInputWeights(cNeck))
ReturnFalse;
if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronDiffusion::SetOpenCL(COpenCLMy * obj)
{
CNeuronUShapeAttention::SetOpenCL(obj);
cAddNoise.SetOpenCL(OpenCL);
cResidual.SetOpenCL(OpenCL);
cRevIn.SetOpenCL(OpenCL);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Save(const int file_handle)
{
if(!CNeuronUShapeAttention::Save(file_handle))
ReturnFalse;
if(!cAddNoise.Save(file_handle))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
for(int i = 0; i < 2; i++)
{
if(!LoadInsideLayer(file_handle, cAttention[i].AsObject()))
ReturnFalse;
if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject()))
ReturnFalse;
}
//---
int type = FileReadInteger(file_handle);
if(!!cNeck)
{
if(cNeck.Type() != type)
DeleteObj(cNeck);
}
//---
if(!cNeck)
{
switch(type)
{
case defNeuronUShapeAttention:
cNeck = new CNeuronUShapeAttention();
if(!cNeck)
ReturnFalse;
break;
case defNeuronDiffusion:
cNeck = new CNeuronDiffusion();
if(!cNeck ||
!((CNeuronDiffusion*)cNeck).Init(0, 0, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1))
ReturnFalse;
break;
case defNeuronConvOCL:
cNeck = new CNeuronConvOCL();
if(!cNeck)
ReturnFalse;
break;
default:
ReturnFalse;
}
}
cNeck.SetOpenCL(OpenCL);
if(!cNeck.Load(file_handle))
ReturnFalse;
//---
if(!cAddNoise.Init(0, 0, OpenCL, Neurons(), iBatch, optimization) ||
!LoadInsideLayer(file_handle, cAddNoise.AsObject()))
ReturnFalse;
//---
if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch))
ReturnFalse;
if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true))
ReturnFalse;
if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject()))
ReturnFalse;
//---
if(!SetOutput(cRevIn.getOutput(), true))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + 1)*units_count, optimization_type, batch))
ReturnFalse;
iWindow = window;
iUnits = units_count;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::feedForward(CNeuronBaseOCL* NeuronOCL)
{
if(!NeuronOCL)
ReturnFalse;
//---
uint global_work_offset[2] = {0, 0};
uint global_work_size[2] = {iUnits, iWindow};
uint local_work_size[2] = {1, iWindow};
int kernel = def_k_HyperProjection;
ResetLastError();
setBuffer(kernel, def_k_lp_inputs, NeuronOCL.getOutputIndex())
setBuffer(kernel, def_k_lp_outputs, getOutputIndex())
//---
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef _DEBUG
if(!NeuronOCL.getOutput().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::calcInputGradients(CNeuronBaseOCL* prevLayer)
{
if(!prevLayer)
ReturnFalse;
//---
uint global_work_offset[2] = {0, 0};
uint global_work_size[2] = {iUnits, iWindow};
int kernel = def_k_HyperProjectionGrad;
ResetLastError();
setBuffer(kernel, def_k_lpg_inputs, prevLayer.getOutputIndex())
setBuffer(kernel, def_k_lpg_inputs_gr, prevLayer.getGradientIndex())
setBuffer(kernel, def_k_lpg_outputs_gr, getGradientIndex())
//---
kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!prevLayer.getGradient().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
if(FileIsEnding(file_handle))
ReturnFalse;
iWindow = (int)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
iUnits = (int)FileReadInteger(file_handle);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * centroids, optimization_type, batch))
ReturnFalse;
//---
iWindows = window;
iUnits = units_count;
iCentroids = centroids;
//---
cHyperCentroids.Clear();
cHyperCurvatures.Clear();
cHyperCentroids.SetOpenCL(OpenCL);
cHyperCurvatures.SetOpenCL(OpenCL);
//---
CNeuronTransposeOCL *transp = new CNeuronTransposeOCL();
if(!transp ||
!transp.Init(0, 0, OpenCL, iUnits, iWindows, optimization, iBatch) ||
!cHyperCentroids.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
transp.SetActivationFunction(None);
//---
CNeuronConvOCL *conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, 1, OpenCL, iUnits, iUnits, iCentroids, iWindows, 1, optimization, iBatch) ||
!cHyperCentroids.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(TANH);
//---
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, 2, OpenCL, iCentroids, iCentroids, iCentroids, 1, iWindows, optimization, iBatch) ||
!cHyperCentroids.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(None);
//---
transp = new CNeuronTransposeOCL();
if(!transp ||
!transp.Init(0, 3, OpenCL, iWindows, iCentroids, optimization, iBatch) ||
!cHyperCentroids.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
//---
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, 4, OpenCL, iWindows, iWindows, iWindows, iCentroids, 1, optimization, iBatch) ||
!cHyperCurvatures.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(TANH);
//---
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, 5, OpenCL, iWindows, iWindows, 1, 1, iCentroids, optimization, iBatch) ||
!cHyperCurvatures.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(None);
//---
uint size = iCentroids * iUnits * sizeof(float);
iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
if(iProducts < 0)
ReturnFalse;
iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
if(iDistances < 0)
ReturnFalse;
iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
if(iNormes < 0)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::LogMap(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs)
{
if(!featers || !centroids || !curvatures || !outputs)
ReturnFalse;
//---
uint global_work_offset[3] = {0, 0, 0};
uint global_work_size[3] = {iUnits, iCentroids, iWindows};
uint local_work_size[3] = {1, 1, iWindows};
int kernel = def_k_LogMap;
ResetLastError();
setBuffer(kernel, def_k_logmap_centroids, centroids.getOutputIndex())
setBuffer(kernel, def_k_logmap_curvatures, curvatures.getOutputIndex())
setBuffer(kernel, def_k_logmap_features, featers.getOutputIndex())
setBuffer(kernel, def_k_logmap_outputs, outputs.getOutputIndex())
setBuffer(kernel, def_k_logmap_distance, iDistances)
setBuffer(kernel, def_k_logmap_product, iProducts)
setBuffer(kernel, def_k_logmap_norma, iNormes)
//---
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef _DEBUG
if(!outputs.getOutput().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::LogMapGrad(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs)
{
if(!featers || !centroids || !curvatures || !outputs)
ReturnFalse;
//---
//---
CBufferFloat *temp = featers.getGradient();
if(!temp ||
!temp.Fill(0) ||
!temp.BufferWrite())
ReturnFalse;
temp = centroids.getGradient();
if(!temp ||
!temp.Fill(0) ||
!temp.BufferWrite())
ReturnFalse;
temp = curvatures.getGradient();
if(!temp ||
!temp.Fill(0) ||
!temp.BufferWrite())
ReturnFalse;
//---
uint global_work_offset[3] = {0, 0, 0};
uint global_work_size[3] = {iUnits, iCentroids, iWindows};
uint local_work_size[3] = {1, 1, iWindows};
int kernel = def_k_LogMapGrad;
ResetLastError();
setBuffer(kernel, def_k_logmapgr_centroids, centroids.getOutputIndex())
setBuffer(kernel, def_k_logmapgr_centroids_gr, centroids.getGradientIndex())
setBuffer(kernel, def_k_logmapgr_curvatures, curvatures.getOutputIndex())
setBuffer(kernel, def_k_logmapgr_curvatures_gr, curvatures.getGradientIndex())
setBuffer(kernel, def_k_logmapgr_features, featers.getOutputIndex())
setBuffer(kernel, def_k_logmapgr_features_gr, featers.getGradientIndex())
setBuffer(kernel, def_k_logmapgr_outputs, outputs.getOutputIndex())
setBuffer(kernel, def_k_logmapgr_outputs_gr, outputs.getGradientIndex())
setBuffer(kernel, def_k_logmapgr_distance, iDistances)
setBuffer(kernel, def_k_logmapgr_product, iProducts)
setBuffer(kernel, def_k_logmapgr_norma, iNormes)
//---
kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef _DEBUG
if(!featers.getGradient().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::feedForward(CNeuronBaseOCL* NeuronOCL)
{
CNeuronBaseOCL *prev = NeuronOCL;
CNeuronBaseOCL *centroids = NULL;
CNeuronBaseOCL *curvatures = NULL;
//--- Centroids
for(int i = 0; i < cHyperCentroids.Total(); i++)
{
centroids = cHyperCentroids[i];
if(!centroids ||
!centroids.FeedForward(prev))
ReturnFalse;
prev = centroids;
}
//--- Curvatures
for(int i = 0; i < cHyperCurvatures.Total(); i++)
{
curvatures = cHyperCurvatures[i];
if(!curvatures ||
!curvatures.FeedForward(prev))
ReturnFalse;
prev = curvatures;
}
if(!LogMap(NeuronOCL, centroids, curvatures, AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::calcInputGradients(CNeuronBaseOCL* prevLayer)
{
if(!prevLayer)
ReturnFalse;
//---
CObject *next = NULL;
CNeuronBaseOCL *centroids = cHyperCentroids[-1];
CNeuronBaseOCL *curvatures = cHyperCurvatures[-1];
//---
if(!LogMapGrad(prevLayer, centroids, curvatures, AsObject()))
ReturnFalse;
//--- Curvatures
for(int i = cHyperCurvatures.Total() - 2; i >= 0; i--)
{
next = curvatures;
curvatures = cHyperCurvatures[i];
if(!curvatures ||
!curvatures.CalcHiddenGradients(next))
ReturnFalse;
}
CBufferFloat *temp = centroids.getGradient();
if(centroids.Activation() != None)
if(!DeActivation(centroids.getOutput(), temp, temp, centroids.Activation()))
ReturnFalse;
if(!centroids.SetGradient(centroids.getPrevOutput(), false) ||
!centroids.CalcHiddenGradients(curvatures.AsObject()) ||
!SumAndNormalize(temp, centroids.getGradient(), temp, iWindows, false, 0, 0, 0, 1) ||
!centroids.SetGradient(temp, false)
)
ReturnFalse;
//--- Centroids
for(int i = cHyperCentroids.Total() - 2; i >= 0; i--)
{
next = centroids;
centroids = cHyperCentroids[i];
if(!centroids ||
!centroids.CalcHiddenGradients(next))
ReturnFalse;
}
//---
temp = prevLayer.getGradient();
if(prevLayer.Activation() != None)
if(!DeActivation(prevLayer.getOutput(), temp, temp, prevLayer.Activation()))
ReturnFalse;
if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
!prevLayer.CalcHiddenGradients(centroids.AsObject()) ||
!SumAndNormalize(temp, prevLayer.getGradient(), temp, iWindows, false, 0, 0, 0, 1) ||
!prevLayer.SetGradient(temp, false)
)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
{
CNeuronBaseOCL *prev = NeuronOCL;
CNeuronBaseOCL *centroids = NULL;
CNeuronBaseOCL *curvatures = NULL;
//--- Centroids
for(int i = 0; i < cHyperCentroids.Total(); i++)
{
centroids = cHyperCentroids[i];
if(!centroids ||
!centroids.UpdateInputWeights(prev))
ReturnFalse;
prev = centroids;
}
//--- Curvatures
for(int i = 0; i < cHyperCurvatures.Total(); i++)
{
curvatures = cHyperCurvatures[i];
if(!curvatures ||
!curvatures.UpdateInputWeights(prev))
ReturnFalse;
prev = curvatures;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronHyperboloids::SetOpenCL(COpenCLMy * obj)
{
if(!!OpenCL)
{
OpenCL.BufferFree(iProducts);
OpenCL.BufferFree(iDistances);
OpenCL.BufferFree(iNormes);
}
//---
CNeuronBaseOCL::SetOpenCL(obj);
cHyperCentroids.SetOpenCL(OpenCL);
cHyperCurvatures.SetOpenCL(OpenCL);
//---
uint size = iCentroids * iUnits * sizeof(float);
iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
if(!cHyperCentroids.Save(file_handle))
ReturnFalse;
if(!cHyperCurvatures.Save(file_handle))
ReturnFalse;
if(FileWriteInteger(file_handle, int(iWindows)) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, int(iCentroids)) < INT_VALUE)
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
if(!cHyperCentroids.Load(file_handle))
ReturnFalse;
if(!cHyperCurvatures.Load(file_handle))
ReturnFalse;
if(FileIsEnding(file_handle))
ReturnFalse;
iWindows = (int)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
iUnits = (int)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
iCentroids = (int)FileReadInteger(file_handle);
SetOpenCL(OpenCL);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronHyperboloids::TrainMode(bool flag)
{
CNeuronBaseOCL::TrainMode(flag);
CNeuronBaseOCL *neuron = NULL;
//---
for(int i = 0; i < cHyperCentroids.Total(); i++)
{
neuron = cHyperCentroids[i];
if(!neuron)
continue;
neuron.TrainMode(bTrain);
}
//---
for(int i = 0; i < cHyperCurvatures.Total(); i++)
{
neuron = cHyperCurvatures[i];
if(!neuron)
continue;
neuron.TrainMode(bTrain);
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronHypDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch))
ReturnFalse;
//---
cLayers.Clear();
cLayers.SetOpenCL(OpenCL);
int layer = 0;
//--- Projection
CNeuronHyperProjection *lorenz = new CNeuronHyperProjection();
if(!lorenz ||
!lorenz.Init(0, layer, OpenCL, window, units_count, optimization, iBatch) ||
!cLayers.Add(lorenz))
{
DeleteObj(lorenz);
ReturnFalse;
}
layer++;
//--- Encoder
CNeuronRMAT *rmat = new CNeuronRMAT();
if(!rmat ||
!rmat.Init(0, layer, OpenCL, window + 1, window_key, units_count, heads, layers, optimization, iBatch) ||
!cLayers.Add(rmat))
{
DeleteObj(rmat);
ReturnFalse;
}
layer++;
//---
CNeuronConvOCL *conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, layer, OpenCL, window + 1, window + 1, 2 * window, units_count, 1, optimization, iBatch) ||
!cLayers.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
layer++;
conv.SetActivationFunction(TANH);
//---
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, layer, OpenCL, 2 * window, 2 * window, 3, units_count, 1, optimization, iBatch) ||
!cLayers.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
layer++;
//--- LogMap projecction
CNeuronHyperboloids *logmap = new CNeuronHyperboloids();
if(!logmap ||
!logmap.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) ||
!cLayers.Add(logmap))
{
DeleteObj(logmap);
ReturnFalse;
}
layer++;
//--- Diffusion model
CNeuronDiffusion *diff = new CNeuronDiffusion();
if(!diff ||
!diff.Init(0, layer, OpenCL, 3, window_key, heads, units_count * centroids, 2, layers, optimization, iBatch) ||
!cLayers.Add(diff))
{
DeleteObj(diff);
ReturnFalse;
}
layer++;
//--- Pooling
CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling();
if(!pooling ||
!pooling.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) ||
!cLayers.Add(pooling))
{
DeleteObj(pooling);
ReturnFalse;
}
layer++;
//--- Resize to source size
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, layer, OpenCL, 3, 3, window, units_count, 1, optimization, iBatch) ||
!cLayers.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
//---
if(!SetOutput(conv.getOutput(), true) ||
!SetGradient(conv.getGradient(), true))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint segments, float rho, ENUM_OPTIMIZATION optimization_type, uint batch)
{
if((window * units_count) % segments > 0)
ReturnFalse;
if(!CNeuronBaseSAMOCL::Init(numOutputs, myIndex, open_cl, window * units_count, rho, optimization_type, batch))
ReturnFalse;
//---
uint count = Neurons() / segments;
//---
if(!acTranspose[0].Init(0, 0, OpenCL, segments, count, optimization, iBatch))
ReturnFalse;
acTranspose[0].SetActivationFunction(None);
//---
if(!acPSBlocks[0].Init(0, 1, OpenCL, segments, segments, units_count / segments, 1, fRho, optimization, iBatch))
ReturnFalse;
for(int i = 0; i < 2; i++)
{
if(!acAttention[i].Init(0, i + 2, OpenCL, segments, segments, units_count / segments, 2, optimization, iBatch))
ReturnFalse;
if(!acPSBlocks[i + 1].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject()))
ReturnFalse;
}
//---
if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch))
ReturnFalse;
if(!cResidual.SetGradient(acAttention[1].getGradient(), true))
ReturnFalse;
cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation());
//---
if(!acTranspose[1].Init(0, 5, OpenCL, count, segments, optimization, iBatch))
ReturnFalse;
acTranspose[1].SetActivationFunction((ENUM_ACTIVATION)acPSBlocks[2].Activation());
//---
if(!SetOutput(acTranspose[1].getOutput(), true) ||
!SetGradient(acTranspose[1].getGradient(), true))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::feedForward(CNeuronBaseOCL* NeuronOCL)
{
//--- Dimension Transformation
if(!acTranspose[0].FeedForward(NeuronOCL))
ReturnFalse;
//--- Segment Attention
CObject* prev = acTranspose[0].AsObject();
for(int i = 0; i < 2; i++)
{
if(!acPSBlocks[i].FeedForward(prev))
ReturnFalse;
if(!acAttention[i].FeedForward(acPSBlocks[i].AsObject()))
ReturnFalse;
prev = acAttention[i].AsObject();
}
//--- Residual Add
if(!SumAndNormalize(acTranspose[0].getOutput(), acAttention[1].getOutput(), cResidual.getOutput(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
ReturnFalse;
//--- PS Block
if(!acPSBlocks[2].FeedForward(cResidual.AsObject()))
ReturnFalse;
//--- Inverse Transformation
if(!acTranspose[1].FeedForward(acPSBlocks[2].AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::calcInputGradients(CNeuronBaseOCL* NeuronOCL)
{
if(!NeuronOCL)
ReturnFalse;
//---
if(!acPSBlocks[2].CalcHiddenGradients(acTranspose[1].AsObject()))
ReturnFalse;
//---
if(!cResidual.CalcHiddenGradients(acPSBlocks[2].AsObject()))
ReturnFalse;
//---
if(!acPSBlocks[1].CalcHiddenGradients(acAttention[1].AsObject()))
ReturnFalse;
if(!acAttention[0].CalcHiddenGradients(acPSBlocks[1].AsObject()))
ReturnFalse;
if(!acPSBlocks[0].CalcHiddenGradients(acAttention[0].AsObject()))
ReturnFalse;
//---
if(!acTranspose[0].CalcHiddenGradients(acPSBlocks[0].AsObject()))
ReturnFalse;
if(acTranspose[0].Activation() == None)
{
if(!SumAndNormalize(acTranspose[0].getGradient(), cResidual.getGradient(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
ReturnFalse;
}
else
{
if(!DeActivation(acTranspose[0].getOutput(), cResidual.getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].Activation()) ||
!SumAndNormalize(acTranspose[0].getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
ReturnFalse;
}
if(!NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject()))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
{
if(!acPSBlocks[2].UpdateInputWeights(cResidual.AsObject()))
ReturnFalse;
//---
CObject* prev = acAttention[0].AsObject();
for(int i = 1; i >= 0; i--)
{
if(!acAttention[i].UpdateInputWeights(acPSBlocks[i].AsObject()))
ReturnFalse;
if(!acPSBlocks[i].UpdateInputWeights(prev))
ReturnFalse;
prev = acTranspose[0].AsObject();
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronPSformer::SetOpenCL(COpenCLMy * obj)
{
CNeuronBaseSAMOCL::SetOpenCL(obj);
for(int i = 0; i < 2; i++)
{
acTranspose[i].SetOpenCL(OpenCL);
acPSBlocks[i].SetOpenCL(OpenCL);
acAttention[i].SetOpenCL(OpenCL);
}
acPSBlocks[2].SetOpenCL(OpenCL);
cResidual.SetOpenCL(OpenCL);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Save(const int file_handle)
{
if(!CNeuronBaseSAMOCL::Save(file_handle))
ReturnFalse;
if(!acPSBlocks[0].Save(file_handle))
ReturnFalse;
for(int i = 0; i < 2; i++)
if(!acTranspose[i].Save(file_handle) ||
!acAttention[i].Save(file_handle))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Load(const int file_handle)
{
if(!CNeuronBaseSAMOCL::Load(file_handle))
ReturnFalse;
//---
if(!LoadInsideLayer(file_handle, acPSBlocks[0].AsObject()))
ReturnFalse;
for(int i = 0; i < 2; i++)
if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject()) ||
!LoadInsideLayer(file_handle, acAttention[i].AsObject()))
ReturnFalse;
//---
for(int i = 1; i < 3; i++)
if(!acPSBlocks[i].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject()))
ReturnFalse;
//---
if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch))
ReturnFalse;
if(!cResidual.SetGradient(acAttention[1].getGradient(), true))
ReturnFalse;
cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation());
//---
if(!SetOutput(acTranspose[1].getOutput(), true) ||
!SetGradient(acTranspose[1].getGradient(), true))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronAdaBN::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl,
uint window, uint window_out, uint units_count,
uint & bottlenecks[], uint top_k, uint variables,
ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch))
ReturnFalse;
int index = 0;
if(!cGates.Init(0, index, OpenCL, window, units_count * variables, bottlenecks.Size(), top_k, optimization, iBatch))
ReturnFalse;
//---
index++;
if(!cXProj.Init(0, index, OpenCL, window, window, window, units_count, optimization, iBatch))
ReturnFalse;
cXProj.SetActivationFunction(None);
//---
cExperts.Clear();
cExperts.SetOpenCL(OpenCL);
CNeuronConvOCL *conv = NULL;
CNeuronMultiWindowsConvOCL *mwconv = NULL;
CNeuronTransposeRCDOCL *transp = NULL;
//---
uint bn_size = 0;
for(uint i = 0; i < bottlenecks.Size(); i++)
bn_size += bottlenecks[i];
//---
index++;
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, index, OpenCL, window, window, bn_size, units_count, variables, optimization, iBatch) ||
!cExperts.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(LReLU);
//---
index++;
mwconv = new CNeuronMultiWindowsConvOCL();
if(!mwconv ||
!mwconv.Init(0, index, OpenCL, bottlenecks, window_out, units_count, variables, optimization, iBatch) ||
!cExperts.Add(mwconv))
{
DeleteObj(conv);
ReturnFalse;
}
mwconv.SetActivationFunction(LReLU);
transp = new CNeuronTransposeRCDOCL();
index++;
if(!transp ||
!transp.Init(0, index, OpenCL, units_count * variables, bottlenecks.Size(), window_out, optimization, iBatch) ||
!cExperts.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
index++;
conv = new CNeuronConvOCL();
if(!conv ||
!conv.Init(0, index, OpenCL, window_out, window_out, window, units_count * variables, bottlenecks.Size(), optimization, iBatch) ||
!cExperts.Add(conv))
{
DeleteObj(conv);
ReturnFalse;
}
conv.SetActivationFunction(None);
transp = new CNeuronTransposeRCDOCL();
index++;
if(!transp ||
!transp.Init(0, index, OpenCL, bottlenecks.Size(), units_count * variables, window, optimization, iBatch) ||
!cExperts.Add(transp))
{
DeleteObj(transp);
ReturnFalse;
}
transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
uint units, uint window,
uint embed_dim1, uint period1, uint frame1,
uint embed_dim2, uint period2, uint frame2,
ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
(window + embed_dim1 + embed_dim2)*units, optimization_type, batch))
ReturnFalse;
//---
if(!caEmbeddings[0].Init(0, 0, OpenCL, embed_dim1 * period1, optimization, iBatch))
ReturnFalse;
if(!caEmbeddings[1].Init(0, 1, OpenCL, embed_dim2 * period2, optimization, iBatch))
ReturnFalse;
//---
iWindow = window;
iUnits = units;
aiEmbeddingDim[0] = embed_dim1;
aiEmbeddingDim[1] = embed_dim2;
aiFrames[0] = MathMax(1, frame1);
aiFrames[1] = MathMax(1, frame2);
aiPeriod[0] = period1;
aiPeriod[1] = period2;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
{
if(!OpenCL || !NeuronOCL || !SecondInput)
ReturnFalse;
if(NeuronOCL.Neurons() < int(iUnits * iWindow) ||
SecondInput.Total() < (int)iUnits)
ReturnFalse;
//---
uint global_work_offset[3] = {0, 0, 0};
uint global_work_size[3] = {iUnits,
MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]),
3
};
uint kernel = def_k_ConcatByLabel;
setBuffer(kernel, def_k_cbl_data, NeuronOCL.getOutputIndex())
setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex())
setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getOutputIndex())
setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getOutputIndex())
setBuffer(kernel, def_k_cbl_output, getOutputIndex())
setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow)
setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0])
setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1])
setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0])
setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1])
setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0])
setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1])
//---
kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!Output.BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
{
if(!OpenCL || !NeuronOCL || !SecondInput)
ReturnFalse;
if(NeuronOCL.Neurons() < int(iUnits * iWindow) ||
SecondInput.Total() < (int)iUnits)
ReturnFalse;
//---
uint global_work_offset[3] = {0, 0, 0};
uint global_work_size[3] = {MathMax(iUnits, aiPeriod[ArrayMaximum(aiPeriod)]),
MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]),
3
};
uint kernel = def_k_ConcatByLabelGrad;
setBuffer(kernel, def_k_cbl_data, NeuronOCL.getGradientIndex())
setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex())
setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getGradientIndex())
setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getGradientIndex())
setBuffer(kernel, def_k_cbl_output, getGradientIndex())
setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow)
setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0])
setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1])
setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0])
setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1])
setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0])
setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1])
setArgument(kernel, def_k_cbl_gr_units, (int)iUnits)
//---
kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef _DEBUG
if(!NeuronOCL.getGradient().BufferRead())
ReturnFalse;
#endif
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
{
if(bTrain)
for(uint i = 0; i < caEmbeddings.Size(); i++)
if(!caEmbeddings[i].FeedForward())
ReturnFalse;
//---
return ConcatByLabel(NeuronOCL, SecondInput);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
CBufferFloat *SecondInput,
CBufferFloat *SecondGradient,
ENUM_ACTIVATION SecondActivation = None)
{
if(!ConcatByLabelGrad(NeuronOCL, SecondInput))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
{
for(uint i = 0; i < caEmbeddings.Size(); i++)
if(!caEmbeddings[i].UpdateInputWeights())
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau)
{
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
ReturnFalse;
//---
CNeuronTempEmbedding* Source = source;
for(uint i = 0; i < caEmbeddings.Size(); i++)
if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
//---
for(uint i = 0; i < caEmbeddings.Size(); i++)
if(!caEmbeddings[i].Save(file_handle))
ReturnFalse;
//---
if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE)
ReturnFalse;
for(uint i = 0; i < caEmbeddings.Size(); i++)
{
if(FileWriteInteger(file_handle, (int)aiEmbeddingDim[i]) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, (int)aiFrames[i]) < INT_VALUE)
ReturnFalse;
if(FileWriteInteger(file_handle, (int)aiPeriod[i]) < INT_VALUE)
ReturnFalse;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
//---
for(uint i = 0; i < caEmbeddings.Size(); i++)
if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject()))
ReturnFalse;
//---
if(FileIsEnding(file_handle))
ReturnFalse;
iWindow = (uint)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
iUnits = (uint)FileReadInteger(file_handle);
for(uint i = 0; i < caEmbeddings.Size(); i++)
{
if(FileIsEnding(file_handle))
ReturnFalse;
aiEmbeddingDim[i] = (uint)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
aiFrames[i] = (uint)FileReadInteger(file_handle);
if(FileIsEnding(file_handle))
ReturnFalse;
aiPeriod[i] = (uint)FileReadInteger(file_handle);
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronTempEmbedding::SetOpenCL(COpenCLMy *obj)
{
CNeuronBaseOCL::SetOpenCL(obj);
//---
for(uint i = 0; i < caEmbeddings.Size(); i++)
caEmbeddings[i].SetOpenCL(OpenCL);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
uint dimension, uint units, uint scales, uint bottleneck,
ENUM_OPTIMIZATION optimization_type, uint batch)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units * scales, optimization_type, batch))
ReturnFalse;
//---
cExcitation.Clear();
cExcitation.SetOpenCL(OpenCL);
CNeuronSpikeConvBlock* conv = NULL;
CNeuronTransposeOCL* transp = NULL;
CNeuronSoftMaxOCL* softmax = NULL;
uint index = 0;
//--- [Scale, Unit, Dimension]
transp = new CNeuronTransposeOCL();
if(!transp ||
!transp.Init(0, index, OpenCL, scales, units * dimension, optimization, iBatch) ||
!cExcitation.Add(transp))
{
DeleteObj(transp)
ReturnFalse;
}
index++;
//--- [Unit, Dimension, Scale]
conv = new CNeuronSpikeConvBlock();
if(!conv ||
!conv.Init(0, index, OpenCL, scales, scales, 1, dimension * units, 1, optimization, iBatch) ||
!cExcitation.Add(conv))
{
DeleteObj(conv)
ReturnFalse;
}
index++;
//--- [Unit, Dimension]
conv = new CNeuronSpikeConvBlock();
if(!conv ||
!conv.Init(0, index, OpenCL, dimension, dimension, bottleneck, units, 1, optimization, iBatch) ||
!cExcitation.Add(conv))
{
DeleteObj(conv)
ReturnFalse;
}
index++;
//--- [Unit, Bottleneck]
conv = new CNeuronSpikeConvBlock();
if(!conv ||
!conv.Init(0, index, OpenCL, bottleneck, bottleneck, scales * dimension, units, 1, optimization, iBatch) ||
!cExcitation.Add(conv))
{
DeleteObj(conv)
ReturnFalse;
}
index++;
//--- [Unit, Dimension, Scale]
softmax = new CNeuronSoftMaxOCL();
if(!softmax ||
!softmax.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) ||
!cExcitation.Add(softmax))
{
DeleteObj(softmax)
ReturnFalse;
}
softmax.SetHeads(dimension * units);
index++;
//--- [Unit, Dimension, Scale]
transp = new CNeuronTransposeOCL();
if(!transp ||
!transp.Init(0, index, OpenCL, units * dimension, scales, optimization, iBatch) ||
!cExcitation.Add(transp))
{
DeleteObj(transp)
ReturnFalse;
}
index++;
//--- [Scale, Unit, Dimension]
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::feedForward(CNeuronBaseOCL *NeuronOCL)
{
CNeuronBaseOCL* prev = NeuronOCL;
CNeuronBaseOCL* curr = NULL;
for(int i = 0; i < cExcitation.Total(); i++)
{
curr = cExcitation[i];
if(!curr ||
!curr.FeedForward(prev))
ReturnFalse;
prev = curr;
}
if(!ElementMult(NeuronOCL.getOutput(), prev.getOutput(), Output))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
{
CNeuronBaseOCL* next = cExcitation[-1];
CNeuronBaseOCL* curr = NULL;
//---
if(!NeuronOCL || !next)
ReturnFalse;
if(!ElementMultGrad(NeuronOCL.getOutput(), PrevOutput,
next.getOutput(), next.getGradient(),
Gradient, NeuronOCL.Activation(), next.Activation()))
ReturnFalse;
//---
for(int i = cExcitation.Total() - 2; i >= 0; i--)
{
curr = cExcitation[i];
if(!curr ||
!curr.CalcHiddenGradients(next))
ReturnFalse;
next = curr;
}
//---
if(!NeuronOCL.CalcHiddenGradients(next) ||
!SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
{
CNeuronBaseOCL* prev = NeuronOCL;
CNeuronBaseOCL* curr = NULL;
for(int i = 0; i < cExcitation.Total(); i++)
{
curr = cExcitation[i];
if(!curr ||
!curr.UpdateInputWeights(prev))
ReturnFalse;
prev = curr;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::WeightsUpdate(CNeuronBaseOCL *source, float tau)
{
if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
ReturnFalse;
//---
CNeuronMultiScaleExcitation* Source = source;
if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
ReturnFalse;
if(!cExcitation.Save(file_handle))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
ReturnFalse;
if(!cExcitation.Load(file_handle))
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Clear(void)
{
if(!CNeuronBaseOCL::Clear())
ReturnFalse;
if(!cExcitation.ClearStates())
ReturnFalse;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronMultiScaleExcitation::SetOpenCL(COpenCLMy *obj)
{
CNeuronBaseOCL::SetOpenCL(obj);
cExcitation.SetOpenCL(OpenCL);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNeuronMultiScaleExcitation::TrainMode(bool flag)
{
CNeuronBaseOCL::TrainMode(flag);
cExcitation.TrainMode(bTrain);
}
Reference in a new issue View git blame Copy permalink