#ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE //+------------------------------------------------------------------+ //| Logical method implementations generated from NeuroNet.mqh: RecurrentState //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::CNeuronLSTM(void) { ForgetGate = new CLayer(); InputGate = new CLayer(); OutputGate = new CLayer(); NewContent = new CLayer(); Memory = new CArrayFloat(); PrevMemory = new CArrayFloat(); Input = new CArrayFloat(); InputGradient = new CArrayFloat(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::~CNeuronLSTM(void) { DeleteObj(ForgetGate); DeleteObj(InputGate); DeleteObj(OutputGate); DeleteObj(NewContent); DeleteObj(Memory); DeleteObj(PrevMemory); DeleteObj(Input); DeleteObj(InputGradient); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { if(units_count <= 0) ReturnFalse; //--- Init Layers if(!CNeuronProof::Init(numOutputs, myIndex, window, step, units_count, optimization_type)) ReturnFalse; if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(InputGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(NewContent, units_count, window + units_count, optimization_type)) ReturnFalse; if(!Memory.Reserve(units_count)) ReturnFalse; if(!PrevMemory.Reserve(units_count)) ReturnFalse; CNeuron *temp; for(int i = 0; i < units_count; i++) { if(!Memory.Add(0)) ReturnFalse; if(!PrevMemory.Add(0)) ReturnFalse; temp = OutputLayer.At(i); temp.setOutputVal(0); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(layer) == POINTER_INVALID) { layer = new CLayer(numOutputs); if(CheckPointer(layer) == POINTER_INVALID) ReturnFalse; } else layer.Clear(); if(!layer.Reserve(numUnits)) ReturnFalse; //--- CNeuron *temp; for(int i = 0; i < numUnits; i++) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Init(numOutputs + 1, i, optimization_type)) ReturnFalse; if(!layer.Add(temp)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0) ReturnFalse; CNeuronBase *temp; CConnection *temp_con; if(CheckPointer(Input) == POINTER_INVALID) { Input = new CArrayFloat(); if(CheckPointer(Input) == POINTER_INVALID) ReturnFalse; } else Input.Clear(); //--- Concatenate input sequence int total = prevLayer.Total(); if(!Input.Reserve(total + OutputLayer.Total())) ReturnFalse; for(int i = 0; i < total; i++) { temp = prevLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) ReturnFalse; } total = OutputLayer.Total(); for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) ReturnFalse; } int total_data = Input.Total(); //--- Calculated forget gate CArrayFloat *forget_gate = CalculateGate(ForgetGate, Input); if(CheckPointer(forget_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated input gate CArrayFloat *input_gate = CalculateGate(InputGate, Input); if(CheckPointer(input_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated output gate CArrayFloat *output_gate = CalculateGate(OutputGate, Input); if(CheckPointer(output_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated new content CArrayFloat *new_content = new CArrayFloat(); if(CheckPointer(new_content) == POINTER_INVALID) ReturnFalse; total = NewContent.Total(); for(int i = 0; i < total; i++) { temp = NewContent.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; float val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) ReturnFalse; val += temp_con.weight * Input.At(c); } val = TanhFunction(val); temp.setOutputVal(val); if(!new_content.Add(val)) ReturnFalse; } //--- Calculated output sequences for(int i = 0; i < total; i++) { if(PrevMemory.Total() <= i) PrevMemory.Add(Memory.At(i)); else PrevMemory.Update(i, Memory.At(i)); float value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i); if(!Memory.Update(i, value)) ReturnFalse; temp = OutputLayer.At(i); value = TanhFunction(value) * output_gate.At(i); temp.setOutputVal(value); } //--- DeleteObj(forget_gate); DeleteObj(input_gate); DeleteObj(new_content); DeleteObj(output_gate); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CArrayFloat *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayFloat *sequence) { CNeuronBase *temp; CConnection *temp_con; CArrayFloat *result = new CArrayFloat(); if(CheckPointer(gate) == POINTER_INVALID) return NULL; int total = gate.Total(); int total_data = sequence.Total(); for(int i = 0; i < total; i++) { temp = gate.At(i); if(CheckPointer(temp) == POINTER_INVALID) { DeleteObj(result); return NULL; } float val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) { DeleteObj(result); return NULL; } val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c)); } val = SigmoidFunction(val); temp.setOutputVal(val); if(!result.Add(val)) { DeleteObj(result); return NULL; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(InputGradient) == POINTER_INVALID) { InputGradient = new CArrayFloat(); if(CheckPointer(InputGradient) == POINTER_INVALID) ReturnFalse; } else InputGradient.Clear(); //--- int total = OutputLayer.Total(); CNeuron *temp; CArrayFloat *MemoryGradient = new CArrayFloat(); CNeuron *gate; CConnection *con; //--- if(nextLayer != OutputLayer) for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(temp.sumDOW(nextLayer)); } //--- Calculated memory and output gate gradients if(CheckPointer(MemoryGradient) == POINTER_INVALID) ReturnFalse; if(!MemoryGradient.Reserve(total)) ReturnFalse; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; float value = temp.getGradient() * gate.getOutputVal(); value = TanhFunctionDerivative(Memory.At(i)) * value; if(i >= MemoryGradient.Total()) { if(!MemoryGradient.Add(value)) ReturnFalse; } else { value = MemoryGradient.At(i) + value; if(!MemoryGradient.Update(i, value)) ReturnFalse; } gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0); temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0); } //--- Calculated input gradients int total_inp = temp.getConnections().Total(); for(int n = 0; n < total_inp; n++) { float value = 0; for(int i = 0; i < total; i++) { temp = ForgetGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = InputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = OutputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = NewContent.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; } if(InputGradient.Total() >= n) { if(!InputGradient.Add(value)) ReturnFalse; } else if(!InputGradient.Update(n, value)) ReturnFalse; } //--- Calculated gradients for prev. state int shift = total_inp - total; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(InputGradient.At(shift + i)); } //--- Calculated memory and output gate gradients for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; float value = temp.getGradient() * gate.getPrevVal(); value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value; if(!MemoryGradient.Update(i, value)) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0)); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0)); } //--- DeleteObj(MemoryGradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID) ReturnFalse; //--- if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input) || !updateInputWeights(NewContent, Input)) { ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayFloat *input_data) { if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID) ReturnFalse; CNeuronBase *neuron; CConnection *con; int total_n = gate.Total(); int total_data = input_data.Total(); float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); for(int n = 0; n < total_n; n++) { neuron = gate.At(n); if(CheckPointer(neuron) == POINTER_INVALID) ReturnFalse; for(int i = 0; i < total_data; i++) { con = neuron.getConnections().At(i); if(CheckPointer(con) == POINTER_INVALID) ReturnFalse; float data = input_data.At(i); float g = neuron.getGradient(); if(optimization == SGD) con.weight += con.deltaWeight = (g != 0 && data != 0 ? lr * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight; else { con.mt = b1 * con.mt + (1 - b1) * g; con.vt = (float)(b2 * con.vt + (1 - b2) * pow(g, 2) + 0.00000001); con.weight += con.deltaWeight = (float)(lt * con.mt / sqrt(con.vt)); t++; } } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index) ReturnFalse; //--- prevNeuron.setGradient(InputGradient.At(index)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- int total = prevLayer.Total(); if(total <= 0) ReturnFalse; CNeuronBase *neuron; bool result = true; for(int i = 0; (i < total && result); i++) { neuron = prevLayer.At(i); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } result = calcInputGradients(neuron, i); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Save(const int file_handle) { if(!CNeuronProof::Save(file_handle)) ReturnFalse; if(!ForgetGate.Save(file_handle)) ReturnFalse; if(!InputGate.Save(file_handle)) ReturnFalse; if(!OutputGate.Save(file_handle)) ReturnFalse; if(!NewContent.Save(file_handle)) ReturnFalse; if(!Memory.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Load(const int file_handle) { if(!CNeuronProof::Load(file_handle)) ReturnFalse; if(!ForgetGate.Load(file_handle)) ReturnFalse; if(!InputGate.Load(file_handle)) ReturnFalse; if(!OutputGate.Load(file_handle)) ReturnFalse; if(!NewContent.Load(file_handle)) ReturnFalse; if(!Memory.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::CNeuronLSTMOCL(void) : m_iMemory(-1), m_iConcatenated(-1), m_iConcatenatedGradient(-1), m_iHiddenState(-1), m_iWeightsGradient(-1), m_iInputs(-1), m_iVariables(1) {} //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::~CNeuronLSTMOCL(void) { if(!OpenCL) return; OpenCL.BufferFree(m_iConcatenated); OpenCL.BufferFree(m_iConcatenatedGradient); OpenCL.BufferFree(m_iHiddenState); OpenCL.BufferFree(m_iMemory); OpenCL.BufferFree(m_iWeightsGradient); m_cFirstMomentumLSTM.BufferFree(); m_cSecondMomentumLSTM.BufferFree(); m_cWeightsLSTM.BufferFree(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return Init(numOutputs, myIndex, open_cl, numNeurons, 1, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons * variables, optimization_type, batch)) ReturnFalse; //--- m_iVariables = int(variables); m_iMemory = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 2, CL_MEM_READ_WRITE); if(m_iMemory < 0) ReturnFalse; m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables), CL_MEM_READ_WRITE); if(m_iHiddenState < 0) ReturnFalse; m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 4, CL_MEM_READ_WRITE); if(m_iConcatenated < 0) ReturnFalse; m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 4, CL_MEM_READ_WRITE); if(m_iConcatenatedGradient < 0) ReturnFalse; if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() <= 0 || NeuronOCL.getOutputIndex() < 0 || !OpenCL) ReturnFalse; //--- if(m_iInputs != NeuronOCL.Neurons() / m_iVariables) { if(!SetInputs(NeuronOCL.Neurons() / m_iVariables)) ReturnFalse; } //--- if(m_iMemory < 0 || m_iConcatenated < 0) ReturnFalse; //--- setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_concatenated, m_iConcatenated) setArgument(def_k_LSTM_FeedForward, def_k_lstmff_inputs_size, m_iInputs) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_memory, m_iMemory) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_outputs, getOutputIndex()) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_weights, m_cWeightsLSTM.GetIndex()) uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {Neurons() / m_iVariables, 4, m_iVariables}; uint local_work_size[] = {1, 4, 1}; kernelExecuteLoc(def_k_LSTM_FeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::SetInputs(int count) { m_iInputs = count; count = (int)((m_iInputs + Neurons() / m_iVariables + 1) * (Neurons() / m_iVariables) * 4); if(!m_cWeightsLSTM.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(Neurons() + 1)); for(int i = 0; i < count; i++) { if(!m_cWeightsLSTM.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!m_cWeightsLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(!m_cFirstMomentumLSTM.BufferInit(count, 0)) ReturnFalse; if(!m_cFirstMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(!m_cSecondMomentumLSTM.BufferInit(count, 0)) ReturnFalse; if(!m_cSecondMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; if(m_iWeightsGradient >= 0) OpenCL.BufferFree(m_iWeightsGradient); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * count, CL_MEM_READ_WRITE); if(m_iWeightsGradient < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() <= 0 || NeuronOCL.getGradientIndex() < 0 || NeuronOCL.getOutputIndex() < 0 || !OpenCL) ReturnFalse; //--- if(m_cWeightsLSTM.GetIndex() < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; if(m_iInputs < 0 || m_iConcatenated < 0 || m_iMemory < 0 || m_iConcatenatedGradient < 0 || m_iHiddenState < 0 || m_iInputs != NeuronOCL.Neurons() / m_iVariables) ReturnFalse; //--- setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_concatenated, m_iConcatenated) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_concatenated_gradient, m_iConcatenatedGradient) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_gradient, getGradientIndex()) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_memory, m_iMemory) uint global_work_offset[] = {0, 0}; uint global_work_size[] = {Neurons() / m_iVariables, m_iVariables}; kernelExecute(def_k_LSTM_ConcatenatedGradient, global_work_offset, global_work_size) #ifdef _DEBUG vector temp = vector::Zeros(Neurons() * 4); if(!OpenCL.BufferToVector(m_iConcatenatedGradient, temp, temp.Size())) ReturnFalse; #endif //--- uint local_work_size[] = {1, m_iVariables}; setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_concatenated_gradient, m_iConcatenatedGradient) setArgument(def_k_LSTM_HiddenGradient, def_k_lstmhg_hidden_size, Neurons() / m_iVariables) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_hidden_state, m_iHiddenState) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs_gradient, NeuronOCL.getGradientIndex()) setArgument(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs_size, m_iInputs) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_output, getOutputIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_weeights, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_weights_gradient, m_iWeightsGradient) kernelExecuteLoc(def_k_LSTM_HiddenGradient, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getOutput().BufferRead()) ReturnFalse; #endif //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || m_cWeightsLSTM.GetIndex() < 0 || m_iWeightsGradient < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_weights, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_weights_gradient, m_iWeightsGradient) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_matrix_m, m_cFirstMomentumLSTM.GetIndex()) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_matrix_v, m_cSecondMomentumLSTM.GetIndex()) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_l, lr) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_b1, b1) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_b2, b2) uint global_work_offset[] = {0, 0}; uint global_work_size[] = {(m_iInputs + Neurons()) / m_iVariables + 1, Neurons() / m_iVariables}; kernelExecute(def_k_LSTM_UpdateWeightsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!m_cWeightsLSTM.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronLSTMOCL *Source = source; if(!OpenCL || m_cWeightsLSTM.GetIndex() < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {m_cWeightsLSTM.Total()}; ResetLastError(); setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, Source.m_cWeightsLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, m_cFirstMomentumLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, m_cSecondMomentumLSTM.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!m_cWeightsLSTM.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < sizeof(m_iInputs)) ReturnFalse; if(FileWriteInteger(file_handle, m_iVariables, INT_VALUE) < sizeof(m_iInputs)) ReturnFalse; if((m_cWeightsLSTM.GetIndex() >= 0 && !m_cWeightsLSTM.BufferRead()) || !m_cWeightsLSTM.Save(file_handle)) ReturnFalse; if((m_cFirstMomentumLSTM.GetIndex() >= 0 && !m_cFirstMomentumLSTM.BufferRead()) || !m_cFirstMomentumLSTM.Save(file_handle)) ReturnFalse; if((m_cSecondMomentumLSTM.GetIndex() >= 0 && !m_cSecondMomentumLSTM.BufferRead()) || !m_cSecondMomentumLSTM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; m_iInputs = FileReadInteger(file_handle); m_iVariables = FileReadInteger(file_handle); //--- m_cWeightsLSTM.BufferFree(); if(!m_cWeightsLSTM.Load(file_handle) || !m_cWeightsLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- m_cFirstMomentumLSTM.BufferFree(); if(!m_cFirstMomentumLSTM.Load(file_handle) || !m_cFirstMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- m_cSecondMomentumLSTM.BufferFree(); if(!m_cSecondMomentumLSTM.Load(file_handle) || !m_cSecondMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(m_iMemory >= 0) OpenCL.BufferFree(m_iMemory); m_iMemory = OpenCL.AddBuffer(sizeof(float) * 2 * Neurons(), CL_MEM_READ_WRITE); if(m_iMemory < 0) ReturnFalse; //--- if(m_iConcatenated >= 0) OpenCL.BufferFree(m_iConcatenated); m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * 4 * Neurons(), CL_MEM_READ_WRITE); if(m_iConcatenated < 0) ReturnFalse; //--- if(m_iConcatenatedGradient >= 0) OpenCL.BufferFree(m_iConcatenatedGradient); m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * 4 * Neurons(), CL_MEM_READ_WRITE); if(m_iConcatenatedGradient < 0) ReturnFalse; //--- if(m_iHiddenState >= 0) OpenCL.BufferFree(m_iHiddenState); m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * Neurons(), CL_MEM_READ_WRITE); if(m_iHiddenState < 0) ReturnFalse; //--- if(m_iWeightsGradient >= 0) OpenCL.BufferFree(m_iWeightsGradient); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * m_cWeightsLSTM.Total(), CL_MEM_READ_WRITE); if(m_iWeightsGradient < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- float emp[]; ArrayResize(emp, Neurons() * 2); ArrayInitialize(emp, 0); if(!OpenCL.BufferWrite(m_iHiddenState, emp, 0, 0, Neurons())) ReturnFalse; if(!OpenCL.BufferWrite(m_iMemory, emp, 0, 0, Neurons() * 2)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLSTMOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { OpenCL.BufferFree(m_iMemory); OpenCL.BufferFree(m_iHiddenState); OpenCL.BufferFree(m_iConcatenated); OpenCL.BufferFree(m_iConcatenatedGradient); OpenCL.BufferFree(m_iWeightsGradient); } CNeuronBaseOCL::SetOpenCL(obj); m_cWeightsLSTM.BufferCreate(OpenCL); m_cFirstMomentumLSTM.BufferCreate(OpenCL); m_cSecondMomentumLSTM.BufferCreate(OpenCL); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * m_cWeightsLSTM.Total(), CL_MEM_READ_WRITE); int numNeurons = Neurons(); m_iMemory = OpenCL.AddBuffer(sizeof(float) * numNeurons * 2, CL_MEM_READ_WRITE); m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * numNeurons, CL_MEM_READ_WRITE); m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * numNeurons * 4, CL_MEM_READ_WRITE); m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * numNeurons * 4, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- if(!cHiddenStates.Init(0, 0, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cHiddenStates.SetActivationFunction(None); iWindowHidden = window_key; //--- if(!cA.Init(0, 1, OpenCL, iWindowHidden, iWindowHidden, iWindowHidden, units_count, 1, optimization, iBatch)) ReturnFalse; cA.SetActivationFunction(SIGMOID); //--- if(!cB.Init(0, 2, OpenCL, window, window, iWindowHidden, units_count, 1, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(SIGMOID); //--- if(!cAB.Init(0, 3, OpenCL, 2 * iWindowHidden * units_count, optimization, iBatch)) ReturnFalse; cAB.SetActivationFunction(None); //--- if(!cC.Init(0, 4, OpenCL, 2 * iWindowHidden, 2 * iWindowHidden, window, units_count, 1, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); //--- SetActivationFunction(None); if(!SetOutput(cC.getOutput()) || !SetGradient(cC.getGradient())) ReturnFalse; //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cA.FeedForward(cHiddenStates.AsObject())) ReturnFalse; if(!cB.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cA.getOutput(), cB.getOutput(), cAB.getOutput(), iWindowHidden, iWindowHidden, cA.Neurons() / iWindowHidden)) ReturnFalse; if(!cC.FeedForward(cAB.AsObject())) ReturnFalse; //--- if(!bTrain) if(!SumAndNormalize(cA.getOutput(), cB.getOutput(), cHiddenStates.getOutput(), iWindowHidden, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cAB.CalcHiddenGradients(cC.AsObject())) ReturnFalse; //--- if(!DeConcat(cA.getGradient(), cB.getGradient(), cAB.getGradient(), iWindowHidden, iWindowHidden, cA.Neurons() / iWindowHidden)) ReturnFalse; if(cA.Activation() != None && !DeActivation(cA.getOutput(), cA.getGradient(), cA.getGradient(), cA.Activation())) ReturnFalse; if(cB.Activation() != None && !DeActivation(cB.getOutput(), cB.getGradient(), cB.getGradient(), cB.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cB.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cA.UpdateInputWeights(cHiddenStates.AsObject())) ReturnFalse; if(!SumAndNormalize(cA.getOutput(), cB.getOutput(), cHiddenStates.getOutput(), iWindowHidden, true)) ReturnFalse; //--- if(!cB.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cC.UpdateInputWeights(cAB.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cHiddenStates.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cB.SetOpenCL(OpenCL); cAB.SetOpenCL(OpenCL); cC.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowHidden)) < INT_VALUE) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cB.Save(file_handle)) ReturnFalse; if(!cC.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindowHidden = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC.AsObject())) ReturnFalse; //--- if(!cHiddenStates.Init(0, 0, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; if(!cAB.Init(0, 3, OpenCL, 2 * cA.Neurons(), optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cC.getOutput()) || !SetGradient(cC.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; CBufferFloat *output = cHiddenStates.getOutput(); if(!output || !output.Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cXProject.Init(0, 0, OpenCL, window, window, window_key + 2, units_count, 1, optimization, iBatch)) ReturnFalse; cXProject.SetActivationFunction(None); if(!cZProject.Init(0, 1, OpenCL, window, window, window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cZProject.SetActivationFunction(SIGMOID); if(!cInsideConv.Init(0, 2, OpenCL, 3, 1, 1, window_key, units_count, optimization, iBatch)) ReturnFalse; cInsideConv.SetActivationFunction(SIGMOID); if(!cSSM.Init(0, 3, OpenCL, window_key, window_key, units_count, optimization, iBatch)) ReturnFalse; if(!cZSSM.Init(0, 4, OpenCL, 2 * window_key * units_count, optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); if(!cOutProject.Init(0, 5, OpenCL, 2 * window_key, 2 * window_key, window, units_count, 1, optimization, iBatch)) ReturnFalse; cOutProject.SetActivationFunction(None); //--- if(!Temp.BufferInit(window * units_count, 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SetOutput(cOutProject.getOutput())) ReturnFalse; if(!SetGradient(cOutProject.getGradient())) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cXProject.FeedForward(NeuronOCL)) ReturnFalse; if(!cZProject.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!cInsideConv.FeedForward(cXProject.AsObject())) ReturnFalse; if(!cSSM.FeedForward(cInsideConv.AsObject())) ReturnFalse; if(!Concat(cSSM.getOutput(), cZProject.getOutput(), cZSSM.getOutput(), 1, 1, cSSM.Neurons())) ReturnFalse; if(!cOutProject.FeedForward(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cZSSM.CalcHiddenGradients(cOutProject.AsObject())) ReturnFalse; if(!DeConcat(cSSM.getGradient(), cZProject.getGradient(), cZSSM.getGradient(), 1, 1, cSSM.Neurons())) ReturnFalse; if(cZProject.Activation() != None && !DeActivation(cZProject.getOutput(), cZProject.getGradient(), cZProject.getGradient(), cZProject.Activation())) ReturnFalse; if(!cInsideConv.CalcHiddenGradients(cSSM.AsObject())) ReturnFalse; if(!cXProject.CalcHiddenGradients(cInsideConv.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cZProject.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), GetPointer(Temp), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cXProject.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), GetPointer(Temp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cXProject.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cZProject.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cInsideConv.UpdateInputWeights(cXProject.AsObject())) ReturnFalse; if(!cSSM.UpdateInputWeights(cInsideConv.AsObject())) ReturnFalse; if(!cOutProject.UpdateInputWeights(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMambaOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cXProject.SetOpenCL(OpenCL); cZProject.SetOpenCL(OpenCL); cInsideConv.SetOpenCL(OpenCL); cSSM.SetOpenCL(OpenCL); cZSSM.SetOpenCL(OpenCL); cOutProject.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cXProject.Save(file_handle)) ReturnFalse; if(!cZProject.Save(file_handle)) ReturnFalse; if(!cInsideConv.Save(file_handle)) ReturnFalse; if(!cSSM.Save(file_handle)) ReturnFalse; if(!cOutProject.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cXProject.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cZProject.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSSM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cOutProject.AsObject())) ReturnFalse; //--- if(!cZSSM.Init(0, 3, OpenCL, 2 * cZProject.Neurons(), optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); //--- if(!SetOutput(cOutProject.getOutput())) ReturnFalse; if(!SetGradient(cOutProject.getGradient())) ReturnFalse; //--- if(!Temp.BufferInit(Neurons(), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Clear(void) { return cSSM.Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; //--- if(!cMamba.Init(0, 0, OpenCL, window, window_key, units_count, optimization, iBatch)) ReturnFalse; if(!cMambaResidual.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cMambaResidual.SetActivationFunction(None); if(!cFF[0].Init(0, 2, OpenCL, window, window, 4 * window, units_count, 1, optimization, iBatch)) ReturnFalse; cFF[0].SetActivationFunction(LReLU); if(!cFF[1].Init(0, 2, OpenCL, 4 * window, 4 * window, window, units_count, 1, optimization, iBatch)) ReturnFalse; cFF[1].SetActivationFunction(None); //--- SetActivationFunction(None); SetGradient(cFF[1].getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cMamba.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormalize(cMamba.getOutput(), NeuronOCL.getOutput(), cMambaResidual.getOutput(), iWindow, true)) ReturnFalse; if(!cFF[0].FeedForward(cMambaResidual.AsObject())) ReturnFalse; if(!cFF[1].FeedForward(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cMambaResidual.getOutput(), cFF[1].getOutput(), getOutput(), iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cFF[0].CalcHiddenGradients(cFF[1].AsObject())) ReturnFalse; if(!cMambaResidual.CalcHiddenGradients(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cMambaResidual.getGradient(), getGradient(), cMamba.getGradient(), iWindow, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMamba.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), cMambaResidual.getGradient(), cMamba.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(cMambaResidual.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; } else if(!SumAndNormalize(cMamba.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cMamba.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFF[0].UpdateInputWeights(cMambaResidual.AsObject())) ReturnFalse; if(!cFF[1].UpdateInputWeights(cFF[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; //--- if(!cMamba.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFF[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cMamba.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cFF[i].AsObject())) ReturnFalse; if(!cMambaResidual.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- SetGradient(cFF[1].getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMambaBlockOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cMamba.SetOpenCL(OpenCL); cMambaResidual.SetOpenCL(OpenCL); cFF[0].SetOpenCL(OpenCL); cFF[1].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_out, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindowOut = window_out; iUnitsOut = units_out; //--- int index = 0; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- Projection Time cProjectionX_Time.Clear(); cProjectionX_Time.SetOpenCL(OpenCL); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, window_in, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iUnitsOut, window_in, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- Projection Variables cProjectionX_Variable.Clear(); cProjectionX_Variable.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iUnitsOut, units_in, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Variable.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- HiddenState index++; if(!cHiddenStates.Init(0, index, OpenCL, 2 * iUnitsOut * iWindowOut, optimization, iBatch)) ReturnFalse; //--- A*H index++; if(!cA.Init(0, index, OpenCL, iWindowOut, iWindowOut, 2 * iWindowOut, iUnitsOut, 2, optimization, iBatch)) ReturnFalse; if(!SumAndNormalize(cA.GetWeightsConv(), cA.GetWeightsConv(), cA.GetWeightsConv(), iWindowOut, false, 0, 0, 0, 0.05f)) ReturnFalse; cA.SetActivationFunction(MinusSoftPlus); //--- B index++; if(!cB_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cB_Time.SetActivationFunction(TANH); index++; if(!cB_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cB_Variable.SetActivationFunction(TANH); //--- C index++; if(!cC_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cC_Time.SetActivationFunction(TANH); index++; if(!cC_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cC_Variable.SetActivationFunction(TANH); //--- Delta index++; if(!cDelta_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cDelta_Time.SetActivationFunction(SoftPlus); index++; if(!cDelta_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cDelta_Variable.SetActivationFunction(SoftPlus); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; //--- Projection Time int total = cProjectionX_Time.Total(); for(int i = 0; i < total; i++) { x_time = cProjectionX_Time.At(i); if(!x_time || !x_time.FeedForward(inp)) ReturnFalse; inp = x_time; } //--- Projection Variable inp = NeuronOCL; total = cProjectionX_Variable.Total(); for(int i = 0; i < total; i++) { x_var = cProjectionX_Variable.At(i); if(!x_var || !x_var.FeedForward(inp)) ReturnFalse; inp = x_var; } //--- if(!cA.FeedForward(cHiddenStates.AsObject())) ReturnFalse; if(!cB_Time.FeedForward(x_time) || !cB_Variable.FeedForward(x_var)) ReturnFalse; if(!cC_Time.FeedForward(x_time) || !cC_Variable.FeedForward(x_var)) ReturnFalse; if(!cDelta_Time.FeedForward(x_time) || !cDelta_Variable.FeedForward(x_var)) ReturnFalse; //--- if(!cHiddenStates.SwapOutputs()) ReturnFalse; //--- return feedForwardSSM2D(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::feedForwardSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {global_work_size[0], 1}; int kernel = def_k_SSM2D_FeedForward; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2d_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_y, getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cHiddenStates.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::calcInputGradientsSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {1, global_work_size[1]}; int kernel = def_k_SSM2D_CalcHiddenGradient; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2dhg_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_ah, cA.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_time, cB_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_var, cB_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_time, cC_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_var, cC_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_time, cDelta_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_var, cDelta_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_time, x_time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_var, x_var.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_y, getGradientIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!x_var.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!calcInputGradientsSSM2D()) ReturnFalse; //--- Deactivation CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!x_time || !x_var) ReturnFalse; if(x_time.Activation() != None) if(!DeActivation(x_time.getOutput(), x_time.getGradient(), x_time.getGradient(), x_time.Activation())) ReturnFalse; if(x_var.Activation() != None) if(!DeActivation(x_var.getOutput(), x_var.getGradient(), x_var.getGradient(), x_var.Activation())) ReturnFalse; if(cB_Time.Activation() != None) if(!DeActivation(cB_Time.getOutput(), cB_Time.getGradient(), cB_Time.getGradient(), cB_Time.Activation())) ReturnFalse; if(cB_Variable.Activation() != None) if(!DeActivation(cB_Variable.getOutput(), cB_Variable.getGradient(), cB_Variable.getGradient(), cB_Variable.Activation())) ReturnFalse; if(cC_Time.Activation() != None) if(!DeActivation(cC_Time.getOutput(), cC_Time.getGradient(), cC_Time.getGradient(), cC_Time.Activation())) ReturnFalse; if(cC_Variable.Activation() != None) if(!DeActivation(cC_Variable.getOutput(), cC_Variable.getGradient(), cC_Variable.getGradient(), cC_Variable.Activation())) ReturnFalse; if(cDelta_Time.Activation() != None) if(!DeActivation(cDelta_Time.getOutput(), cDelta_Time.getGradient(), cDelta_Time.getGradient(), cDelta_Time.Activation())) ReturnFalse; if(cDelta_Variable.Activation() != None) if(!DeActivation(cDelta_Variable.getOutput(), cDelta_Variable.getGradient(), cDelta_Variable.getGradient(), cDelta_Variable.Activation())) ReturnFalse; if(cA.Activation() != None) if(!DeActivation(cA.getOutput(), cA.getGradient(), cA.getGradient(), cA.Activation())) ReturnFalse; //--- Gradient to projections X CBufferFloat *grad_x_time = x_time.getGradient(); CBufferFloat *grad_x_var = x_var.getGradient(); if(!x_time.SetGradient(x_time.getPrevOutput(), false) || !x_var.SetGradient(x_var.getPrevOutput(), false)) ReturnFalse; //--- B -> X if(!x_time.CalcHiddenGradients(cB_Time.AsObject()) || !SumAndNormalize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cB_Variable.AsObject()) || !SumAndNormalize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; //--- C -> X if(!x_time.CalcHiddenGradients(cC_Time.AsObject()) || !SumAndNormalize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cC_Variable.AsObject()) || !SumAndNormalize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; //--- Delta -> X if(!x_time.CalcHiddenGradients(cDelta_Time.AsObject()) || !SumAndNormalize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cDelta_Variable.AsObject()) || !SumAndNormalize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_time.SetGradient(grad_x_time, false) || !x_var.SetGradient(grad_x_var, false)) ReturnFalse; //--- Projection Variable int total = cProjectionX_Variable.Total() - 2; for(int i = total; i >= 0; i--) { x_var = cProjectionX_Variable[i]; if(!x_var || !x_var.CalcHiddenGradients(cProjectionX_Variable[i + 1])) ReturnFalse; } //--- Projection Time total = cProjectionX_Time.Total() - 2; for(int i = total; i >= 0; i--) { x_time = cProjectionX_Time[i]; if(!x_time || !x_time.CalcHiddenGradients(cProjectionX_Time[i + 1])) ReturnFalse; } //--- Projections -> inputs if(!NeuronOCL.CalcHiddenGradients(x_var.AsObject())) ReturnFalse; grad_x_time = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(x_time.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(x_time.AsObject()) || !SumAndNormalize(grad_x_time, NeuronOCL.getGradient(), grad_x_time, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(grad_x_time, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cB_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cB_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- C -> X if(!cC_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cC_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- Delta -> X if(!cDelta_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cDelta_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- Projection Variable CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; int total = cProjectionX_Variable.Total() - 1; for(int i = total; i > 0; i--) { x_var = cProjectionX_Variable[i]; if(!x_var || !x_var.UpdateInputWeights(cProjectionX_Variable[i - 1])) ReturnFalse; } //--- Projection Time total = cProjectionX_Time.Total() - 1; for(int i = total; i > 0; i--) { x_time = cProjectionX_Time[i]; if(!x_time || !x_time.UpdateInputWeights(cProjectionX_Time[i - 1])) ReturnFalse; } //--- Projections -> inputs x_var = cProjectionX_Variable[0]; x_time = cProjectionX_Time[0]; if(!x_var.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!x_time.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- A*H if(!cHiddenStates.SwapOutputs() || !cA.UpdateInputWeights(cHiddenStates.AsObject()) || !cHiddenStates.SwapOutputs()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuron2DSSMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cHiddenStates.SetOpenCL(OpenCL); cProjectionX_Time.SetOpenCL(OpenCL); cProjectionX_Variable.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cB_Time.SetOpenCL(OpenCL); cB_Variable.SetOpenCL(OpenCL); cC_Time.SetOpenCL(OpenCL); cC_Variable.SetOpenCL(OpenCL); cDelta_Time.SetOpenCL(OpenCL); cDelta_Variable.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cHiddenStates.Save(file_handle)) ReturnFalse; if(!cProjectionX_Time.Save(file_handle)) ReturnFalse; if(!cProjectionX_Variable.Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cB_Time.Save(file_handle)) ReturnFalse; if(!cB_Variable.Save(file_handle)) ReturnFalse; if(!cC_Time.Save(file_handle)) ReturnFalse; if(!cC_Variable.Save(file_handle)) ReturnFalse; if(!cDelta_Time.Save(file_handle)) ReturnFalse; if(!cDelta_Variable.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cHiddenStates.AsObject())) ReturnFalse; if(!cProjectionX_Time.Load(file_handle)) ReturnFalse; if(!cProjectionX_Variable.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB_Variable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC_Variable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_Variable.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cHiddenStates.Clear()) ReturnFalse; if(!cA.Clear()) ReturnFalse; if(!cB_Time.Clear()) ReturnFalse; if(!cB_Variable.Clear()) ReturnFalse; if(!cC_Time.Clear()) ReturnFalse; if(!cC_Variable.Clear()) ReturnFalse; if(!cDelta_Time.Clear()) ReturnFalse; if(!cDelta_Variable.Clear()) ReturnFalse; //--- for(int i = 0; i < cProjectionX_Time.Total(); i++) { if(!cProjectionX_Time[i] || !((CNeuronBaseOCL*)cProjectionX_Time[i]).Clear()) ReturnFalse; } for(int i = 0; i < cProjectionX_Variable.Total(); i++) { if(!cProjectionX_Variable[i] || !((CNeuronBaseOCL*)cProjectionX_Variable[i]).Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return CNeuronCGLSTMOCL::Init(numOutputs, myIndex, open_cl, numNeurons, numNeurons, 1, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- if(!cConcatenateInputs.Init(0, 0, OpenCL, (count + window)*variables, optimization, iBatch)) ReturnFalse; cConcatenateInputs.SetActivationFunction(None); //--- if(!cProjection.Init(0, 1, OpenCL, count + window, count + window, count * 4, 1, variables, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(None); //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::CSLSTM_feedForward(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {cProjection.GetFilters() / 4, cProjection.GetVariables()}; uint kernel = def_k_CSLSTM_FeedForward; setBuffer(kernel, def_k_cslstmff_concatenated, cProjection.getOutputIndex()) setBuffer(kernel, def_k_cslstmff_memory, cProjection.getPrevOutIndex()) setBuffer(kernel, def_k_cslstmff_output, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::CSLSTM_CalcHiddenGradient(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {cProjection.GetFilters(), cProjection.GetVariables()}; uint kernel = def_k_CSLSTM_CalcHiddenGradient; setBuffer(kernel, def_k_cslstmhg_concatenated, cProjection.getOutputIndex()) setBuffer(kernel, def_k_cslstmhg_concatenated_grad, cProjection.getGradientIndex()) setBuffer(kernel, def_k_cslstmhg_memory, cProjection.getPrevOutIndex()) setBuffer(kernel, def_k_cslstmhg_output_grad, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cProjection.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- int hidden = (int)cProjection.GetFilters() / 4; int inputs = (int)cProjection.GetWindow() - hidden; int variables = (int)cProjection.GetVariables(); //--- if(!Concat(NeuronOCL.getOutput(), getOutput(), cConcatenateInputs.getOutput(), inputs, hidden, variables)) ReturnFalse; if(!cProjection.FeedForward(cConcatenateInputs.AsObject())) ReturnFalse; //--- return CSLSTM_feedForward(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- int hidden = (int)cProjection.GetFilters() / 4; int inputs = (int)cProjection.GetWindow() - hidden; int variables = (int)cProjection.GetVariables(); //--- if(!CSLSTM_CalcHiddenGradient()) ReturnFalse; if(!cConcatenateInputs.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), getPrevOutput(), cConcatenateInputs.getGradient(), inputs, hidden, variables)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { return cProjection.UpdateInputWeights(cConcatenateInputs.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!cConcatenateInputs.Init(0, 0, OpenCL, cProjection.GetWindow()*cProjection.GetVariables(), optimization, iBatch)) ReturnFalse; cConcatenateInputs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCGLSTMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cConcatenateInputs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cProjection.WeightsUpdate(((CNeuronCGLSTMOCL*)source).cProjection.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cProjection.Clear()) ReturnFalse; if(!cProjection.getPrevOutput().Fill(0)) ReturnFalse; if(!cConcatenateInputs.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint & periods[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(periods.Size() <= 0) ReturnFalse; int freqs = (int(window_out / 2 + 2 * periods.Size()) - 1) / int(2 * periods.Size()); if(freqs <= 0) ReturnFalse; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cProjection.Init(0, index, OpenCL, window, window, window_out - 2 * freqs * periods.Size(), units_count, 1, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); index++; if(!cNorm.Init(0, index, OpenCL, cProjection.Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; if(!cProjectionWithTE.Init(0, index, OpenCL, window, units_count, periods, freqs, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(cProjection.AsObject())) ReturnFalse; if(!cProjectionWithTE.FeedForward(NeuronOCL, SecondInput)) ReturnFalse; if(!Concat(cNorm.getOutput(), cProjectionWithTE.getOutput(), Output, cProjection.GetFilters(), cProjectionWithTE.GetWindowOut(), cProjection.GetUnits())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(cNorm.getGradient(), cProjectionWithTE.getGradient(), Gradient, cProjection.GetFilters(), cProjectionWithTE.GetWindowOut(), cProjection.GetUnits())) ReturnFalse; if(!cProjection.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cProjectionWithTE.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, cProjection.GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(cProjection.AsObject())) ReturnFalse; if(!cProjectionWithTE.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CMamba4CastEmbeding* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; if(!cProjectionWithTE.WeightsUpdate(Source.cProjectionWithTE.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(!cProjectionWithTE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionWithTE.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CMamba4CastEmbeding::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cProjectionWithTE.SetOpenCL(OpenCL); }