//+------------------------------------------------------------------+ //| NeuroNet_TimeSeries.mqh | //| Copyright 2026, DNG | //| https://www.mql5.com/ru/users/dng | //+------------------------------------------------------------------+ #ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE //+------------------------------------------------------------------+ //| OMPB: online posterior mean bridge | //+------------------------------------------------------------------+ CNeuronOMPBOCL::CNeuronOMPBOCL(void) : iDimension(0), iVariables(0), iSamples(0), iReferenceCapacity(0), iCurrentCapacity(0), iReferenceCount(0), iCurrentCount(0), eMode(OMPB_BYPASS), m_fTau(1.0f), m_fDisagreementMult(0.0f), m_fKLDMult(0.0f), m_fAlphaPrior(-6.0f), m_fAlphaMult(0.0f), m_fMaxKL(0.0f), m_fLastDisagreement(0.0f), m_fLastKL(0.0f), m_fLastAlphaPrior(0.0f), iInvalidFallbacks(0), iKLRejects(0), bSourceAnchor(false), bForwardAccepted(false), bNumericRejected(false), bCurrentTransactionPending(false), iCurrentTransactionCount(0) { cReference = NULL; cCurrent = NULL; } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronOMPBOCL GPU resources. | //+------------------------------------------------------------------+ CNeuronOMPBOCL::~CNeuronOMPBOCL(void) { DeleteObj(cReference); DeleteObj(cCurrent); } //+------------------------------------------------------------------+ //| Initializes CNeuronOMPBOCL device buffers. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint variables, uint samples, uint reference_capacity, uint current_capacity, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- An incomplete reinitialization must not leave an older forward updateable. bForwardAccepted = false; bNumericRejected = false; bCurrentTransactionPending = false; iCurrentTransactionCount = 0; //--- Validate the fixed OMPB tensor dimensions before allocating resources. if(dimension == 0) ReturnFalseEx("dimension is zero"); if(variables == 0) ReturnFalseEx("variables is zero"); if(samples == 0) ReturnFalseEx("samples is zero"); if(reference_capacity == 0) ReturnFalseEx("reference capacity is zero"); if(current_capacity == 0) ReturnFalseEx("current capacity is zero"); if(samples > 64) ReturnFalseEx("samples exceeds 64"); //--- Reject every fixed OMPB product before a wrapped value reaches an allocation. uint numeric_total = 0; if(!ValidateRuntimeSizes(dimension, variables, samples, reference_capacity, current_capacity, numeric_total)) ReturnFalseEx("runtime tensor size"); //--- Initialize the base OMPB neuron storage. if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * variables, optimization_type, batch)) ReturnFalseEx("base neuron"); //--- Persist the validated OMPB dimensions for subsequent buffer allocation. iDimension = dimension; iVariables = variables; iSamples = samples; iReferenceCapacity = reference_capacity; iCurrentCapacity = current_capacity; activation = None; //--- Initialize the posterior residual and deterministic gate components. if(!cResidual.Init(0, 0, OpenCL, variables, dimension, dimension, samples, optimization_type, batch)) ReturnFalseEx("residual posterior"); if(!cAlpha.Init(0, 0, OpenCL, variables, optimization_type, batch)) ReturnFalseEx("alpha parameters"); //--- CParams stores a logical alpha as an input weight and a bias. CBufferFloat *alpha_weights = cAlpha.getWeightsParams(); if(!alpha_weights) ReturnFalseEx("alpha weights are null"); if(alpha_weights.Total() != int(2 * variables)) ReturnFalseEx("alpha physical shape mismatch"); if(!alpha_weights.Fill(0.0f)) ReturnFalseEx("alpha posterior reset"); if(!alpha_weights.BufferRead()) ReturnFalseEx("read alpha weights"); for(uint index = 0; index < variables; index++) if(!alpha_weights.Update(2 * index, -6.0f)) ReturnFalseEx("initialize alpha logit"); if(!alpha_weights.BufferWrite()) ReturnFalseEx("write alpha weights"); if(!cAlpha.FeedForward()) ReturnFalseEx("alpha forward"); if(!cGate.Init(0, 0, OpenCL, dimension * variables, optimization_type, batch)) ReturnFalseEx("gate buffer"); if(!cCandidate.BufferInit(dimension * variables, 0.0f)) ReturnFalseEx("candidate host buffer"); if(!cCandidate.BufferCreate(OpenCL)) ReturnFalseEx("candidate device buffer"); if(!cInputGradient.BufferInit(dimension * variables, 0.0f)) ReturnFalseEx("input gradient host buffer"); if(!cInputGradient.BufferCreate(OpenCL)) ReturnFalseEx("input gradient device buffer"); if(!InitializeAlphaRuntime()) ReturnFalseEx("alpha runtime buffers"); if(!InitializeNumericRuntime(numeric_total)) ReturnFalseEx("numeric runtime buffers"); //--- Allocate the fixed-size reference and current history owners. if(!cReference) cReference = new CNeuronAddToStack(); if(!cCurrent) cCurrent = new CNeuronAddToStack(); if(!cReference) ReturnFalseEx("reference history allocation"); if(!cCurrent) ReturnFalseEx("current history allocation"); if(!cReference.Init(0, 0, OpenCL, reference_capacity, dimension, variables, optimization_type, batch)) ReturnFalseEx("reference history"); if(!cCurrent.Init(0, 0, OpenCL, current_capacity, dimension, variables, optimization_type, batch)) ReturnFalseEx("current history"); //--- Initialize the history transposes and posterior sample holders. if(!cReferenceHistoryTranspose.Init(0, 0, OpenCL, variables, reference_capacity, dimension, optimization_type, batch)) ReturnFalseEx("reference history transpose"); if(!cCurrentHistoryTranspose.Init(0, 0, OpenCL, variables, current_capacity, dimension, optimization_type, batch)) ReturnFalseEx("current history transpose"); if(!cReferenceSamples.Init(0, 0, OpenCL, samples * variables * reference_capacity * dimension, optimization_type, batch)) ReturnFalseEx("reference posterior samples"); if(!cCurrentSamples.Init(0, 0, OpenCL, samples * variables * current_capacity * dimension, optimization_type, batch)) ReturnFalseEx("current posterior samples"); if(!cReferenceSamplesTranspose.Init(0, 0, OpenCL, samples, variables * reference_capacity, dimension, optimization_type, batch)) ReturnFalseEx("reference samples transpose"); if(!cCurrentSamplesTranspose.Init(0, 0, OpenCL, samples, variables * current_capacity, dimension, optimization_type, batch)) ReturnFalseEx("current samples transpose"); //--- Initialize host-side disagreement and KL metric buffers. if(!cReferenceDistance.BufferInit(variables * reference_capacity * samples * samples, 0.0f)) ReturnFalseEx("reference distance host buffer"); if(!cCurrentDistance.BufferInit(variables * current_capacity * samples * samples, 0.0f)) ReturnFalseEx("current distance host buffer"); if(!cDisagreement.BufferInit(variables, 0.0f)) ReturnFalseEx("disagreement host buffer"); if(!cKLMuWeights.BufferInit(variables * dimension * dimension, 0.0f)) ReturnFalseEx("KL weight mean host buffer"); if(!cKLLogVarWeights.BufferInit(variables * dimension * dimension, 0.0f)) ReturnFalseEx("KL weight log variance host buffer"); if(!cKLMuBias.BufferInit(variables * dimension, 0.0f)) ReturnFalseEx("KL bias mean host buffer"); if(!cKLLogVarBias.BufferInit(variables * dimension, 0.0f)) ReturnFalseEx("KL bias log variance host buffer"); if(!cKLMu.BufferInit(variables * dimension * (dimension + 1), 0.0f)) ReturnFalseEx("KL mean host buffer"); if(!cKLLogVar.BufferInit(variables * dimension * (dimension + 1), 0.0f)) ReturnFalseEx("KL log variance host buffer"); if(!cKLValues.BufferInit(variables * dimension * (dimension + 1), 0.0f)) ReturnFalseEx("KL values host buffer"); //--- Create the OpenCL buffers for disagreement and KL diagnostics. if(!cReferenceDistance.BufferCreate(OpenCL)) ReturnFalseEx("reference distance device buffer"); if(!cCurrentDistance.BufferCreate(OpenCL)) ReturnFalseEx("current distance device buffer"); if(!cDisagreement.BufferCreate(OpenCL)) ReturnFalseEx("disagreement device buffer"); if(!cKLMuWeights.BufferCreate(OpenCL)) ReturnFalseEx("KL weight mean device buffer"); if(!cKLLogVarWeights.BufferCreate(OpenCL)) ReturnFalseEx("KL weight log variance device buffer"); if(!cKLMuBias.BufferCreate(OpenCL)) ReturnFalseEx("KL bias mean device buffer"); if(!cKLLogVarBias.BufferCreate(OpenCL)) ReturnFalseEx("KL bias log variance device buffer"); if(!cKLMu.BufferCreate(OpenCL)) ReturnFalseEx("KL mean device buffer"); if(!cKLLogVar.BufferCreate(OpenCL)) ReturnFalseEx("KL log variance device buffer"); if(!cKLValues.BufferCreate(OpenCL)) ReturnFalseEx("KL values device buffer"); //--- Finalize the posterior regularizer and deterministic gate state. cResidual.SetKLDMult(m_fKLDMult); if(!PrepareGate()) ReturnFalseEx("prepare gate"); return(true); } //+------------------------------------------------------------------+ //| Sets the mode of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetMode(const ENUM_OMPB_MODE mode) { if(mode < OMPB_BYPASS || mode > OMPB_INFERENCE) ReturnFalseEx("mode outside supported range"); eMode = mode; return(true); } //+------------------------------------------------------------------+ //| Starts an opt-in transaction for the Current history update. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::BeginCurrentTransaction(void) { //--- Validate transaction ownership and the Current history buffers. if(bCurrentTransactionPending) ReturnFalseEx("current transaction is already pending"); if(!cCurrent || !cCurrent.getOutput() || !cCurrent.getPrevOutput()) ReturnFalseEx("current transaction buffers are null"); const uint total = iCurrentCapacity * iDimension * iVariables; if(cCurrent.getOutput().Total() != int(total) || cCurrent.getPrevOutput().Total() != int(total)) ReturnFalseEx("current transaction buffer shape mismatch"); //--- Snapshot the complete ring before feed-forward can mutate it. if(!CopyBufferRaw(cCurrent.getOutput(), cCurrent.getPrevOutput(), total)) ReturnFalseEx("current transaction snapshot"); iCurrentTransactionCount = iCurrentCount; bCurrentTransactionPending = true; return(true); } //+------------------------------------------------------------------+ //| Commits the pending Current history transaction. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::CommitCurrentTransaction(void) { if(!bCurrentTransactionPending) ReturnFalseEx("current transaction is not pending"); bCurrentTransactionPending = false; return(true); } //+------------------------------------------------------------------+ //| Restores Current history after a rejected Stage 02 update. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::RollbackCurrentTransaction(void) { //--- Preserve the pending state until the complete snapshot is restored. if(!bCurrentTransactionPending) ReturnFalseEx("current transaction is not pending"); if(!cCurrent || !cCurrent.getOutput() || !cCurrent.getPrevOutput()) ReturnFalseEx("current transaction buffers are null"); const uint total = iCurrentCapacity * iDimension * iVariables; if(!CopyBufferRaw(cCurrent.getPrevOutput(), cCurrent.getOutput(), total)) ReturnFalseEx("current transaction restore"); //--- Publish the saved count only after the device history is restored. iCurrentCount = iCurrentTransactionCount; bCurrentTransactionPending = false; bForwardAccepted = false; return(true); } //+------------------------------------------------------------------+ //| Sets the tau of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetTau(const float value) { if(!MathIsValidNumber(value) || value <= 0.0f) ReturnFalseEx("value is nonfinite or non-positive"); m_fTau = value; return(true); } //+------------------------------------------------------------------+ //| Sets the disagreement mult of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetDisagreementMult(const float value) { if(!MathIsValidNumber(value) || value < 0.0f) ReturnFalseEx("value is nonfinite or negative"); m_fDisagreementMult = value; return(true); } //+------------------------------------------------------------------+ //| Sets the kld mult of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetKLDMult(const float value) { if(!MathIsValidNumber(value) || value < 0.0f) ReturnFalseEx("value is nonfinite or negative"); m_fKLDMult = value; cResidual.SetKLDMult(value); return(true); } //+------------------------------------------------------------------+ //| Sets the alpha prior of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetAlphaPrior(const float value) { if(!MathIsValidNumber(value)) ReturnFalseEx("value is nonfinite"); //--- Publish the new configuration only after its device-side constant refreshes. if(cAlphaPriorOffset.Total() > 0) { if(!cAlphaPriorOffset.Update(0, -value)) ReturnFalseEx("alpha prior offset"); if(!cAlphaPriorOffset.BufferWrite()) ReturnFalseEx("write alpha prior offset"); } m_fAlphaPrior = value; return(true); } //+------------------------------------------------------------------+ //| Sets the alpha mult of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetAlphaMult(const float value) { if(!MathIsValidNumber(value) || value < 0.0f) ReturnFalseEx("value is nonfinite or negative"); m_fAlphaMult = value; return(true); } //+------------------------------------------------------------------+ //| Sets the max kl of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetMaxKL(const float value) { if(!MathIsValidNumber(value) || value < 0.0f) ReturnFalseEx("value is nonfinite or negative"); m_fMaxKL = value; return(true); } //+------------------------------------------------------------------+ //| Passes the input through unchanged (identity mapping). | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::Identity(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalseEx("source layer is null"); if(NeuronOCL.Neurons() != int(iDimension * iVariables)) ReturnFalseEx("source dimension mismatch"); if(!CopyBufferRaw(NeuronOCL.getOutput(), getOutput(), iDimension * iVariables)) ReturnFalseEx("output copy"); return(true); } //+------------------------------------------------------------------+ //| Prepares the gate buffers of the CNeuronOMPBOCL layer. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::PrepareGate(void) { bool rejected = false; if(!PrepareGateChecked(rejected)) return(false); return(!rejected); } //+------------------------------------------------------------------+ //| Builds the deterministic gate and reports checked rejection. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::PrepareGateChecked(bool &rejected) { rejected = false; //--- CParams computes the deterministic alpha tensor outside CNet batch sampling. cAlpha.TrainMode(true); if(!cAlpha.FeedForwardChecked(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), rejected)) { cAlpha.TrainMode(bTrain); ReturnFalseEx("checked alpha forward"); } if(rejected) { cAlpha.TrainMode(bTrain); return(true); } if(!MatMul(cAlpha.getOutput(), GetPointer(cAlphaOnes), cGate.getOutput(), 1, 1, iDimension, iVariables, false, 0, 0, 0, GetPointer(cNumericFlags))) { cAlpha.TrainMode(bTrain); ReturnFalseEx("checked broadcast alpha logits"); } if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iDimension * iVariables, rejected)) { cAlpha.TrainMode(bTrain); ReturnFalseEx("broadcast alpha numeric reduction"); } if(rejected) { cAlpha.TrainMode(bTrain); return(true); } if(!Activation(cGate.getOutput(), cGate.getOutput(), SIGMOID)) { cAlpha.TrainMode(bTrain); ReturnFalseEx("sigmoid activation"); } cAlpha.TrainMode(bTrain); return(true); } //+------------------------------------------------------------------+ //| Allocates transient and constant OMPB alpha GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::InitializeAlphaRuntime(void) { //--- Allocate the feature-gradient scratch used by the gate backward pass. if(!cAlphaGradient.BufferInit(iDimension * iVariables, 0.0f)) ReturnFalseEx("alpha feature gradient host buffer"); if(!cAlphaGradient.BufferCreate(OpenCL)) ReturnFalseEx("alpha feature gradient device buffer"); //--- Allocate the runtime-only disagreement direction for alpha logits. if(!cAlphaDisagreement.BufferInit(iVariables, 0.0f)) ReturnFalseEx("alpha disagreement host buffer"); if(!cAlphaDisagreement.BufferCreate(OpenCL)) ReturnFalseEx("alpha disagreement device buffer"); //--- Build the immutable broadcast vector and configurable prior offset. if(!cAlphaOnes.BufferInit(iDimension, 1.0f)) ReturnFalseEx("alpha broadcast ones host buffer"); if(!cAlphaOnes.BufferCreate(OpenCL)) ReturnFalseEx("alpha broadcast ones device buffer"); if(!cAlphaPriorOffset.BufferInit(1, -m_fAlphaPrior)) ReturnFalseEx("alpha prior offset host buffer"); if(!cAlphaPriorOffset.BufferCreate(OpenCL)) ReturnFalseEx("alpha prior offset device buffer"); //--- Allocate the GPU-only prior reduction intermediates. if(!cAlphaDelta.BufferInit(iVariables, 0.0f)) ReturnFalseEx("alpha prior delta host buffer"); if(!cAlphaDelta.BufferCreate(OpenCL)) ReturnFalseEx("alpha prior delta device buffer"); if(!cAlphaPriorSum.BufferInit(1, 0.0f)) ReturnFalseEx("alpha prior sum host buffer"); if(!cAlphaPriorSum.BufferCreate(OpenCL)) ReturnFalseEx("alpha prior sum device buffer"); //--- Finalize alpha runtime allocation. return(true); } //+------------------------------------------------------------------+ //| Multiplies an OMPB allocation span without exceeding INT_MAX. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::MultiplyRuntimeSize(const ulong left, const ulong right, ulong &result) { //--- Reject a zero or overflowing allocation factor before multiplication. result = 0; if(right == 0 || left > ulong(INT_MAX) / right) return(false); //--- Finalize the bounded allocation product. result = left * right; return(true); } //+------------------------------------------------------------------+ //| Validates OMPB allocation products and derives checked scratch. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::ValidateRuntimeSizes(const uint dimension, const uint variables, const uint samples, const uint reference_capacity, const uint current_capacity, uint &numeric_total) { //--- Reset the output span and reject an increment that wraps before multiplication. numeric_total = 0; if(dimension == UINT_MAX) return(false); //--- Derive every allocation span with staged INT_MAX guards. ulong output_total = 0; ulong alpha_total = 0; ulong weights_total = 0; ulong bias_total = 0; ulong posterior_total = 0; ulong posterior_storage_total = 0; ulong reference_history_total = 0; ulong current_history_total = 0; ulong reference_samples_total = 0; ulong current_samples_total = 0; ulong reference_distance_total = 0; ulong current_distance_total = 0; ulong reference_transpose_total = 0; ulong current_transpose_total = 0; ulong kl_values_total = 0; if(!MultiplyRuntimeSize(dimension, variables, output_total) || !MultiplyRuntimeSize(2, variables, alpha_total) || !MultiplyRuntimeSize(variables, dimension, weights_total) || !MultiplyRuntimeSize(weights_total, dimension, weights_total) || !MultiplyRuntimeSize(variables, dimension, bias_total) || !MultiplyRuntimeSize(weights_total + bias_total, 2, posterior_total) || !MultiplyRuntimeSize(weights_total + bias_total, 4, posterior_storage_total) || !MultiplyRuntimeSize(reference_capacity, variables, reference_history_total) || !MultiplyRuntimeSize(reference_history_total, dimension, reference_history_total) || !MultiplyRuntimeSize(current_capacity, variables, current_history_total) || !MultiplyRuntimeSize(current_history_total, dimension, current_history_total) || !MultiplyRuntimeSize(samples, reference_history_total, reference_samples_total) || !MultiplyRuntimeSize(samples, current_history_total, current_samples_total) || !MultiplyRuntimeSize(samples, samples, reference_distance_total) || !MultiplyRuntimeSize(reference_distance_total, variables, reference_distance_total) || !MultiplyRuntimeSize(reference_distance_total, reference_capacity, reference_distance_total) || !MultiplyRuntimeSize(samples, samples, current_distance_total) || !MultiplyRuntimeSize(current_distance_total, variables, current_distance_total) || !MultiplyRuntimeSize(current_distance_total, current_capacity, current_distance_total) || !MultiplyRuntimeSize(variables, reference_capacity, reference_transpose_total) || !MultiplyRuntimeSize(variables, current_capacity, current_transpose_total) || !MultiplyRuntimeSize(variables, dimension, kl_values_total) || !MultiplyRuntimeSize(kl_values_total, dimension + 1, kl_values_total)) return(false); //--- Select the largest current or deferred checked output span. ulong scratch_total = output_total; if(posterior_total > scratch_total) scratch_total = posterior_total; if(reference_samples_total > scratch_total) scratch_total = reference_samples_total; if(current_samples_total > scratch_total) scratch_total = current_samples_total; if(reference_distance_total > scratch_total) scratch_total = reference_distance_total; if(current_distance_total > scratch_total) scratch_total = current_distance_total; //--- Finalize the validated scratch capacity for CBufferFloat allocation. numeric_total = (uint)scratch_total; return(numeric_total > 0); } //+------------------------------------------------------------------+ //| Allocates transient checked numeric status buffers. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::InitializeNumericRuntime(const uint total) { //--- Validate the previously bounded caller-owned scratch span. if(total == 0) ReturnFalseEx("numeric scratch size is zero"); //--- Allocate transient device status, unit, and scalar-reduction buffers. if(!cNumericFlags.BufferInit(total, 0.0f)) ReturnFalseEx("numeric flags host buffer"); if(!cNumericFlags.BufferCreate(OpenCL)) ReturnFalseEx("numeric flags device buffer"); if(!cNumericOnes.BufferInit(total, 1.0f)) ReturnFalseEx("numeric ones host buffer"); if(!cNumericOnes.BufferCreate(OpenCL)) ReturnFalseEx("numeric ones device buffer"); if(!cNumericScalar.BufferInit(1, 0.0f)) ReturnFalseEx("numeric scalar host buffer"); if(!cNumericScalar.BufferCreate(OpenCL)) ReturnFalseEx("numeric scalar device buffer"); if(!cKLMetricScalar.BufferInit(1, 0.0f)) ReturnFalseEx("KL metric scalar host buffer"); if(!cKLMetricScalar.BufferCreate(OpenCL)) ReturnFalseEx("KL metric scalar device buffer"); if(!cDisagreementPositive.BufferInit(iVariables, 0.0f)) ReturnFalseEx("disagreement positive host buffer"); if(!cDisagreementPositive.BufferCreate(OpenCL)) ReturnFalseEx("disagreement positive device buffer"); if(!cDisagreementNegative.BufferInit(iVariables, 0.0f)) ReturnFalseEx("disagreement negative host buffer"); if(!cDisagreementNegative.BufferCreate(OpenCL)) ReturnFalseEx("disagreement negative device buffer"); if(!cDisagreementMetricScalar.BufferInit(1, 0.0f)) ReturnFalseEx("disagreement metric scalar host buffer"); if(!cDisagreementMetricScalar.BufferCreate(OpenCL)) ReturnFalseEx("disagreement metric scalar device buffer"); //--- Finalize numeric runtime allocation without serializing transient state. return(true); } //+------------------------------------------------------------------+ //| KL is measured from the actual posterior parameter tensors. The | //| VAE mean sampler supplies its gradient exactly once during | //| ordinary Bayesian backward; this method is metric/guard only. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::CalculateKL(void) { m_fLastKL = 0.0f; if(eMode != OMPB_CALIBRATE || bSourceAnchor) return(true); uint weights = iVariables * iDimension * iDimension; uint bias = iVariables * iDimension; uint total = weights + bias; CParams *posterior_weights = cResidual.PosteriorWeights(); CParams *posterior_bias = cResidual.PosteriorBias(); //--- Decompose posterior mean and log variance before the KL kernel dispatch. if(!posterior_weights) ReturnFalseEx("weight posterior is null"); if(!posterior_bias) ReturnFalseEx("bias posterior is null"); if(!DeConcat(GetPointer(cKLMuWeights), GetPointer(cKLLogVarWeights), posterior_weights.getOutput(), weights, weights, 1)) ReturnFalseEx("split weight posterior"); if(!DeConcat(GetPointer(cKLMuBias), GetPointer(cKLLogVarBias), posterior_bias.getOutput(), bias, bias, 1)) ReturnFalseEx("split bias posterior"); if(!Concat(GetPointer(cKLMuWeights), GetPointer(cKLMuBias), GetPointer(cKLMu), weights, bias, 1)) ReturnFalseEx("concatenate posterior means"); if(!Concat(GetPointer(cKLLogVarWeights), GetPointer(cKLLogVarBias), GetPointer(cKLLogVar), weights, bias, 1)) ReturnFalseEx("concatenate posterior log variances"); uint global_work_offset[1] = {0}; uint global_work_size[1] = {total}; setBuffer(def_k_OMPBGaussianKL, def_k_ompbkl_mean, cKLMu.GetIndex()) setBuffer(def_k_OMPBGaussianKL, def_k_ompbkl_logvar, cKLLogVar.GetIndex()) setBuffer(def_k_OMPBGaussianKL, def_k_ompbkl_output, cKLValues.GetIndex()) setArgument(def_k_OMPBGaussianKL, def_k_ompbkl_total, (int)total) kernelExecute(def_k_OMPBGaussianKL, global_work_offset, global_work_size) //--- Reduce checked KL values on-device and expose only their scalar sum to the host. bool rejected = false; if(!MatMul(GetPointer(cKLValues), GetPointer(cNumericOnes), GetPointer(cKLMetricScalar), 1, total, 1, 1, false, 0, 0, 0, GetPointer(cNumericFlags))) ReturnFalseEx("checked KL sum"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), 1, rejected)) ReturnFalseEx("KL numeric reduction"); if(rejected) { bNumericRejected = true; return(true); } if(!cKLMetricScalar.BufferRead()) ReturnFalseEx("read KL metric scalar"); if(!MathIsValidNumber(cKLMetricScalar[0])) { bNumericRejected = true; return(true); } //--- Preserve the full posterior KL sum for the MaxKL contract. m_fLastKL = cKLMetricScalar[0]; return(true); } //+------------------------------------------------------------------+ //| Alpha prior is a small parameter-only correction. GateElement | //| has already written the negative forecast derivative into alpha. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::ApplyAlphaPrior(void) { m_fLastAlphaPrior = 0.0f; if(!bTrain || eMode != OMPB_CALIBRATE || bSourceAnchor || m_fAlphaMult <= 0.0f) return(true); CBufferFloat *gradient = cAlpha.getGradient(); if(!gradient) ReturnFalseEx("alpha gradient is null"); //--- Form alpha-prior, reduce its squared norm, and retain only its scalar metric on host. if(!SumVecMatrix(GetPointer(cAlphaPriorOffset), cAlpha.getOutput(), GetPointer(cAlphaDelta), 1, 1, 0, 0, 0, 1.0f, iVariables)) ReturnFalseEx("alpha prior delta"); if(!MatMul(GetPointer(cAlphaDelta), GetPointer(cAlphaDelta), GetPointer(cAlphaPriorSum), 1, iVariables, 1, 1, false)) ReturnFalseEx("alpha prior squared sum"); if(!cAlphaPriorSum.BufferRead()) ReturnFalseEx("read alpha prior squared sum"); if(!MathIsValidNumber(cAlphaPriorSum[0])) ReturnFalseEx("nonfinite alpha prior squared sum"); m_fLastAlphaPrior = 0.5f * m_fAlphaMult * cAlphaPriorSum[0]; //--- The optimizer consumes ascent directions, so add -lambda*(alpha-prior). if(!SumAndNormalize(GetPointer(cAlphaDelta), GetPointer(cAlphaDelta), GetPointer(cAlphaDelta), iVariables, false, 0, 0, 0, -0.5f * m_fAlphaMult)) ReturnFalseEx("alpha prior gradient"); if(!SumAndNormalize(gradient, GetPointer(cAlphaDelta), gradient, iVariables, false, 0, 0, 0, 1.0f)) ReturnFalseEx("merge alpha prior gradient"); return(true); } //+------------------------------------------------------------------+ //| Returns whether the CNeuronOMPBOCL is finite output. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::IsFiniteOutput(void) { if(!Output || !Output.BufferRead()) return(false); for(int i = 0; i < Output.Total(); i++) if(!MathIsValidNumber(Output[i])) return(false); return(true); } //+------------------------------------------------------------------+ //| Builds OMPB sample distributions with fixed capacity. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::CalculateDistribution(CNeuronAddToStack *history, CNeuronTransposeRCDOCL *history_transpose, CNeuronBaseOCL *raw_samples, CNeuronTransposeRCDOCL *samples_transpose, uint capacity, CBufferFloat *distance) { if(!history) ReturnFalseEx("history is null"); if(!history_transpose) ReturnFalseEx("history transpose is null"); if(!raw_samples) ReturnFalseEx("raw samples are null"); if(!samples_transpose) ReturnFalseEx("samples transpose is null"); if(!distance) ReturnFalseEx("distance buffer is null"); if(capacity == 0) ReturnFalseEx("capacity is zero"); if(history.getOutput() == NULL) ReturnFalseEx("history output is null"); if(raw_samples.getOutput() == NULL) ReturnFalseEx("sample output is null"); //--- Transform fixed history into checked posterior samples and their distance layout. if(!history_transpose.FeedForward(history)) ReturnFalseEx("history transpose"); bool rejected = false; if(!cResidual.ForwardSamplesBatchChecked(history.getOutput(), raw_samples.getOutput(), capacity, GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), rejected)) ReturnFalseEx("checked posterior samples"); if(rejected) { bNumericRejected = true; return(true); } if(!samples_transpose.FeedForward(raw_samples)) ReturnFalseEx("samples transpose"); if(!CalcDistance(samples_transpose.getOutput(), distance, iDimension, iSamples, iVariables * capacity, false, GetPointer(cNumericFlags))) ReturnFalseEx("checked raw grouped distance"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables * capacity * iSamples * iSamples, rejected)) ReturnFalseEx("distance numeric reduction"); if(rejected) bNumericRejected = true; return(true); } //+------------------------------------------------------------------+ //| Prepares regularizers after fixed-shape histories are ready. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::PrepareRegularizers(void) { m_fLastDisagreement = 0.0f; if(!cAlphaDisagreement.Fill(0.0f)) ReturnFalseEx("clear alpha disagreement"); if(eMode != OMPB_CALIBRATE || bSourceAnchor || !bTrain || iReferenceCount < iReferenceCapacity || iCurrentCount < iCurrentCapacity) return(true); const float inv_tau2 = 1.0f / MathPow(MathAbs(m_fTau), 2); //--- Build the source and target posterior distributions before disagreement. if(!CalculateDistribution(cReference, GetPointer(cReferenceHistoryTranspose), GetPointer(cReferenceSamples), GetPointer(cReferenceSamplesTranspose), iReferenceCapacity, GetPointer(cReferenceDistance))) ReturnFalseEx("reference distribution"); if(bNumericRejected) return(true); if(!CalculateDistribution(cCurrent, GetPointer(cCurrentHistoryTranspose), GetPointer(cCurrentSamples), GetPointer(cCurrentSamplesTranspose), iCurrentCapacity, GetPointer(cCurrentDistance))) ReturnFalseEx("current distribution"); if(bNumericRejected) return(true); uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {64, iVariables}; uint local_work_size[2] = {64, 1}; setBuffer(def_k_OMPBDisagreement, def_k_ompbd_reference, cReferenceDistance.GetIndex()) setBuffer(def_k_OMPBDisagreement, def_k_ompbd_current, cCurrentDistance.GetIndex()) setBuffer(def_k_OMPBDisagreement, def_k_ompbd_output, cDisagreement.GetIndex()) setArgument(def_k_OMPBDisagreement, def_k_ompbd_source_states, (int)iReferenceCapacity) setArgument(def_k_OMPBDisagreement, def_k_ompbd_target_states, (int)iCurrentCapacity) setArgument(def_k_OMPBDisagreement, def_k_ompbd_samples, (int)iSamples) setArgument(def_k_OMPBDisagreement, def_k_ompbd_inv_tau2, inv_tau2) setBuffer(def_k_OMPBDisagreement, def_k_ompbd_gate, cGate.getOutputIndex()) setBuffer(def_k_OMPBDisagreement, def_k_ompbd_alpha_gradient, cAlphaDisagreement.GetIndex()) setArgument(def_k_OMPBDisagreement, def_k_ompbd_gate_stride, (int)iDimension) setArgument(def_k_OMPBDisagreement, def_k_ompbd_multiplier, -m_fDisagreementMult / (float)iVariables) setBuffer(def_k_OMPBDisagreement, def_k_ompbd_numeric_flags, cNumericFlags.GetIndex()) setArgument(def_k_OMPBDisagreement, def_k_ompbd_flags_enabled, 1) kernelExecuteLoc(def_k_OMPBDisagreement, global_work_offset, global_work_size, local_work_size) //--- Reject the raw kernel status before consuming its sanitized disagreement output. bool rejected = false; if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables, rejected)) ReturnFalseEx("disagreement numeric reduction"); if(rejected) { bNumericRejected = true; return(true); } //--- Form GPU L1(delta) from paired LReLU transforms without changing the objective. if(!Activation(GetPointer(cDisagreement), GetPointer(cDisagreementPositive), LReLU)) ReturnFalseEx("disagreement positive activation"); if(!SumAndNormalize(GetPointer(cDisagreement), GetPointer(cDisagreement), GetPointer(cDisagreementNegative), iVariables, false, 0, 0, 0, -0.5f, GetPointer(cNumericFlags))) ReturnFalseEx("disagreement negative sum"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables, rejected)) ReturnFalseEx("disagreement negative reduction"); if(rejected) { bNumericRejected = true; return(true); } if(!Activation(GetPointer(cDisagreementNegative), GetPointer(cDisagreementNegative), LReLU)) ReturnFalseEx("disagreement negative activation"); if(!SumAndNormalize(GetPointer(cDisagreementPositive), GetPointer(cDisagreementNegative), GetPointer(cDisagreementPositive), iVariables, false, 0, 0, 0, 1.0f / 0.99f, GetPointer(cNumericFlags))) ReturnFalseEx("disagreement absolute sum"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables, rejected)) ReturnFalseEx("disagreement absolute reduction"); if(rejected) { bNumericRejected = true; return(true); } if(!MatMul(GetPointer(cDisagreementPositive), GetPointer(cNumericOnes), GetPointer(cDisagreementMetricScalar), 1, iVariables, 1, 1, false, 0, 0, 0, GetPointer(cNumericFlags))) ReturnFalseEx("disagreement metric sum"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), 1, rejected)) ReturnFalseEx("disagreement metric reduction"); if(rejected) { bNumericRejected = true; return(true); } if(!cDisagreementMetricScalar.BufferRead()) ReturnFalseEx("read disagreement metric scalar"); if(!MathIsValidNumber(cDisagreementMetricScalar[0])) { bNumericRejected = true; return(true); } m_fLastDisagreement = cDisagreementMetricScalar[0] / (float)iVariables; if(m_fDisagreementMult <= 0.0f) return(true); if(!cReferenceSamplesTranspose.getGradient().Fill(0.0f)) ReturnFalseEx("clear reference sample gradient"); if(!cCurrentSamplesTranspose.getGradient().Fill(0.0f)) ReturnFalseEx("clear current sample gradient"); //--- OpenCLExecute receives the dimensionality from work_size. Its offset //--- must carry the same three dimensions; a one-item offset array leaves //--- dimensions 1/2 invalid for OMPBDisagreementGrad. uint source_work_offset[3] = {0, 0, 0}; uint source_work_size[3] = {iVariables * iReferenceCapacity, iSamples, iDimension}; setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_corrections, cReferenceSamplesTranspose.getOutputIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_distance, cReferenceDistance.GetIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_disagreement, cDisagreement.GetIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_gradient, cReferenceSamplesTranspose.getGradientIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_states, (int)iReferenceCapacity) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_samples, (int)iSamples) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_dimension, (int)iDimension) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_variables, (int)iVariables) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_source_sign, 1) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_multiplier, -m_fDisagreementMult / (float)iVariables) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_inv_tau2, inv_tau2) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_gate, cGate.getOutputIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_gate_stride, (int)iDimension) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_numeric_flags, cNumericFlags.GetIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_flags_enabled, 1) kernelExecute(def_k_OMPBDisagreementGrad, source_work_offset, source_work_size) //--- Reject source-gradient arithmetic before it can enter posterior backpropagation. if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables * iReferenceCapacity * iSamples * iDimension, rejected)) ReturnFalseEx("reference disagreement gradient reduction"); if(rejected) { bNumericRejected = true; return(true); } //--- The predecessor owns CalcHiddenGradients(next): this dispatches the //--- TransposeRCD inverse kernel into the raw [K, V * N, D] sample gradient. if(!cReferenceSamples.CalcHiddenGradients(cReferenceSamplesTranspose.AsObject())) ReturnFalseEx("reference transpose backward"); if(!cResidual.BackwardSamplesBatch(cReference.getOutput(), cReferenceSamples.getGradient(), iReferenceCapacity)) ReturnFalseEx("reference posterior backward"); uint target_work_offset[3] = {0, 0, 0}; uint target_work_size[3] = {iVariables * iCurrentCapacity, iSamples, iDimension}; setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_corrections, cCurrentSamplesTranspose.getOutputIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_distance, cCurrentDistance.GetIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_disagreement, cDisagreement.GetIndex()) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_gradient, cCurrentSamplesTranspose.getGradientIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_states, (int)iCurrentCapacity) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_samples, (int)iSamples) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_dimension, (int)iDimension) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_variables, (int)iVariables) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_source_sign, -1) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_multiplier, -m_fDisagreementMult / (float)iVariables) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_inv_tau2, inv_tau2) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_gate, cGate.getOutputIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_gate_stride, (int)iDimension) setBuffer(def_k_OMPBDisagreementGrad, def_k_ompbdg_numeric_flags, cNumericFlags.GetIndex()) setArgument(def_k_OMPBDisagreementGrad, def_k_ompbdg_flags_enabled, 1) kernelExecute(def_k_OMPBDisagreementGrad, target_work_offset, target_work_size) //--- Reject target-gradient arithmetic before it can enter posterior backpropagation. if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iVariables * iCurrentCapacity * iSamples * iDimension, rejected)) ReturnFalseEx("current disagreement gradient reduction"); if(rejected) { bNumericRejected = true; return(true); } if(!cCurrentSamples.CalcHiddenGradients(cCurrentSamplesTranspose.AsObject())) ReturnFalseEx("current transpose backward"); if(!cResidual.BackwardSamplesBatch(cCurrent.getOutput(), cCurrentSamples.getGradient(), iCurrentCapacity)) ReturnFalseEx("current posterior backward"); return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronOMPBOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Every forward invalidates older updates, including identity-only modes. bForwardAccepted = false; bNumericRejected = false; if(!NeuronOCL) ReturnFalseEx("source layer is null"); if(NeuronOCL.Neurons() != int(iDimension * iVariables)) ReturnFalseEx("source dimension mismatch"); if(eMode == OMPB_BYPASS) return(Identity(NeuronOCL)); if(eMode == OMPB_REFERENCE) { if(!Identity(NeuronOCL)) ReturnFalseEx("reference identity output"); //--- Freeze the completed reference until an explicit ResetReference call. if(iReferenceCount < iReferenceCapacity) { if(!cReference.FeedForward(NeuronOCL.AsObject())) ReturnFalseEx("reference history update"); iReferenceCount++; } return(true); } //--- Source anchors retain mean training while suppressing Bayesian regularizers. cResidual.SetRegularizersEnabled(!bSourceAnchor); bool rejected = false; if(!cResidual.FeedForwardChecked(NeuronOCL.AsObject(), GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), rejected)) ReturnFalseEx("checked residual posterior mean"); if(rejected) { iInvalidFallbacks++; return(Identity(NeuronOCL)); } if(!PrepareGateChecked(rejected)) ReturnFalseEx("checked prepare gate"); if(rejected) { iInvalidFallbacks++; return(Identity(NeuronOCL)); } //--- Form H + delta without overwriting the posterior residual output. if(!SumAndNormalize(NeuronOCL.getOutput(), cResidual.getOutput(), &cCandidate, iDimension, false, 0, 0, 0, 1.0f, GetPointer(cNumericFlags))) ReturnFalseEx("checked residual candidate"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iDimension * iVariables, rejected)) ReturnFalseEx("candidate numeric reduction"); if(rejected) { iInvalidFallbacks++; return(Identity(NeuronOCL)); } //--- Gate the residual candidate against the original input state. if(!GateElementMult(&cCandidate, NeuronOCL.getOutput(), cGate.getOutput(), getOutput(), GetPointer(cNumericFlags))) ReturnFalseEx("checked gated output"); if(!ReduceNumericFlags(GetPointer(cNumericFlags), GetPointer(cNumericOnes), GetPointer(cNumericScalar), iDimension * iVariables, rejected)) ReturnFalseEx("gated output numeric reduction"); if(rejected) { iInvalidFallbacks++; return(Identity(NeuronOCL)); } if(!CalculateKL()) ReturnFalseEx("KL calculation"); if(bNumericRejected == true) { iInvalidFallbacks++; return(Identity(NeuronOCL)); } if(m_fMaxKL > 0.0f && m_fLastKL > m_fMaxKL) { iKLRejects++; return(Identity(NeuronOCL)); } //--- Source anchor uses the mean Forecast path without mutating target history. bool own_current_transaction = false; if(eMode == OMPB_CALIBRATE && !bSourceAnchor && !bCurrentTransactionPending) { if(!BeginCurrentTransaction()) ReturnFalseEx("current transaction begin"); own_current_transaction = true; } if(eMode == OMPB_CALIBRATE && !bSourceAnchor) { if(!cCurrent.FeedForward(NeuronOCL.AsObject())) { if(own_current_transaction && !RollbackCurrentTransaction()) ReturnFalseEx("current transaction rollback after history update"); ReturnFalseEx("current history update"); } iCurrentCount = MathMin(iCurrentCount + 1, iCurrentCapacity); } if(!PrepareRegularizers()) { if(own_current_transaction && !RollbackCurrentTransaction()) ReturnFalseEx("current transaction rollback after regularizer preparation"); ReturnFalseEx("regularizer preparation"); } if(bNumericRejected) { if(own_current_transaction && !RollbackCurrentTransaction()) ReturnFalseEx("current transaction rollback after numeric rejection"); iInvalidFallbacks++; return(Identity(NeuronOCL)); } //--- Finalize the residual output after history and regularizer handling. if(own_current_transaction) { if(!CommitCurrentTransaction()) { if(bCurrentTransactionPending && !RollbackCurrentTransaction()) ReturnFalseEx("current transaction rollback after commit failure"); ReturnFalseEx("current transaction commit"); } } bForwardAccepted = true; return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronOMPBOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalseEx("source layer is null"); if(eMode == OMPB_BYPASS || eMode == OMPB_REFERENCE) { if(!CopyBufferRaw(getGradient(), NeuronOCL.getGradient(), iDimension * iVariables)) ReturnFalseEx("identity gradient copy"); return(true); } //--- A rejected forward owns only the direct identity gradient path. if(!bForwardAccepted) { if(!CopyBufferRaw(getGradient(), NeuronOCL.getGradient(), iDimension * iVariables)) ReturnFalseEx("rejected identity gradient copy"); return(true); } //--- Reset OMPB gradient buffers before propagating the gated residual path. if(!cInputGradient.Fill(0.0f)) ReturnFalseEx("clear input gradient"); if(!cResidual.getGradient().Fill(0.0f)) ReturnFalseEx("clear residual gradient"); if(!cAlpha.getGradient().Fill(0.0f)) ReturnFalseEx("clear alpha gradient"); if(!cAlphaGradient.Fill(0.0f)) ReturnFalseEx("clear alpha feature gradient"); //--- Differentiate the gate over candidate=H+delta and the direct H branch. if(!GateElementMultGrad(&cCandidate, cResidual.getGradient(), NeuronOCL.getOutput(), &cInputGradient, cGate.getOutput(), GetPointer(cAlphaGradient), getGradient(), None, None, SIGMOID)) ReturnFalseEx("gated gradient"); if(!MatMul(GetPointer(cAlphaGradient), GetPointer(cAlphaOnes), cAlpha.getGradient(), 1, iDimension, 1, iVariables, false)) ReturnFalseEx("reduce alpha feature gradient"); //--- Merge the transient disagreement direction after forecast reduction. if(!SumAndNormalize(cAlpha.getGradient(), GetPointer(cAlphaDisagreement), cAlpha.getGradient(), iVariables, false, 0, 0, 0, 1.0f)) ReturnFalseEx("merge alpha disagreement"); //--- Include candidate's H identity branch before the Bayesian backward pass. if(!SumAndNormalize(&cInputGradient, cResidual.getGradient(), &cInputGradient, iDimension, false, 0, 0, 0, 1.0f)) ReturnFalseEx("candidate input gradient"); if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalseEx("residual gradient"); //--- Merge the direct residual identity branch after Bayesian affine propagation. if(!SumAndNormalize(&cInputGradient, NeuronOCL.getGradient(), NeuronOCL.getGradient(), iDimension, false, 0, 0, 0, 1.0f)) ReturnFalseEx("input gradient merge"); if(!ApplyAlphaPrior()) ReturnFalseEx("alpha prior"); //--- Finalize the calibrated residual gradient after alpha regularization. return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronOMPBOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!bForwardAccepted) return(true); if(!bTrain || eMode != OMPB_CALIBRATE || bSourceAnchor) return(true); if(!NeuronOCL) ReturnFalseEx("source layer is null"); if(!cResidual.UpdateInputWeights(NeuronOCL.AsObject())) ReturnFalseEx("residual posterior"); if(!cAlpha.UpdateInputWeights()) ReturnFalseEx("alpha parameters"); return(true); } //+------------------------------------------------------------------+ //| Clears the reference linkage of the CNeuronOMPBOCL layer. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::ResetReference(void) { iReferenceCount = 0; if(!cReference) ReturnFalseEx("reference history is null"); if(!cReference.Clear()) ReturnFalseEx("reference history"); if(!cReferenceHistoryTranspose.Clear()) ReturnFalseEx("reference history transpose"); if(!cReferenceSamples.Clear()) ReturnFalseEx("reference samples"); if(!cReferenceSamplesTranspose.Clear()) ReturnFalseEx("reference samples transpose"); if(!cReferenceDistance.Fill(0.0f)) ReturnFalseEx("reference distance"); return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronOMPBOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::Save(const int file_handle) { //--- Write the established OMPB state before the durable history tail. if(bCurrentTransactionPending) ReturnFalseEx("current transaction is pending"); if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalseEx("base neuron"); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iSamples); dFileWriteUInt(file_handle, iReferenceCapacity); dFileWriteUInt(file_handle, iCurrentCapacity); dFileWriteFloat(file_handle, m_fTau); dFileWriteFloat(file_handle, m_fDisagreementMult); dFileWriteFloat(file_handle, m_fKLDMult); dFileWriteFloat(file_handle, m_fAlphaPrior); dFileWriteFloat(file_handle, m_fAlphaMult); dFileWriteFloat(file_handle, m_fMaxKL); if(!cResidual.Save(file_handle)) ReturnFalseEx("residual posterior"); if(!cAlpha.Save(file_handle)) ReturnFalseEx("alpha parameters"); //--- Persist the canonical newest-first history snapshots with their fill counts. if(!cReference || !cReference.getOutput()) ReturnFalseEx("reference history"); if(!cCurrent || !cCurrent.getOutput()) ReturnFalseEx("current history"); if(iReferenceCount > iReferenceCapacity) ReturnFalseEx("reference history count"); if(iCurrentCount > iCurrentCapacity) ReturnFalseEx("current history count"); if(cReference.getOutput().Total() != int(iReferenceCapacity * iDimension * iVariables)) ReturnFalseEx("reference history shape"); if(cCurrent.getOutput().Total() != int(iCurrentCapacity * iDimension * iVariables)) ReturnFalseEx("current history shape"); dFileWriteUInt(file_handle, iReferenceCount); dFileWriteUInt(file_handle, iCurrentCount); if(!cReference.getOutput().Save(file_handle)) ReturnFalseEx("reference history data"); if(!cCurrent.getOutput().Save(file_handle)) ReturnFalseEx("current history data"); //--- Finalize the symmetric OMPB persistence record. return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronOMPBOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::Load(const int file_handle) { //--- A partial load cannot retain permission to update from an older forward. bForwardAccepted = false; bNumericRejected = false; bCurrentTransactionPending = false; iCurrentTransactionCount = 0; if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalseEx("base neuron"); //--- Read the serialized OMPB configuration in the established field order. dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iSamples); dFileReadUInt(file_handle, iReferenceCapacity); dFileReadUInt(file_handle, iCurrentCapacity); dFileReadUFloat(file_handle, m_fTau); dFileReadUFloat(file_handle, m_fDisagreementMult); dFileReadUFloat(file_handle, m_fKLDMult); dFileReadUFloat(file_handle, m_fAlphaPrior); dFileReadUFloat(file_handle, m_fAlphaMult); dFileReadUFloat(file_handle, m_fMaxKL); //--- Validate configuration before reconstructing runtime-only resources. if(iDimension == 0) ReturnFalseEx("dimension is zero"); if(iVariables == 0) ReturnFalseEx("variables is zero"); if(iSamples == 0 || iSamples > 64) ReturnFalseEx("samples outside [1, 64]"); if(iReferenceCapacity == 0) ReturnFalseEx("reference capacity is zero"); if(iCurrentCapacity == 0) ReturnFalseEx("current capacity is zero"); if(!MathIsValidNumber(m_fTau) || m_fTau <= 0.0f) ReturnFalseEx("tau is nonfinite or non-positive"); if(!MathIsValidNumber(m_fDisagreementMult) || m_fDisagreementMult < 0.0f) ReturnFalseEx("disagreement multiplier is nonfinite or negative"); if(!MathIsValidNumber(m_fKLDMult) || m_fKLDMult < 0.0f) ReturnFalseEx("KL multiplier is nonfinite or negative"); if(!MathIsValidNumber(m_fAlphaPrior)) ReturnFalseEx("alpha prior is nonfinite"); if(!MathIsValidNumber(m_fAlphaMult) || m_fAlphaMult < 0.0f) ReturnFalseEx("alpha multiplier is nonfinite or negative"); if(!MathIsValidNumber(m_fMaxKL) || m_fMaxKL < 0.0f) ReturnFalseEx("maximum KL is nonfinite or negative"); //--- Reject wrapped runtime products before loading nested resources or buffers. uint numeric_total = 0; if(!ValidateRuntimeSizes(iDimension, iVariables, iSamples, iReferenceCapacity, iCurrentCapacity, numeric_total)) ReturnFalseEx("runtime tensor size"); //--- Require the persisted outer layer to match the validated OMPB output span. const int expected_outer_total = (int)((ulong)iVariables * (ulong)iDimension); CBufferFloat *outer_output = getOutput(); CBufferFloat *outer_gradient = getGradient(); if(!outer_output || !outer_gradient) ReturnFalseEx("outer buffers are null"); if(outer_output.Total() != expected_outer_total || outer_gradient.Total() != expected_outer_total) ReturnFalseEx("outer buffer shape mismatch"); if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalseEx("residual posterior"); //--- The nested posterior must share the OMPB variable, affine, and sample contract. if(cResidual.Variables() != iVariables || cResidual.Inputs() != iDimension || cResidual.Outputs() != iDimension || cResidual.Samples() != iSamples) ReturnFalseEx("residual posterior shape mismatch"); if(!LoadInsideLayer(file_handle, cAlpha.AsObject())) ReturnFalseEx("alpha parameters"); CBufferFloat *alpha_output = cAlpha.getOutput(); CBufferFloat *alpha_gradient = cAlpha.getGradient(); CBufferFloat *alpha_weights = cAlpha.getWeightsParams(); if(!alpha_output || !alpha_gradient || !alpha_weights) ReturnFalseEx("alpha buffers are null"); if(cAlpha.Neurons() != int(iVariables) || alpha_output.Total() != int(iVariables) || alpha_gradient.Total() != int(iVariables) || alpha_weights.Total() != int(2 * iVariables)) ReturnFalseEx("alpha logical shape mismatch"); //--- Reset diagnostics, invalidate old histories, and preserve safe bypass mode. eMode = OMPB_BYPASS; bSourceAnchor = false; iReferenceCount = 0; iCurrentCount = 0; m_fLastDisagreement = 0.0f; m_fLastKL = 0.0f; m_fLastAlphaPrior = 0.0f; iInvalidFallbacks = 0; iKLRejects = 0; //--- Recreate the deterministic gate, residual candidate, and input-gradient buffers. if(!cGate.Init(0, 0, OpenCL, iDimension * iVariables, optimization, iBatch)) ReturnFalseEx("gate buffer"); if(!cCandidate.BufferInit(iDimension * iVariables, 0.0f)) ReturnFalseEx("candidate host buffer"); if(!cCandidate.BufferCreate(OpenCL)) ReturnFalseEx("candidate device buffer"); if(!cInputGradient.BufferInit(iDimension * iVariables, 0.0f)) ReturnFalseEx("input gradient host buffer"); if(!cInputGradient.BufferCreate(OpenCL)) ReturnFalseEx("input gradient device buffer"); if(!InitializeAlphaRuntime()) ReturnFalseEx("alpha runtime buffers"); if(!InitializeNumericRuntime(numeric_total)) ReturnFalseEx("numeric runtime buffers"); //--- Recreate the fixed-size reference and current history owners. if(!cReference) cReference = new CNeuronAddToStack(); if(!cCurrent) cCurrent = new CNeuronAddToStack(); if(!cReference) ReturnFalseEx("reference history allocation"); if(!cCurrent) ReturnFalseEx("current history allocation"); if(!cReference.Init(0, 0, OpenCL, iReferenceCapacity, iDimension, iVariables, optimization, iBatch)) ReturnFalseEx("reference history"); if(!cCurrent.Init(0, 0, OpenCL, iCurrentCapacity, iDimension, iVariables, optimization, iBatch)) ReturnFalseEx("current history"); //--- Restore the canonical history snapshots directly; AddToStack::Load clears Output. CBufferFloat *reference_history = cReference.getOutput(); CBufferFloat *current_history = cCurrent.getOutput(); if(!reference_history || !current_history) ReturnFalseEx("history buffers are null"); const int reference_history_total = reference_history.Total(); const int current_history_total = current_history.Total(); uint reference_count = 0; uint current_count = 0; if(FileSize(file_handle) - FileTell(file_handle) < 2 * INT_VALUE) ReturnFalseEx("history counts are missing"); dFileReadUInt(file_handle, reference_count); dFileReadUInt(file_handle, current_count); if(reference_count > iReferenceCapacity) ReturnFalseEx("reference history count"); if(current_count > iCurrentCapacity) ReturnFalseEx("current history count"); if(!(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? reference_history.LoadInferenceFinite(file_handle, OpenCL) : reference_history.Load(file_handle))) ReturnFalseEx("reference history data"); if(!(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? current_history.LoadInferenceFinite(file_handle, OpenCL) : current_history.Load(file_handle))) ReturnFalseEx("current history data"); if(reference_history.Total() != reference_history_total) ReturnFalseEx("reference history shape"); if(current_history.Total() != current_history_total) ReturnFalseEx("current history shape"); //--- Recreate history transposes and posterior sample holders. if(!cReferenceHistoryTranspose.Init(0, 0, OpenCL, iVariables, iReferenceCapacity, iDimension, optimization, iBatch)) ReturnFalseEx("reference history transpose"); if(!cCurrentHistoryTranspose.Init(0, 0, OpenCL, iVariables, iCurrentCapacity, iDimension, optimization, iBatch)) ReturnFalseEx("current history transpose"); if(!cReferenceSamples.Init(0, 0, OpenCL, iSamples * iVariables * iReferenceCapacity * iDimension, optimization, iBatch)) ReturnFalseEx("reference posterior samples"); if(!cCurrentSamples.Init(0, 0, OpenCL, iSamples * iVariables * iCurrentCapacity * iDimension, optimization, iBatch)) ReturnFalseEx("current posterior samples"); if(!cReferenceSamplesTranspose.Init(0, 0, OpenCL, iSamples, iVariables * iReferenceCapacity, iDimension, optimization, iBatch)) ReturnFalseEx("reference samples transpose"); if(!cCurrentSamplesTranspose.Init(0, 0, OpenCL, iSamples, iVariables * iCurrentCapacity, iDimension, optimization, iBatch)) ReturnFalseEx("current samples transpose"); //--- Recreate host-side disagreement and KL metric buffers. if(!cReferenceDistance.BufferInit(iVariables * iReferenceCapacity * iSamples * iSamples, 0.0f)) ReturnFalseEx("reference distance host buffer"); if(!cCurrentDistance.BufferInit(iVariables * iCurrentCapacity * iSamples * iSamples, 0.0f)) ReturnFalseEx("current distance host buffer"); if(!cDisagreement.BufferInit(iVariables, 0.0f)) ReturnFalseEx("disagreement host buffer"); if(!cKLMuWeights.BufferInit(iVariables * iDimension * iDimension, 0.0f)) ReturnFalseEx("KL weight mean host buffer"); if(!cKLLogVarWeights.BufferInit(iVariables * iDimension * iDimension, 0.0f)) ReturnFalseEx("KL weight log variance host buffer"); if(!cKLMuBias.BufferInit(iVariables * iDimension, 0.0f)) ReturnFalseEx("KL bias mean host buffer"); if(!cKLLogVarBias.BufferInit(iVariables * iDimension, 0.0f)) ReturnFalseEx("KL bias log variance host buffer"); if(!cKLMu.BufferInit(iVariables * iDimension * (iDimension + 1), 0.0f)) ReturnFalseEx("KL mean host buffer"); if(!cKLLogVar.BufferInit(iVariables * iDimension * (iDimension + 1), 0.0f)) ReturnFalseEx("KL log variance host buffer"); if(!cKLValues.BufferInit(iVariables * iDimension * (iDimension + 1), 0.0f)) ReturnFalseEx("KL values host buffer"); //--- Create the OpenCL buffers for disagreement and KL diagnostics. if(!cReferenceDistance.BufferCreate(OpenCL)) ReturnFalseEx("reference distance device buffer"); if(!cCurrentDistance.BufferCreate(OpenCL)) ReturnFalseEx("current distance device buffer"); if(!cDisagreement.BufferCreate(OpenCL)) ReturnFalseEx("disagreement device buffer"); if(!cKLMuWeights.BufferCreate(OpenCL)) ReturnFalseEx("KL weight mean device buffer"); if(!cKLLogVarWeights.BufferCreate(OpenCL)) ReturnFalseEx("KL weight log variance device buffer"); if(!cKLMuBias.BufferCreate(OpenCL)) ReturnFalseEx("KL bias mean device buffer"); if(!cKLLogVarBias.BufferCreate(OpenCL)) ReturnFalseEx("KL bias log variance device buffer"); if(!cKLMu.BufferCreate(OpenCL)) ReturnFalseEx("KL mean device buffer"); if(!cKLLogVar.BufferCreate(OpenCL)) ReturnFalseEx("KL log variance device buffer"); if(!cKLValues.BufferCreate(OpenCL)) ReturnFalseEx("KL values device buffer"); //--- Finalize posterior regularization and reconstruct the deterministic gate. cResidual.SetKLDMult(m_fKLDMult); if(!PrepareGate()) ReturnFalseEx("prepare gate"); //--- Publish durable history counts only after the complete Load has succeeded. iReferenceCount = reference_count; iCurrentCount = current_count; return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronOMPBOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronOMPBOCL::SetOpenCL(COpenCLMy *obj) { //--- A context rebind invalidates permission to update from an older forward. bForwardAccepted = false; bNumericRejected = false; bCurrentTransactionPending = false; iCurrentTransactionCount = 0; CNeuronBaseOCL::SetOpenCL(obj); cResidual.SetOpenCL(OpenCL); cAlpha.SetOpenCL(OpenCL); cGate.SetOpenCL(OpenCL); if(cReference) cReference.SetOpenCL(OpenCL); if(cCurrent) cCurrent.SetOpenCL(OpenCL); cReferenceHistoryTranspose.SetOpenCL(OpenCL); cCurrentHistoryTranspose.SetOpenCL(OpenCL); cReferenceSamples.SetOpenCL(OpenCL); cCurrentSamples.SetOpenCL(OpenCL); cReferenceSamplesTranspose.SetOpenCL(OpenCL); cCurrentSamplesTranspose.SetOpenCL(OpenCL); if(cCandidate.Total() > 0) cCandidate.BufferCreate(OpenCL); if(cInputGradient.Total() > 0) cInputGradient.BufferCreate(OpenCL); if(cAlphaGradient.Total() > 0) cAlphaGradient.BufferCreate(OpenCL); if(cAlphaDisagreement.Total() > 0) cAlphaDisagreement.BufferCreate(OpenCL); if(cAlphaOnes.Total() > 0) cAlphaOnes.BufferCreate(OpenCL); if(cAlphaPriorOffset.Total() > 0) cAlphaPriorOffset.BufferCreate(OpenCL); if(cAlphaDelta.Total() > 0) cAlphaDelta.BufferCreate(OpenCL); if(cAlphaPriorSum.Total() > 0) cAlphaPriorSum.BufferCreate(OpenCL); if(cNumericFlags.Total() > 0) cNumericFlags.BufferCreate(OpenCL); if(cNumericOnes.Total() > 0) cNumericOnes.BufferCreate(OpenCL); if(cNumericScalar.Total() > 0) cNumericScalar.BufferCreate(OpenCL); if(cKLMetricScalar.Total() > 0) cKLMetricScalar.BufferCreate(OpenCL); if(cDisagreementPositive.Total() > 0) cDisagreementPositive.BufferCreate(OpenCL); if(cDisagreementNegative.Total() > 0) cDisagreementNegative.BufferCreate(OpenCL); if(cDisagreementMetricScalar.Total() > 0) cDisagreementMetricScalar.BufferCreate(OpenCL); if(cReferenceDistance.Total() > 0) cReferenceDistance.BufferCreate(OpenCL); if(cCurrentDistance.Total() > 0) cCurrentDistance.BufferCreate(OpenCL); if(cDisagreement.Total() > 0) cDisagreement.BufferCreate(OpenCL); if(cKLMuWeights.Total() > 0) cKLMuWeights.BufferCreate(OpenCL); if(cKLLogVarWeights.Total() > 0) cKLLogVarWeights.BufferCreate(OpenCL); if(cKLMuBias.Total() > 0) cKLMuBias.BufferCreate(OpenCL); if(cKLLogVarBias.Total() > 0) cKLLogVarBias.BufferCreate(OpenCL); if(cKLMu.Total() > 0) cKLMu.BufferCreate(OpenCL); if(cKLLogVar.Total() > 0) cKLLogVar.BufferCreate(OpenCL); if(cKLValues.Total() > 0) cKLValues.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Checks checked-transfer support of the full OMPB ownership tree. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SupportsOpenCLChecked(void) const { //--- Validate the target identity and every history pointer before mutation. if(Type() != defNeuronOMPBOCL || CheckPointer(cReference) == POINTER_INVALID || CheckPointer(cCurrent) == POINTER_INVALID) return(false); //--- Validate every nested owner that will transfer before base publication. if(!cResidual.SupportsOpenCLChecked() || !cAlpha.SupportsOpenCLChecked() || !cGate.SupportsOpenCLChecked() || !cReference.SupportsOpenCLChecked() || !cCurrent.SupportsOpenCLChecked() || !cReferenceHistoryTranspose.SupportsOpenCLChecked() || !cCurrentHistoryTranspose.SupportsOpenCLChecked() || !cReferenceSamples.SupportsOpenCLChecked() || !cCurrentSamples.SupportsOpenCLChecked() || !cReferenceSamplesTranspose.SupportsOpenCLChecked() || !cCurrentSamplesTranspose.SupportsOpenCLChecked()) return(false); //--- Finalize only when the complete embedded hierarchy is transferable. return(true); } //+------------------------------------------------------------------+ //| Transfers OMPB children and buffers before inherited | //| publication. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and complete ownership tree before state changes. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or tree"); //--- A valid attempt invalidates outcomes tied to the previous forward pass. bForwardAccepted = false; bNumericRejected = false; bCurrentTransactionPending = false; iCurrentTransactionCount = 0; //--- Transfer the complete nested hierarchy before direct and base owners. if(!cResidual.SetOpenCLChecked(obj)) ReturnFalseEx("residual context"); if(!cAlpha.SetOpenCLChecked(obj)) ReturnFalseEx("alpha context"); if(!cGate.SetOpenCLChecked(obj)) ReturnFalseEx("gate context"); if(!cReference.SetOpenCLChecked(obj)) ReturnFalseEx("reference history context"); if(!cCurrent.SetOpenCLChecked(obj)) ReturnFalseEx("current history context"); if(!cReferenceHistoryTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("reference history transpose context"); if(!cCurrentHistoryTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("current history transpose context"); if(!cReferenceSamples.SetOpenCLChecked(obj)) ReturnFalseEx("reference samples context"); if(!cCurrentSamples.SetOpenCLChecked(obj)) ReturnFalseEx("current samples context"); if(!cReferenceSamplesTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("reference samples transpose context"); if(!cCurrentSamplesTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("current samples transpose context"); //--- Read foreign allocated direct buffers before their target binding changes. if(cCandidate.Total() > 0 && cCandidate.GetIndex() >= 0 && cCandidate.GetOpenCL() != obj && !cCandidate.BufferRead()) ReturnFalseEx("candidate read"); if(cInputGradient.Total() > 0 && cInputGradient.GetIndex() >= 0 && cInputGradient.GetOpenCL() != obj && !cInputGradient.BufferRead()) ReturnFalseEx("input gradient read"); if(cAlphaGradient.Total() > 0 && cAlphaGradient.GetIndex() >= 0 && cAlphaGradient.GetOpenCL() != obj && !cAlphaGradient.BufferRead()) ReturnFalseEx("alpha gradient read"); if(cAlphaDisagreement.Total() > 0 && cAlphaDisagreement.GetIndex() >= 0 && cAlphaDisagreement.GetOpenCL() != obj && !cAlphaDisagreement.BufferRead()) ReturnFalseEx("alpha disagreement read"); if(cAlphaOnes.Total() > 0 && cAlphaOnes.GetIndex() >= 0 && cAlphaOnes.GetOpenCL() != obj && !cAlphaOnes.BufferRead()) ReturnFalseEx("alpha ones read"); if(cAlphaPriorOffset.Total() > 0 && cAlphaPriorOffset.GetIndex() >= 0 && cAlphaPriorOffset.GetOpenCL() != obj && !cAlphaPriorOffset.BufferRead()) ReturnFalseEx("alpha prior offset read"); if(cAlphaDelta.Total() > 0 && cAlphaDelta.GetIndex() >= 0 && cAlphaDelta.GetOpenCL() != obj && !cAlphaDelta.BufferRead()) ReturnFalseEx("alpha delta read"); if(cAlphaPriorSum.Total() > 0 && cAlphaPriorSum.GetIndex() >= 0 && cAlphaPriorSum.GetOpenCL() != obj && !cAlphaPriorSum.BufferRead()) ReturnFalseEx("alpha prior sum read"); if(cNumericFlags.Total() > 0 && cNumericFlags.GetIndex() >= 0 && cNumericFlags.GetOpenCL() != obj && !cNumericFlags.BufferRead()) ReturnFalseEx("numeric flags read"); if(cNumericOnes.Total() > 0 && cNumericOnes.GetIndex() >= 0 && cNumericOnes.GetOpenCL() != obj && !cNumericOnes.BufferRead()) ReturnFalseEx("numeric ones read"); if(cNumericScalar.Total() > 0 && cNumericScalar.GetIndex() >= 0 && cNumericScalar.GetOpenCL() != obj && !cNumericScalar.BufferRead()) ReturnFalseEx("numeric scalar read"); if(cKLMetricScalar.Total() > 0 && cKLMetricScalar.GetIndex() >= 0 && cKLMetricScalar.GetOpenCL() != obj && !cKLMetricScalar.BufferRead()) ReturnFalseEx("KL metric scalar read"); if(cDisagreementPositive.Total() > 0 && cDisagreementPositive.GetIndex() >= 0 && cDisagreementPositive.GetOpenCL() != obj && !cDisagreementPositive.BufferRead()) ReturnFalseEx("disagreement positive read"); if(cDisagreementNegative.Total() > 0 && cDisagreementNegative.GetIndex() >= 0 && cDisagreementNegative.GetOpenCL() != obj && !cDisagreementNegative.BufferRead()) ReturnFalseEx("disagreement negative read"); if(cDisagreementMetricScalar.Total() > 0 && cDisagreementMetricScalar.GetIndex() >= 0 && cDisagreementMetricScalar.GetOpenCL() != obj && !cDisagreementMetricScalar.BufferRead()) ReturnFalseEx("disagreement metric scalar read"); if(cReferenceDistance.Total() > 0 && cReferenceDistance.GetIndex() >= 0 && cReferenceDistance.GetOpenCL() != obj && !cReferenceDistance.BufferRead()) ReturnFalseEx("reference distance read"); if(cCurrentDistance.Total() > 0 && cCurrentDistance.GetIndex() >= 0 && cCurrentDistance.GetOpenCL() != obj && !cCurrentDistance.BufferRead()) ReturnFalseEx("current distance read"); if(cDisagreement.Total() > 0 && cDisagreement.GetIndex() >= 0 && cDisagreement.GetOpenCL() != obj && !cDisagreement.BufferRead()) ReturnFalseEx("disagreement read"); if(cKLMuWeights.Total() > 0 && cKLMuWeights.GetIndex() >= 0 && cKLMuWeights.GetOpenCL() != obj && !cKLMuWeights.BufferRead()) ReturnFalseEx("KL mu weights read"); if(cKLLogVarWeights.Total() > 0 && cKLLogVarWeights.GetIndex() >= 0 && cKLLogVarWeights.GetOpenCL() != obj && !cKLLogVarWeights.BufferRead()) ReturnFalseEx("KL log variance weights read"); if(cKLMuBias.Total() > 0 && cKLMuBias.GetIndex() >= 0 && cKLMuBias.GetOpenCL() != obj && !cKLMuBias.BufferRead()) ReturnFalseEx("KL mu bias read"); if(cKLLogVarBias.Total() > 0 && cKLLogVarBias.GetIndex() >= 0 && cKLLogVarBias.GetOpenCL() != obj && !cKLLogVarBias.BufferRead()) ReturnFalseEx("KL log variance bias read"); if(cKLMu.Total() > 0 && cKLMu.GetIndex() >= 0 && cKLMu.GetOpenCL() != obj && !cKLMu.BufferRead()) ReturnFalseEx("KL mu read"); if(cKLLogVar.Total() > 0 && cKLLogVar.GetIndex() >= 0 && cKLLogVar.GetOpenCL() != obj && !cKLLogVar.BufferRead()) ReturnFalseEx("KL log variance read"); if(cKLValues.Total() > 0 && cKLValues.GetIndex() >= 0 && cKLValues.GetOpenCL() != obj && !cKLValues.BufferRead()) ReturnFalseEx("KL values read"); //--- Bind only direct buffers not already allocated on the target context. if(cCandidate.Total() > 0 && (cCandidate.GetOpenCL() != obj || cCandidate.GetIndex() < 0) && !cCandidate.BufferCreate(obj)) ReturnFalseEx("candidate create"); if(cInputGradient.Total() > 0 && (cInputGradient.GetOpenCL() != obj || cInputGradient.GetIndex() < 0) && !cInputGradient.BufferCreate(obj)) ReturnFalseEx("input gradient create"); if(cAlphaGradient.Total() > 0 && (cAlphaGradient.GetOpenCL() != obj || cAlphaGradient.GetIndex() < 0) && !cAlphaGradient.BufferCreate(obj)) ReturnFalseEx("alpha gradient create"); if(cAlphaDisagreement.Total() > 0 && (cAlphaDisagreement.GetOpenCL() != obj || cAlphaDisagreement.GetIndex() < 0) && !cAlphaDisagreement.BufferCreate(obj)) ReturnFalseEx("alpha disagreement create"); if(cAlphaOnes.Total() > 0 && (cAlphaOnes.GetOpenCL() != obj || cAlphaOnes.GetIndex() < 0) && !cAlphaOnes.BufferCreate(obj)) ReturnFalseEx("alpha ones create"); if(cAlphaPriorOffset.Total() > 0 && (cAlphaPriorOffset.GetOpenCL() != obj || cAlphaPriorOffset.GetIndex() < 0) && !cAlphaPriorOffset.BufferCreate(obj)) ReturnFalseEx("alpha prior offset create"); if(cAlphaDelta.Total() > 0 && (cAlphaDelta.GetOpenCL() != obj || cAlphaDelta.GetIndex() < 0) && !cAlphaDelta.BufferCreate(obj)) ReturnFalseEx("alpha delta create"); if(cAlphaPriorSum.Total() > 0 && (cAlphaPriorSum.GetOpenCL() != obj || cAlphaPriorSum.GetIndex() < 0) && !cAlphaPriorSum.BufferCreate(obj)) ReturnFalseEx("alpha prior sum create"); if(cNumericFlags.Total() > 0 && (cNumericFlags.GetOpenCL() != obj || cNumericFlags.GetIndex() < 0) && !cNumericFlags.BufferCreate(obj)) ReturnFalseEx("numeric flags create"); if(cNumericOnes.Total() > 0 && (cNumericOnes.GetOpenCL() != obj || cNumericOnes.GetIndex() < 0) && !cNumericOnes.BufferCreate(obj)) ReturnFalseEx("numeric ones create"); if(cNumericScalar.Total() > 0 && (cNumericScalar.GetOpenCL() != obj || cNumericScalar.GetIndex() < 0) && !cNumericScalar.BufferCreate(obj)) ReturnFalseEx("numeric scalar create"); if(cKLMetricScalar.Total() > 0 && (cKLMetricScalar.GetOpenCL() != obj || cKLMetricScalar.GetIndex() < 0) && !cKLMetricScalar.BufferCreate(obj)) ReturnFalseEx("KL metric scalar create"); if(cDisagreementPositive.Total() > 0 && (cDisagreementPositive.GetOpenCL() != obj || cDisagreementPositive.GetIndex() < 0) && !cDisagreementPositive.BufferCreate(obj)) ReturnFalseEx("disagreement positive create"); if(cDisagreementNegative.Total() > 0 && (cDisagreementNegative.GetOpenCL() != obj || cDisagreementNegative.GetIndex() < 0) && !cDisagreementNegative.BufferCreate(obj)) ReturnFalseEx("disagreement negative create"); if(cDisagreementMetricScalar.Total() > 0 && (cDisagreementMetricScalar.GetOpenCL() != obj || cDisagreementMetricScalar.GetIndex() < 0) && !cDisagreementMetricScalar.BufferCreate(obj)) ReturnFalseEx("disagreement metric scalar create"); if(cReferenceDistance.Total() > 0 && (cReferenceDistance.GetOpenCL() != obj || cReferenceDistance.GetIndex() < 0) && !cReferenceDistance.BufferCreate(obj)) ReturnFalseEx("reference distance create"); if(cCurrentDistance.Total() > 0 && (cCurrentDistance.GetOpenCL() != obj || cCurrentDistance.GetIndex() < 0) && !cCurrentDistance.BufferCreate(obj)) ReturnFalseEx("current distance create"); if(cDisagreement.Total() > 0 && (cDisagreement.GetOpenCL() != obj || cDisagreement.GetIndex() < 0) && !cDisagreement.BufferCreate(obj)) ReturnFalseEx("disagreement create"); if(cKLMuWeights.Total() > 0 && (cKLMuWeights.GetOpenCL() != obj || cKLMuWeights.GetIndex() < 0) && !cKLMuWeights.BufferCreate(obj)) ReturnFalseEx("KL mu weights create"); if(cKLLogVarWeights.Total() > 0 && (cKLLogVarWeights.GetOpenCL() != obj || cKLLogVarWeights.GetIndex() < 0) && !cKLLogVarWeights.BufferCreate(obj)) ReturnFalseEx("KL log variance weights create"); if(cKLMuBias.Total() > 0 && (cKLMuBias.GetOpenCL() != obj || cKLMuBias.GetIndex() < 0) && !cKLMuBias.BufferCreate(obj)) ReturnFalseEx("KL mu bias create"); if(cKLLogVarBias.Total() > 0 && (cKLLogVarBias.GetOpenCL() != obj || cKLLogVarBias.GetIndex() < 0) && !cKLLogVarBias.BufferCreate(obj)) ReturnFalseEx("KL log variance bias create"); if(cKLMu.Total() > 0 && (cKLMu.GetOpenCL() != obj || cKLMu.GetIndex() < 0) && !cKLMu.BufferCreate(obj)) ReturnFalseEx("KL mu create"); if(cKLLogVar.Total() > 0 && (cKLLogVar.GetOpenCL() != obj || cKLLogVar.GetIndex() < 0) && !cKLLogVar.BufferCreate(obj)) ReturnFalseEx("KL log variance create"); if(cKLValues.Total() > 0 && (cKLValues.GetOpenCL() != obj || cKLValues.GetIndex() < 0) && !cKLValues.BufferCreate(obj)) ReturnFalseEx("KL values create"); //--- Publish inherited output and gradient owners only after every child succeeded. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("inherited base context"); //--- Finalize the checked context adoption after every owner succeeded. return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronOMPBOCL layer. | //+------------------------------------------------------------------+ void CNeuronOMPBOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cResidual.TrainMode(flag); cAlpha.TrainMode(flag); cGate.TrainMode(flag); cReferenceHistoryTranspose.TrainMode(flag); cCurrentHistoryTranspose.TrainMode(flag); cReferenceSamples.TrainMode(flag); cCurrentSamples.TrainMode(flag); cReferenceSamplesTranspose.TrainMode(flag); cCurrentSamplesTranspose.TrainMode(flag); } //+------------------------------------------------------------------+ //| Clears the CNeuronOMPBOCL execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::Clear(void) { //--- A partial clear cannot retain permission to update from an older forward. bForwardAccepted = false; bNumericRejected = false; bCurrentTransactionPending = false; iCurrentTransactionCount = 0; //--- Reference history is durable across inference and session clears. iCurrentCount = 0; m_fLastDisagreement = 0.0f; m_fLastKL = 0.0f; m_fLastAlphaPrior = 0.0f; bSourceAnchor = false; //--- Clear transient OMPB layer and target-history state. if(!CNeuronBaseOCL::Clear()) ReturnFalseEx("base neuron"); if(!cResidual.Clear()) ReturnFalseEx("residual posterior"); if(!cAlpha.Clear()) ReturnFalseEx("alpha parameters"); if(!cGate.Clear()) ReturnFalseEx("gate buffer"); if(cCurrent && !cCurrent.Clear()) ReturnFalseEx("current history"); //--- Clear derived histories while retaining the durable raw Reference. if(!cReferenceHistoryTranspose.Clear()) ReturnFalseEx("reference history transpose"); if(!cReferenceSamples.Clear()) ReturnFalseEx("reference samples"); if(!cReferenceSamplesTranspose.Clear()) ReturnFalseEx("reference samples transpose"); if(!cReferenceDistance.Fill(0.0f)) ReturnFalseEx("reference distance"); if(!cCandidate.Fill(0.0f)) ReturnFalseEx("candidate"); if(!cInputGradient.Fill(0.0f)) ReturnFalseEx("input gradient"); if(!cAlphaGradient.Fill(0.0f)) ReturnFalseEx("alpha feature gradient"); if(!cAlphaDisagreement.Fill(0.0f)) ReturnFalseEx("alpha disagreement"); if(!cAlphaDelta.Fill(0.0f)) ReturnFalseEx("alpha prior delta"); if(!cAlphaPriorSum.Fill(0.0f)) ReturnFalseEx("alpha prior sum"); if(!cNumericFlags.Fill(0.0f)) ReturnFalseEx("numeric flags"); if(!cNumericOnes.Fill(1.0f)) ReturnFalseEx("numeric ones"); if(!cNumericScalar.Fill(0.0f)) ReturnFalseEx("numeric scalar"); if(!cKLMetricScalar.Fill(0.0f)) ReturnFalseEx("KL metric scalar"); if(!cDisagreementPositive.Fill(0.0f)) ReturnFalseEx("disagreement positive"); if(!cDisagreementNegative.Fill(0.0f)) ReturnFalseEx("disagreement negative"); if(!cDisagreementMetricScalar.Fill(0.0f)) ReturnFalseEx("disagreement metric scalar"); if(!cCurrentHistoryTranspose.Clear()) ReturnFalseEx("current history transpose"); if(!cCurrentSamples.Clear()) ReturnFalseEx("current samples"); if(!cCurrentSamplesTranspose.Clear()) ReturnFalseEx("current samples transpose"); if(!cCurrentDistance.Fill(0.0f)) ReturnFalseEx("current distance"); if(!cDisagreement.Fill(0.0f)) ReturnFalseEx("disagreement"); if(!cKLMuWeights.Fill(0.0f)) ReturnFalseEx("KL weight mean"); if(!cKLLogVarWeights.Fill(0.0f)) ReturnFalseEx("KL weight log variance"); if(!cKLMuBias.Fill(0.0f)) ReturnFalseEx("KL bias mean"); if(!cKLLogVarBias.Fill(0.0f)) ReturnFalseEx("KL bias log variance"); if(!cKLMu.Fill(0.0f)) ReturnFalseEx("KL mean"); if(!cKLLogVar.Fill(0.0f)) ReturnFalseEx("KL log variance"); if(!cKLValues.Fill(0.0f)) ReturnFalseEx("KL values"); return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronOMPBOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source) ReturnFalseEx("source layer is null"); if(source.Type() != Type()) ReturnFalseEx("source layer type mismatch"); CNeuronOMPBOCL *Source = source; if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau)) ReturnFalseEx("residual posterior"); if(!cAlpha.WeightsUpdate(Source.cAlpha.AsObject(), tau)) ReturnFalseEx("alpha parameters"); return(true); } //+------------------------------------------------------------------+ //| Appends the OMPB parameter fingerprint. | //+------------------------------------------------------------------+ bool CNeuronOMPBOCL::AppendParameterFingerprint(ulong &fingerprint) { fingerprint = (fingerprint ^ ulong(Type())) * ulong(1099511628211); if(!cResidual.AppendParameterFingerprint(fingerprint)) ReturnFalseEx("residual posterior"); if(!cAlpha.AppendParameterFingerprint(fingerprint)) ReturnFalseEx("alpha parameters"); return(true); } //+------------------------------------------------------------------+ //| Returns the layer info of the CNeuronOMPBOCL. | //+------------------------------------------------------------------+ CLayerDescription *CNeuronOMPBOCL::GetLayerInfo(void) { CLayerDescription *result = CNeuronBaseOCL::GetLayerInfo(); if(result) { result.window = iDimension; result.count = iVariables; result.layers = iSamples; ArrayResize(result.units, 2); result.units[0] = iReferenceCapacity; result.units[1] = iCurrentCapacity; result.batch = iBatch; } return(result); } //+------------------------------------------------------------------+ //| Logical method implementations generated from NeuroNet.mqh: | //| TimeSeries. | //+------------------------------------------------------------------+ //| Initializes the CNeuronTiDEOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint history, uint forecast, uint variables, uint features, uint & encoder_decoder[], ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- iHistory = MathMax(history, 1); iForecast = forecast; iVariables = variables; iFeatures = MathMax(features, 1); //--- if(!acFeatureProjection[0].Init(0, 0, OpenCL, iFeatures, iHistory * iVariables, 1, optimization, iBatch)) ReturnFalse; if(!acFeatureProjection[1].Init(0, 1, OpenCL, iFeatures, iForecast * iVariables, 1, optimization, iBatch)) ReturnFalse; //--- int total = ArraySize(encoder_decoder); if(ArrayResize(acEncoderDecoder, total + 1) < total + 1) ReturnFalse; if(total == 0) { if(!acEncoderDecoder[0].Init(0, 2, OpenCL, 2 * iHistory, iForecast, iVariables, optimization, iBatch)) ReturnFalse; } else { if(!acEncoderDecoder[0].Init(0, 2, OpenCL, 2 * iHistory, encoder_decoder[0], iVariables, optimization, iBatch)) ReturnFalse; for(int i = 1; i < total; i++) if(!acEncoderDecoder[i].Init(0, i + 2, OpenCL, encoder_decoder[i - 1], encoder_decoder[i], iVariables, optimization, iBatch)) ReturnFalse; if(!acEncoderDecoder[total].Init(0, total + 2, OpenCL, encoder_decoder[total - 1], iForecast, iVariables, optimization, iBatch)) ReturnFalse; } //--- if(!cGlobalResidual.Init(0, total + 3, OpenCL, iHistory, iHistory, iForecast, iVariables, optimization, iBatch)) ReturnFalse; cGlobalResidual.SetActivationFunction(TANH); if(!cTemporalDecoder.Init(0, total + 4, OpenCL, 2 * iForecast, iForecast, iVariables, optimization, iBatch)) ReturnFalse; //--- if(!cHistoryInput.Init(0, total + 5, OpenCL, iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cFeatureInput.Init(0, total + 6, OpenCL, iFeatures, optimization, iBatch)) ReturnFalse; if(!cEncoderInput.Init(0, total + 7, OpenCL, 2 * iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cTemporalDecoderInput.Init(0, total + 8, OpenCL, 2 * iForecast * iVariables, optimization, iBatch)) ReturnFalse; //--- if(cGlobalResidual.getGradient() != Gradient) if(!cGlobalResidual.SetGradient(Gradient)) ReturnFalse; if(cTemporalDecoder.getGradient() != getGradient()) if(!cTemporalDecoder.SetGradient(Gradient)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTiDEOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(cHistoryInput.getOutputIndex() != NeuronOCL.getOutputIndex()) { CBufferFloat *temp = cHistoryInput.getOutput(); if(!temp.BufferSet(NeuronOCL.getOutputIndex())) ReturnFalse; } if(cFeatureInput.getOutputIndex() != SecondInput.GetIndex()) { CBufferFloat *temp = cFeatureInput.getOutput(); if(!temp.BufferSet(SecondInput.GetIndex())) ReturnFalse; } //--- if(!cGlobalResidual.FeedForward(NeuronOCL)) ReturnFalse; if(!acFeatureProjection[0].FeedForward(NeuronOCL)) ReturnFalse; if(!acFeatureProjection[1].FeedForward(cFeatureInput.AsObject())) ReturnFalse; //--- if(!Concat(NeuronOCL.getOutput(), acFeatureProjection[0].getOutput(), cEncoderInput.getOutput(), iHistory, iHistory, iVariables)) ReturnFalse; uint total = acEncoderDecoder.Size(); CNeuronBaseOCL *prev = cEncoderInput.AsObject(); for(uint i = 0; i < total; i++) { if(!acEncoderDecoder[i].FeedForward(prev)) ReturnFalse; prev = acEncoderDecoder[i].AsObject(); } //--- if(!Concat(prev.getOutput(), acFeatureProjection[1].getOutput(), cTemporalDecoderInput.getOutput(), iForecast, iForecast, iVariables)) ReturnFalse; if(!cTemporalDecoder.FeedForward(cTemporalDecoderInput.AsObject())) ReturnFalse; if(!SumAndNormalize(cGlobalResidual.getOutput(), cTemporalDecoder.getOutput(), Output, iForecast, true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronTiDEOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!cTemporalDecoderInput.CalcHiddenGradients(cTemporalDecoder.AsObject())) ReturnFalse; //--- int total = (int)acEncoderDecoder.Size(); if(!DeConcat(acEncoderDecoder[total - 1].getGradient(), acFeatureProjection[1].getGradient(), cTemporalDecoderInput.getGradient(), iForecast, iForecast, iVariables)) ReturnFalse; for(int i = total - 2; i >= 0; i--) if(!acEncoderDecoder[i].CalcHiddenGradients(acEncoderDecoder[i + 1].AsObject())) ReturnFalse; if(!cEncoderInput.CalcHiddenGradients(acEncoderDecoder[0].AsObject())) ReturnFalse; //--- if(!DeConcat(cHistoryInput.getGradient(), acFeatureProjection[0].getGradient(), cEncoderInput.getGradient(), iHistory, iHistory, iVariables)) ReturnFalse; //--- if(cGlobalResidual.Activation() != None) { if(!DeActivation(cGlobalResidual.getOutput(), cGlobalResidual.getGradient(), cGlobalResidual.getGradient(), cGlobalResidual.Activation())) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(cGlobalResidual.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(cHistoryInput.getOutput(), cHistoryInput.getGradient(), cHistoryInput.getGradient(), SecondActivation)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cHistoryInput.getGradient(), NeuronOCL.getGradient(), iHistory, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cFeatureInput.CalcHiddenGradients(acFeatureProjection[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFeatureInput.getGradient(), cFeatureInput.getGradient(), SecondGradient, iFeatures, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!cFeatureInput.CalcHiddenGradients(acFeatureProjection[1].AsObject())) ReturnFalse; if(!SumAndNormalize(SecondGradient, cFeatureInput.getGradient(), SecondGradient, iFeatures, false, 0, 0, 0, 1.0f)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTiDEOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- if(!cGlobalResidual.UpdateInputWeights(cHistoryInput.AsObject())) ReturnFalse; if(!acFeatureProjection[0].UpdateInputWeights(cHistoryInput.AsObject())) ReturnFalse; if(!acFeatureProjection[1].UpdateInputWeights(cFeatureInput.AsObject())) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); CNeuronBaseOCL *prev = cEncoderInput.AsObject(); for(uint i = 0; i < total; i++) { if(!acEncoderDecoder[i].UpdateInputWeights(prev)) ReturnFalse; prev = acEncoderDecoder[i].AsObject(); } //--- if(!cTemporalDecoder.UpdateInputWeights(cTemporalDecoderInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTiDEOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iHistory, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iForecast, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFeatures, INT_VALUE) < INT_VALUE) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); if(FileWriteInteger(file_handle, (int)total, INT_VALUE) < INT_VALUE) ReturnFalse; for(uint i = 0; i < total; i++) if(!acEncoderDecoder[i].Save(file_handle)) ReturnFalse; if(!cGlobalResidual.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!acFeatureProjection[i].Save(file_handle)) ReturnFalse; if(!cTemporalDecoder.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTiDEOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFeatures = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; //--- int total = FileReadInteger(file_handle); int prev_size = (int)acEncoderDecoder.Size(); if(prev_size != total) if(ArrayResize(acEncoderDecoder, total) < total) ReturnFalse; for(int i = 0; i < total; i++) { if(i >= prev_size) if(!acEncoderDecoder[i].Init(0, i + 2, OpenCL, 1, 1, 1, ADAM, 1)) ReturnFalse; if(!LoadInsideLayer(file_handle, acEncoderDecoder[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cGlobalResidual.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, acFeatureProjection[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalDecoder.AsObject())) ReturnFalse; //--- if(!cHistoryInput.Init(0, total + 5, OpenCL, iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cFeatureInput.Init(0, total + 6, OpenCL, iFeatures, optimization, iBatch)) ReturnFalse; if(!cEncoderInput.Init(0, total + 7, OpenCL, 2 * iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cTemporalDecoderInput.Init(0, total + 8, OpenCL, 2 * iForecast * iVariables, optimization, iBatch)) ReturnFalse; //--- if(cGlobalResidual.getGradient() != Gradient) if(!cGlobalResidual.SetGradient(Gradient)) ReturnFalse; if(cTemporalDecoder.getGradient() != getGradient()) if(!cTemporalDecoder.SetGradient(Gradient)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTiDEOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTiDEOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- uint total = acEncoderDecoder.Size(); for(uint i = 0; i < total; i++) acEncoderDecoder[i].SetOpenCL(OpenCL); cGlobalResidual.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) acFeatureProjection[i].SetOpenCL(OpenCL); cTemporalDecoder.SetOpenCL(OpenCL); //--- cHistoryInput.SetOpenCL(OpenCL); cFeatureInput.SetOpenCL(OpenCL); cEncoderInput.SetOpenCL(OpenCL); cTemporalDecoderInput.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Averages the CNeuronTiDEOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { //--- if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTiDEOCL *Source = source; if(!cGlobalResidual.WeightsUpdate(Source.cGlobalResidual.AsObject(), tau)) ReturnFalse; if(!acFeatureProjection[0].WeightsUpdate(Source.acFeatureProjection[0].AsObject(), tau)) ReturnFalse; if(!acFeatureProjection[1].WeightsUpdate(Source.acFeatureProjection[1].AsObject(), tau)) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); for(uint i = 0; i < total; i++) if(!acEncoderDecoder[i].WeightsUpdate(Source.acEncoderDecoder[i].AsObject(), tau)) ReturnFalse; //--- if(!cTemporalDecoder.WeightsUpdate(Source.cTemporalDecoder.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronFEDW layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronFEDW::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; iFilters = MathMax(filters, 1); //--- if(!cWavlets.Init(0, 0, OpenCL, iWindow, iWindow, iCount, iFilters, optimization, iBatch)) ReturnFalse; cWavlets.SetActivationFunction(None); if(!cNorm.Init(0, 1, OpenCL, iFilters * iCount, 1000, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); if(!cSoftMax.Init(0, 1, OpenCL, iFilters * iCount, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iCount); cSoftMax.SetActivationFunction(None); if(!cReconstruct.Init(0, 2, OpenCL, iWindow, optimization, iBatch)) ReturnFalse; cReconstruct.SetActivationFunction(None); if(!cFF[0].Init(0, 3, OpenCL, iWindow, iWindow, 4 * iWindow, iCount, optimization, iBatch)) ReturnFalse; cFF[0].SetActivationFunction(LReLU); if(!cFF[1].Init(0, 4, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iCount, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- if(Gradient != cFF[1].getGradient()) SetGradient(cFF[1].getGradient()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronFEDW feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronFEDW::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cWavlets.FeedForward(NeuronOCL.AsObject())) ReturnFalse; if(!cNorm.FeedForward(cWavlets.AsObject())) ReturnFalse; if(!cSoftMax.FeedForward(cNorm.AsObject())) ReturnFalse; if(!Reconsruct(cReconstruct.getOutput(), cSoftMax.getOutput())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cReconstruct.getOutput(), cReconstruct.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFF[0].FeedForward(cReconstruct.AsObject())) ReturnFalse; if(!cFF[1].FeedForward(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFF[1].getOutput(), cReconstruct.getOutput(), getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the reconsruct step of the CNeuronFEDW layer. | //+------------------------------------------------------------------+ bool CNeuronFEDW::Reconsruct(CBufferFloat* sequence, CBufferFloat* probability) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = sequence.Total(); global_work_size[1] = 1; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, cWavlets.GetWeightsConv().GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, probability.GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, probability.GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, sequence.GetIndex()) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, probability.Total()) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)iWindow) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)iWindow) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)iFilters) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)None) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)0) kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!cWavlets.GetWeightsConv().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronFEDW | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFEDW::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cFF[0].CalcHiddenGradients(cFF[1].AsObject())) ReturnFalse; if(!cReconstruct.CalcHiddenGradients(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(Gradient, cReconstruct.getGradient(), cReconstruct.getGradient(), iWindow, false)) ReturnFalse; //--- CBufferFloat *temp_r = cReconstruct.getOutput(); if(!cReconstruct.SetOutput(cReconstruct.getGradient(), false)) ReturnFalse; CBufferFloat *temp_w = cWavlets.getOutput(); if(!cWavlets.SetOutput(cSoftMax.getGradient(), false)) ReturnFalse; //--- if(!cWavlets.FeedForward(cReconstruct.AsObject())) ReturnFalse; //--- if(!cWavlets.SetOutput(temp_w, false)) ReturnFalse; if(!cReconstruct.SetOutput(temp_r, false)) ReturnFalse; //--- if(!cNorm.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!cWavlets.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cWavlets.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cReconstruct.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronFEDW trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronFEDW::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cFF[0].UpdateInputWeights(cReconstruct.AsObject())) ReturnFalse; if(!cFF[1].UpdateInputWeights(cFF[0].AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cWavlets.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronFEDW GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronFEDW::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cWavlets.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cFF[0].SetOpenCL(OpenCL); cFF[1].SetOpenCL(OpenCL); cReconstruct.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronFEDW state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronFEDW::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWavlets.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cFF[0].Save(file_handle)) ReturnFalse; if(!cFF[1].Save(file_handle)) ReturnFalse; if(!cReconstruct.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFilters) < INT_VALUE) //--- --- ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronFEDW state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronFEDW::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cWavlets.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSoftMax.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFF[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFF[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cReconstruct.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); iCount = (uint)FileReadInteger(file_handle); iFilters = (uint)FileReadInteger(file_handle); //--- if(Gradient != cFF[1].getGradient()) SetGradient(cFF[1].getGradient()); //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronFEDW weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronFEDW::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFEDW *Source = source; for(int i = 0; i < 2; i++) { if(!cFF[i].WeightsUpdate(Source.cFF[i].AsObject(), tau)) ReturnFalse; } if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronFITSOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(window <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- Save constants iWindow = window; iWindowOut = window_out; iCount = count; activation = None; //--- Calculate FFT and iFFT size int power = int(MathLog(iWindow) / M_LN2); if(MathPow(2, power) != iWindow) power++; iFFTin = uint(MathPow(2, power)); power = int(MathLog(iWindowOut) / M_LN2); if(MathPow(2, power) != iWindowOut) power++; iIFFTin = uint(MathPow(2, power)); //--- Init objects if(!cInputsRe.Init(0, 0, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cInputsIm.Init(0, 1, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTRe.Init(0, 2, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTIm.Init(0, 3, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cDropRe.Init(0, 4, OpenCL, iFFTin * iCount, dropout, optimization, iBatch)) ReturnFalse; if(!cDropIm.Init(0, 5, OpenCL, iFFTin * iCount, dropout, optimization, iBatch)) ReturnFalse; if(!cInsideRe1.Init(0, 6, OpenCL, iFFTin, iFFTin, 4 * iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideRe1.SetActivationFunction(TANH); if(!cInsideIm1.Init(0, 7, OpenCL, iFFTin, iFFTin, 4 * iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideIm1.SetActivationFunction(TANH); if(!cInsideRe2.Init(0, 8, OpenCL, 4 * iIFFTin, 4 * iIFFTin, iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideRe2.SetActivationFunction(None); if(!cInsideIm2.Init(0, 9, OpenCL, 4 * iIFFTin, 4 * iIFFTin, iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideIm2.SetActivationFunction(None); if(!cComplexRe.Init(0, 10, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexIm.Init(0, 11, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTRe.Init(0, 12, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTIm.Init(0, 13, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cClear.BufferInit(MathMax(iFFTin, iIFFTin)*iCount, 0)) ReturnFalse; cClear.BufferCreate(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronFITSOCL layer. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iCount}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iCount)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iCount)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex layer out step of the CNeuronFITSOCL layer. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::ComplexLayerOut(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im ) { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iCount, inp_re.Total() / iCount }; setBuffer(def_k_ComplexLayer, def_k_cl_inputs_re, inp_re.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_inputs_im, inp_im.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_outputs_re, out_re.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_outputs_im, out_im.GetIndex()) kernelExecute(def_k_ComplexLayer, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex layer gradient step of the CNeuronFITSOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::ComplexLayerGradient(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im ) { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iCount, inp_re.Total() / iCount }; setBuffer(def_k_ComplexLayerGradient, def_k_cl_inputs_re, inp_re.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_inputs_im, inp_im.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_outputs_re, out_re.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_outputs_im, out_im.GetIndex()) kernelExecute(def_k_ComplexLayerGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronFITSOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- FFT if(!FFT(NeuronOCL.getOutput(), NULL, cFFTRe.getOutput(), cFFTIm.getOutput(), false)) ReturnFalse; //--- DropOut if(!cDropRe.FeedForward(cFFTRe.AsObject())) ReturnFalse; if(!cDropIm.FeedForward(cFFTIm.AsObject())) ReturnFalse; //--- Complex Layer if(!cInsideRe1.FeedForward(cDropRe.AsObject())) ReturnFalse; if(!cInsideRe2.FeedForward(cInsideRe1.AsObject())) ReturnFalse; if(!cInsideIm1.FeedForward(cDropIm.AsObject())) ReturnFalse; if(!cInsideIm2.FeedForward(cInsideIm1.AsObject())) ReturnFalse; if(!ComplexLayerOut(cInsideRe2.getOutput(), cInsideIm2.getOutput(), cComplexRe.getOutput(), cComplexIm.getOutput())) ReturnFalse; //--- iFFT if(!FFT(cComplexRe.getOutput(), cComplexIm.getOutput(), cIFFTRe.getOutput(), cIFFTIm.getOutput(), true)) ReturnFalse; //--- To Output if(!DeConcat(Output, cIFFTRe.getGradient(), cIFFTRe.getOutput(), iWindowOut, iIFFTin - iWindowOut, iCount)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronFITSOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Copy Gradients if(!SumAndNormalize(cIFFTIm.getOutput(), GetPointer(cClear), cIFFTIm.getGradient(), 1, false, 0, 0, 0, -1)) ReturnFalse; if(!Concat(Gradient, GetPointer(cClear), cIFFTRe.getGradient(), iWindowOut, iIFFTin - iWindowOut, iCount)) ReturnFalse; //--- FFT if(!FFT(cIFFTRe.getGradient(), cIFFTIm.getGradient(), cComplexRe.getGradient(), cComplexIm.getGradient(), false)) ReturnFalse; //--- Complex Layer if(!ComplexLayerGradient(cInsideRe2.getGradient(), cInsideIm2.getGradient(), cComplexRe.getGradient(), cComplexIm.getGradient())) ReturnFalse; if(!cInsideRe1.CalcHiddenGradients(cInsideRe2.AsObject())) ReturnFalse; if(!cInsideIm1.CalcHiddenGradients(cInsideIm2.AsObject())) ReturnFalse; if(!cDropRe.CalcHiddenGradients(cInsideRe1.AsObject())) ReturnFalse; if(!cDropIm.CalcHiddenGradients(cInsideIm1.AsObject())) ReturnFalse; //--- Dropout if(!cFFTRe.CalcHiddenGradients(cDropRe.AsObject())) ReturnFalse; if(!cFFTIm.CalcHiddenGradients(cDropIm.AsObject())) ReturnFalse; //--- IFFT if(!FFT(cFFTRe.getGradient(), cFFTIm.getGradient(), cInputsRe.getGradient(), cInputsIm.getGradient(), true)) ReturnFalse; //--- To Input Layer if(!DeConcat(NeuronOCL.getGradient(), cFFTIm.getGradient(), cFFTRe.getGradient(), iWindow, iFFTin - iWindow, iCount)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronFITSOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cInsideRe1.UpdateInputWeights(cDropRe.AsObject())) ReturnFalse; if(!cInsideIm1.UpdateInputWeights(cDropIm.AsObject())) ReturnFalse; if(!cInsideRe2.UpdateInputWeights(cInsideRe1.AsObject())) ReturnFalse; if(!cInsideIm2.UpdateInputWeights(cInsideIm1.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronFITSOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowOut)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTin)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iIFFTin)) < INT_VALUE) ReturnFalse; //--- Save objects if(!cInsideRe1.Save(file_handle)) ReturnFalse; if(!cInsideIm1.Save(file_handle)) ReturnFalse; if(!cInsideRe2.Save(file_handle)) ReturnFalse; if(!cInsideIm2.Save(file_handle)) ReturnFalse; if(!cDropRe.Save(file_handle)) ReturnFalse; if(!cDropIm.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronFITSOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iWindowOut = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iFFTin = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iIFFTin = uint(FileReadInteger(file_handle)); activation = None; //--- Load objects if(!LoadInsideLayer(file_handle, cInsideRe1.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideIm1.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideRe2.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideIm2.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDropRe.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDropIm.AsObject())) ReturnFalse; //--- Init objects if(!cInputsRe.Init(0, 0, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cInputsIm.Init(0, 1, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTRe.Init(0, 2, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTIm.Init(0, 3, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexRe.Init(0, 8, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexIm.Init(0, 9, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTRe.Init(0, 10, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTIm.Init(0, 11, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cClear.BufferInit(MathMax(iFFTin, iIFFTin)*iCount, 0)) ReturnFalse; cClear.BufferCreate(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronFITSOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronFITSOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInputsRe.SetOpenCL(OpenCL); cInputsIm.SetOpenCL(OpenCL); cFFTRe.SetOpenCL(OpenCL); cFFTIm.SetOpenCL(OpenCL); cDropRe.SetOpenCL(OpenCL); cDropIm.SetOpenCL(OpenCL); cInsideRe1.SetOpenCL(OpenCL); cInsideIm1.SetOpenCL(OpenCL); cInsideRe2.SetOpenCL(OpenCL); cInsideIm2.SetOpenCL(OpenCL); cComplexRe.SetOpenCL(OpenCL); cComplexIm.SetOpenCL(OpenCL); cIFFTRe.SetOpenCL(OpenCL); cIFFTIm.SetOpenCL(OpenCL); cClear.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronFITSOCL layer. | //+------------------------------------------------------------------+ void CNeuronFITSOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cInputsRe.TrainMode(bTrain); cInputsIm.TrainMode(bTrain); cFFTRe.TrainMode(bTrain); cFFTIm.TrainMode(bTrain); cDropRe.TrainMode(bTrain); cDropIm.TrainMode(bTrain); cInsideRe1.TrainMode(bTrain); cInsideIm1.TrainMode(bTrain); cInsideRe2.TrainMode(bTrain); cInsideIm2.TrainMode(bTrain); cComplexRe.TrainMode(bTrain); cComplexIm.TrainMode(bTrain); cIFFTRe.TrainMode(bTrain); cIFFTIm.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronFITSOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFITSOCL *Source = source; if(cInsideRe1.WeightsUpdate(Source.cInsideRe1.AsObject(), tau)) ReturnFalse; if(cInsideIm1.WeightsUpdate(Source.cInsideIm1.AsObject(), tau)) ReturnFalse; if(cInsideRe2.WeightsUpdate(Source.cInsideRe2.AsObject(), tau)) ReturnFalse; if(cInsideIm2.WeightsUpdate(Source.cInsideIm2.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronFreDFOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint count, float Alpha, bool need_transpose = true, ENUM_OPTIMIZATION optimization_type = ADAM, uint batch = 1 ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- bTranspose = need_transpose; iWindow = window; iCount = count; fAlpha = MathMax(0, MathMin(Alpha, 1)); activation = None; //--- Calculate FFTsize uint size = (bTranspose ? count : window); int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) != size) power++; iFFTin = uint(MathPow(2, power)); //--- uint n = (bTranspose ? iWindow : iCount); if(!cForecastFreRe.BufferInit(iFFTin * n, 0) || !cForecastFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastFreIm.BufferInit(iFFTin * n, 0) || !cForecastFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreRe.BufferInit(iFFTin * n, 0) || !cTargetFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreIm.BufferInit(iFFTin * n, 0) || !cTargetFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreRe.BufferInit(iFFTin * n, 0) || !cLossFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreIm.BufferInit(iFFTin * n, 0) || !cLossFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreRe.BufferInit(iFFTin * n, 0) || !cGradientFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreIm.BufferInit(iFFTin * n, 0) || !cGradientFreIm.BufferCreate(OpenCL)) ReturnFalse; if(bTranspose) if(!cTranspose.BufferInit(iWindow * iCount, 0) || !cTranspose.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronFreDFOCL layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {(bTranspose ? iWindow : iCount)}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / (!bTranspose ? iWindow : iCount))) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / (!bTranspose ? iWindow : iCount))) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronFreDFOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; if(NeuronOCL.getOutput() != Output) { Output.BufferFree(); DeleteObj(Output); Output = NeuronOCL.getOutput(); } //--- return(true); } //+------------------------------------------------------------------+ //| Transposes the internal tensors of the CNeuronFreDFOCL layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Transpose(CBufferFloat* inputs, CBufferFloat* outputs, uint rows, uint cols) { if(!inputs || !outputs) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {rows, cols}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, inputs.GetIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, outputs.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the freq msa step of the CNeuronFreDFOCL layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::FreqMSA(CBufferFloat* target, CBufferFloat* forecast, CBufferFloat* gradient) { if(!target || !forecast || !gradient) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {gradient.Total()}; setBuffer(def_k_GradientMSA, def_k_gmsa_target, target.GetIndex()) setBuffer(def_k_GradientMSA, def_k_gmsa_forecast, forecast.GetIndex()) setBuffer(def_k_GradientMSA, def_k_gmsa_gradient, gradient.GetIndex()) kernelExecute(def_k_GradientMSA, global_work_offset, global_work_size) #ifdef _DEBUG if(!gradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cumulative gradient step of the CNeuronFreDFOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::CumulativeGradient(CBufferFloat* gradient1, CBufferFloat* gradient2, CBufferFloat* cummulative, float Alpha ) { if(!gradient1 || !gradient2 || !cummulative) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {cummulative.Total()}; setBuffer(def_k_CumulativeGradient, def_k_cg_gradient1, gradient1.GetIndex()) setBuffer(def_k_CumulativeGradient, def_k_cg_gradient2, gradient2.GetIndex()) setBuffer(def_k_CumulativeGradient, def_k_cg_gradient_out, cummulative.GetIndex()) setArgument(def_k_CumulativeGradient, def_k_cg_alpha, float(Alpha)) kernelExecute(def_k_CumulativeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cummulative.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the calc output gradients step of the CNeuronFreDFOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::calcOutputGradients(CArrayFloat* Target, float & error) { if(!Target) ReturnFalse; if(Target.Total() < Output.Total()) ReturnFalse; //--- if(Target.Total() == Output.Total()) { if(!Gradient.AssignArray(Target)) ReturnFalse; } else { for(int i = 0; i < Output.Total(); i++) { if(!Gradient.Update(i, Target.At(i))) ReturnFalse; } } if(!Gradient.BufferWrite()) ReturnFalse; //--- if(bTranspose) { if(!Transpose(Output, GetPointer(cTranspose), iWindow, iCount)) ReturnFalse; if(!FFT(GetPointer(cTranspose), NULL, GetPointer(cForecastFreRe), GetPointer(cForecastFreIm), false)) ReturnFalse; if(!Transpose(Gradient, GetPointer(cTranspose), iWindow, iCount)) ReturnFalse; if(!FFT(GetPointer(cTranspose), NULL, GetPointer(cTargetFreRe), GetPointer(cTargetFreIm), false)) ReturnFalse; } else { if(!FFT(Output, NULL, GetPointer(cForecastFreRe), GetPointer(cForecastFreIm), false)) ReturnFalse; if(!FFT(Gradient, NULL, GetPointer(cTargetFreRe), GetPointer(cTargetFreIm), false)) ReturnFalse; } //--- if(!FreqMSA(GetPointer(cTargetFreRe), GetPointer(cForecastFreRe), GetPointer(cLossFreRe))) ReturnFalse; if(!FreqMSA(GetPointer(cTargetFreIm), GetPointer(cForecastFreIm), GetPointer(cLossFreIm))) ReturnFalse; if(!FreqMSA(Gradient, Output, Gradient)) ReturnFalse; //--- if(!FFT(GetPointer(cLossFreRe), GetPointer(cLossFreIm), GetPointer(cGradientFreRe), GetPointer(cGradientFreIm), true)) ReturnFalse; if(bTranspose) { if(!Transpose(GetPointer(cGradientFreRe), GetPointer(cTranspose), iCount, iWindow)) ReturnFalse; if(!CumulativeGradient(GetPointer(cTranspose), Gradient, Gradient, fAlpha)) ReturnFalse; } else if(!CumulativeGradient(GetPointer(cGradientFreRe), Gradient, Gradient, fAlpha)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronFreDFOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- return(DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, NeuronOCL.Activation())); } //+------------------------------------------------------------------+ //| Saves the CNeuronFreDFOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTin)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(bTranspose)) < INT_VALUE) ReturnFalse; if(FileWriteFloat(file_handle, fAlpha) < sizeof(float)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronFreDFOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iFFTin = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; bTranspose = bool(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; fAlpha = FileReadFloat(file_handle); //--- uint n = (bTranspose ? iWindow : iCount); if(!cForecastFreRe.BufferInit(iFFTin * n, 0) || !cForecastFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastFreIm.BufferInit(iFFTin * n, 0) || !cForecastFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreRe.BufferInit(iFFTin * n, 0) || !cTargetFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreIm.BufferInit(iFFTin * n, 0) || !cTargetFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreRe.BufferInit(iFFTin * n, 0) || !cLossFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreIm.BufferInit(iFFTin * n, 0) || !cLossFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreRe.BufferInit(iFFTin * n, 0) || !cGradientFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreIm.BufferInit(iFFTin * n, 0) || !cGradientFreIm.BufferCreate(OpenCL)) ReturnFalse; if(bTranspose) { if(!cTranspose.BufferInit(iWindow * iCount, 0) || !cTranspose.BufferCreate(OpenCL)) ReturnFalse; } else { cTranspose.BufferFree(); cTranspose.Clear(); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronFreDFOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronFreDFOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cForecastFreRe.BufferCreate(OpenCL); cForecastFreIm.BufferCreate(OpenCL); cTargetFreRe.BufferCreate(OpenCL); cTargetFreIm.BufferCreate(OpenCL); cLossFreRe.BufferCreate(OpenCL); cLossFreIm.BufferCreate(OpenCL); cGradientFreRe.BufferCreate(OpenCL); cGradientFreIm.BufferCreate(OpenCL); if(bTranspose) cTranspose.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronATFNetOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint history, uint forecast, uint variables, uint heads, uint layers, uint & patch[], ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- iHistory = MathMax(history, 1); iForecast = forecast; iVariables = variables; //--- Calculate FFTsize uint size = iHistory + iForecast; int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) < size) power++; iFFT = uint(MathPow(2, power)); //--- T-Block if(!cNorm.Init(0, 0, OpenCL, iHistory * iVariables, batch, optimization)) ReturnFalse; if(!cPositionEncoder.Init(0, 1, OpenCL, iVariables, iHistory, optimization, batch)) ReturnFalse; if(!cTranspose.Init(0, 2, OpenCL, iHistory, iVariables, optimization, batch)) ReturnFalse; cTranspose.SetActivationFunction(None); uint count = (iHistory - patch[0] + 2 * patch[1] - 1) / patch[1]; if(!cPatching.Init(0, 3, OpenCL, patch[0], patch[1], patch[2], count, iVariables, optimization, batch)) ReturnFalse; caAttention.SetOpenCL(OpenCL); for(uint l = 0; l < layers; l++) { CNeuronConformer *temp = new CNeuronConformer(); if(!temp) ReturnFalse; if(!temp.Init(0, 4 + l, OpenCL, patch[2], 32, heads, iVariables, count, optimization, batch)) { DeleteObj(temp); ReturnFalse; } if(!caAttention.Add(temp)) { DeleteObj(temp); ReturnFalse; } } int total = 3; caProjection.SetOpenCL(OpenCL); uint window = patch[2] * count; for(int l = 0; l < total; l++) { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 4 + layers + l, OpenCL, window, window, (total - l)*iForecast, iVariables, optimization, batch)) { DeleteObj(temp); ReturnFalse; } temp.SetActivationFunction(TANH); if(!caProjection.Add(temp)) { DeleteObj(temp); ReturnFalse; } window = (total - l) * iForecast; } if(!cRevIN.Init(0, 4 + layers + total, OpenCL, iForecast * iVariables, 1, cNorm.AsObject())) ReturnFalse; //--- F-Block if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 0, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMainFreqWeights.BufferInit(iVariables, 0) || !cMainFreqWeights.BufferCreate(OpenCL)) ReturnFalse; if(!cNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cFreqAtteention.Init(0, 2, OpenCL, iFFT, 32, heads, iVariables, layers, optimization, batch)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasReGrad.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasReGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cReconstructInput.BufferInit(iHistory * iVariables, 0) || !cReconstructInput.BufferCreate(OpenCL)) ReturnFalse; if(!cForecast.BufferInit(iForecast * iVariables, 0) || !cForecast.BufferCreate(OpenCL)) ReturnFalse; if(!cReconstructInputGrad.BufferInit(iHistory * iVariables, 0) || !cReconstructInputGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastGrad.BufferInit(iForecast * iVariables, 0) || !cForecastGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronATFNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex normalize step of the CNeuronATFNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex un normalize step of the CNeuronATFNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexUnNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalize, def_k_cn_inputs, cFreqAtteention.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_outputs, cUnNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!cFreqAtteention.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex normalize gradient step of the | //| CNeuronATFNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex un normalize gradient step of the | //| CNeuronATFNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexUnNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_inputs_gr, cFreqAtteention.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_outputs_gr, cUnNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalizeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cFreqAtteention.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the main freq weights step of the CNeuronATFNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::MainFreqWeights(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_MainFreqWeight, def_k_mfw_freq, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_MainFreqWeight, def_k_mfw_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_MainFreqWeight, def_k_mfw_dimension, (int)(iFFT)) kernelExecute(def_k_MainFreqWeight, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the weighted sum step of the CNeuronATFNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::WeightedSum(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_WeightedSum, def_k_ws_inputs1, cForecast.GetIndex()) setBuffer(def_k_WeightedSum, def_k_ws_inputs2, cRevIN.getOutputIndex()) setBuffer(def_k_WeightedSum, def_k_ws_outputs, getOutputIndex()) setBuffer(def_k_WeightedSum, def_k_ws_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_WeightedSum, def_k_ws_dimension, (int)(iForecast)) kernelExecute(def_k_WeightedSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!cMainFreqWeights.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the weighted sum gradient step of the CNeuronATFNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::WeightedSumGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_WeightedSumGradient, def_k_ws_inputs1, cForecastGrad.GetIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_inputs2, cRevIN.getGradientIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_outputs, getGradientIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_WeightedSumGradient, def_k_ws_dimension, (int)(iForecast)) kernelExecute(def_k_WeightedSumGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cMainFreqWeights.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the calc reconstruct gradient step of the | //| CNeuronATFNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::calcReconstructGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iHistory * iVariables; setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_t, cInputs.GetIndex()) setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_o, cReconstructInput.GetIndex()) setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_ig, cReconstructInputGrad.GetIndex()) setArgument(def_k_CalcOutputGradient, def_k_cog_activation, (int)None) setArgument(def_k_CalcOutputGradient, def_k_cog_error, 1) ResetLastError(); kernelExecute(def_k_CalcOutputGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cReconstructInputGrad.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronATFNetOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; if(cInputs != NeuronOCL.getOutput()) cInputs = NeuronOCL.getOutput(); //--- T-Block if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse;; if(!cPositionEncoder.FeedForward(cNorm.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cPositionEncoder.AsObject())) ReturnFalse; if(!cPatching.FeedForward(cTranspose.AsObject())) ReturnFalse; int total = caAttention.Total(); CNeuronBaseOCL* prev = cPatching.AsObject(); for(int i = 0; i < total; i++) { CNeuronBaseOCL* att = caAttention.At(i); if(!att.FeedForward(prev)) ReturnFalse; prev = att; } total = caProjection.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL* proj = caProjection.At(i); if(!proj.FeedForward(prev)) ReturnFalse; prev = proj; } if(!cRevIN.FeedForward(prev)) ReturnFalse; //--- F-Block if(!FFT(cInputs, cInputs, GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexNormalize()) ReturnFalse; if(!MainFreqWeights()) ReturnFalse; if(!cFreqAtteention.FeedForward(cNormFreqComplex.AsObject())) ReturnFalse; if(!ComplexUnNormalize()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cOutputTimeSeriasRe), GetPointer(cOutputTimeSeriasIm), true)) ReturnFalse; if(!DeConcat(GetPointer(cReconstructInput), GetPointer(cForecast), GetPointer(cOutputTimeSeriasReGrad), GetPointer(cOutputTimeSeriasRe), iHistory, iForecast, iFFT - iHistory - iForecast, iVariables)) ReturnFalse; //--- Output if(!WeightedSum()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronATFNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getGradient() || !cInputs) ReturnFalse; //--- Output if(!WeightedSumGradient()) ReturnFalse; //--- T-Block if(cRevIN.Activation() != None && !DeActivation(cRevIN.getOutput(), cRevIN.getGradient(), cRevIN.getGradient(), cRevIN.Activation())) ReturnFalse; CNeuronBaseOCL *next = cRevIN.AsObject(); for(int i = caProjection.Total() - 1; i >= 0; i--) { CNeuronBaseOCL *proj = caProjection.At(i); if(!proj || !proj.CalcHiddenGradients((CObject *)next)) ReturnFalse; next = proj; } for(int i = caAttention.Total() - 1; i >= 0; i--) { CNeuronBaseOCL *att = caAttention.At(i); if(!att || !att.CalcHiddenGradients((CObject *)next)) ReturnFalse; next = att; } if(!cPatching.CalcHiddenGradients((CObject*)next)) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; if(!cPositionEncoder.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cPositionEncoder.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- F-Block if(!CNeuronBaseOCL::SumAndNormalize(GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), iFFT * iVariables, false, 0, 0, 0, -0.5)) ReturnFalse; if(!calcReconstructGradient()) ReturnFalse; if(!Concat(GetPointer(cReconstructInputGrad), GetPointer(cForecastGrad), GetPointer(cZero), GetPointer(cOutputTimeSeriasReGrad), iHistory, iForecast, iFFT - iHistory - iForecast, iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputTimeSeriasReGrad), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexUnNormalizeGradient()) ReturnFalse; if(!cNormFreqComplex.CalcHiddenGradients(cFreqAtteention.AsObject())) ReturnFalse; if(!ComplexNormalizeGradient()) ReturnFalse; if(!DeConcat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), GetPointer(cOutputTimeSeriasRe), GetPointer(cOutputTimeSeriasIm), false)) ReturnFalse; if(!DeConcat(GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasRe), iHistory, iFFT - iHistory, iVariables)) ReturnFalse; //--- if(!CNeuronBaseOCL::SumAndNormalize(NeuronOCL.getGradient(), GetPointer(cInputFreqRe), NeuronOCL.getGradient(), iHistory * iVariables, false, 0, 0, 0, 0.5)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronATFNetOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- F-Block if(!cFreqAtteention.UpdateInputWeights(cNormFreqComplex.AsObject())) ReturnFalse; //--- T-Block if(!cPatching.UpdateInputWeights(cPositionEncoder.AsObject())) ReturnFalse; int total = caAttention.Total(); CNeuronBaseOCL *prev = cPatching.AsObject(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *att = caAttention.At(i); if(!att.UpdateInputWeights(prev)) ReturnFalse; prev = att; } total = caProjection.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *proj = caProjection.At(i); if(!proj.UpdateInputWeights(prev)) ReturnFalse; prev = proj; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronATFNetOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iHistory) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iForecast) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFFT) < INT_VALUE) ReturnFalse; //--- T-Block if(!cNorm.Save(file_handle)) ReturnFalse; if(!cPositionEncoder.Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!caAttention.Save(file_handle)) ReturnFalse; if(!caProjection.Save(file_handle)) ReturnFalse; //--- F-Block if(!cFreqAtteention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronATFNetOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFT = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; //--- T-Block if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPositionEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!caAttention.Load(file_handle)) ReturnFalse; if(!caProjection.Load(file_handle)) ReturnFalse; if(!cTranspose.Init(0, 2, OpenCL, iHistory, iVariables, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); if(!cRevIN.Init(0, 4 + caAttention.Total() + caProjection.Total(), OpenCL, iForecast * iVariables, 1, cNorm.AsObject())) ReturnFalse; //--- F-Block if(!LoadInsideLayer(file_handle, cFreqAtteention.AsObject())) ReturnFalse; cInputFreqRe.BufferFree(); if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cInputFreqIm.BufferFree(); if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 0, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMainFreqWeights.BufferFree(); if(!cMainFreqWeights.BufferInit(iVariables, 0) || !cMainFreqWeights.BufferCreate(OpenCL)) ReturnFalse; if(!cNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMeans.BufferFree(); if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; cVariances.BufferFree(); if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cOutputFreqRe.BufferFree(); if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cOutputFreqIm.BufferFree(); if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasRe.BufferCreate(OpenCL)) ReturnFalse; cOutputTimeSeriasIm.BufferFree(); if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; cOutputTimeSeriasReGrad.BufferFree(); if(!cOutputTimeSeriasReGrad.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasReGrad.BufferCreate(OpenCL)) ReturnFalse; cReconstructInput.BufferFree(); if(!cReconstructInput.BufferInit(iHistory * iVariables, 0) || !cReconstructInput.BufferCreate(OpenCL)) ReturnFalse; cForecast.BufferFree(); if(!cForecast.BufferInit(iForecast * iVariables, 0) || !cForecast.BufferCreate(OpenCL)) ReturnFalse; cReconstructInputGrad.BufferFree(); if(!cReconstructInputGrad.BufferInit(iHistory * iVariables, 0) || !cReconstructInputGrad.BufferCreate(OpenCL)) ReturnFalse; cForecastGrad.BufferFree(); if(!cForecastGrad.BufferInit(iForecast * iVariables, 0) || !cForecastGrad.BufferCreate(OpenCL)) ReturnFalse; cZero.BufferFree(); if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronATFNetOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronATFNetOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- T-Block cNorm.SetOpenCL(OpenCL); cPositionEncoder.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cPatching.SetOpenCL(OpenCL); for(int l = 0; l < caAttention.Total(); l++) { CNeuronBaseOCL *temp = caAttention.At(l); if(!temp) continue; temp.SetOpenCL(OpenCL); } for(int l = 0; l < caProjection.Total(); l++) { CNeuronBaseOCL *temp = caProjection.At(l); if(!temp) continue; temp.SetOpenCL(OpenCL); } cRevIN.SetOpenCL(OpenCL); //--- F-Block cInputFreqRe.BufferCreate(OpenCL); cInputFreqIm.BufferCreate(OpenCL); cInputFreqComplex.SetOpenCL(OpenCL); cMainFreqWeights.BufferCreate(OpenCL); cNormFreqComplex.SetOpenCL(OpenCL); cMeans.BufferCreate(OpenCL); cVariances.BufferCreate(OpenCL); cFreqAtteention.SetOpenCL(OpenCL); cUnNormFreqComplex.SetOpenCL(OpenCL); cOutputFreqRe.BufferCreate(OpenCL); cOutputFreqIm.BufferCreate(OpenCL); cOutputTimeSeriasRe.BufferCreate(OpenCL); cOutputTimeSeriasIm.BufferCreate(OpenCL); cOutputTimeSeriasReGrad.BufferCreate(OpenCL); cReconstructInput.BufferCreate(OpenCL); cForecast.BufferCreate(OpenCL); cReconstructInputGrad.BufferCreate(OpenCL); cForecastGrad.BufferCreate(OpenCL); cZero.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Returns the weights of the CNeuronATFNetOCL. | //+------------------------------------------------------------------+ CBufferFloat* CNeuronATFNetOCL::getWeights(void) { CNeuronConvOCL *conv = caProjection.At(caProjection.Total() - 1); return(conv.GetWeightsConv()); } //+------------------------------------------------------------------+ //| Initializes the CNeuronS3 layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronS3::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; //--- iWindow = MathMax(window, 1); iSegments = (numNeurons + window - 1) / window; //--- if(!cOne.Init(0, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buffer = cOne.getOutput(); if(!buffer || !buffer.BufferInit(buffer.Total(), 1)) ReturnFalse; if(!buffer.BufferWrite()) ReturnFalse; //--- if(!cShufle.Init(0, 1, OpenCL, 1, 1, iSegments, 1, optimization, iBatch)) ReturnFalse; cShufle.SetActivationFunction(None); if(!cProbability.Init(0, 2, OpenCL, iSegments, optimization, iBatch)) ReturnFalse; cProbability.SetActivationFunction(None); cProbability.SetHeads(1); if(!cWeights.Init(0, 3, OpenCL, 1, 1, 2, 1, optimization, iBatch)) ReturnFalse; cWeights.SetActivationFunction(SIGMOID); if(!cPositions.BufferInit(iSegments, 0) || !cPositions.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronS3 feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronS3::feedForwardS3(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSegments}; setBuffer(def_k_FeedForwardS3, def_k_s3_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_outputs, getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_positions, cPositions.GetIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_probability, cProbability.getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_weights, cWeights.getOutputIndex()) setArgument(def_k_FeedForwardS3, def_k_s3_window, (int)(iWindow)) setArgument(def_k_FeedForwardS3, def_k_s3_total, (int)(Neurons())) kernelExecute(def_k_FeedForwardS3, global_work_offset, global_work_size) #ifdef _DEBUG if(!cWeights.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronS3 layer. | //+------------------------------------------------------------------+ bool CNeuronS3::calcInputGradientsS3(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSegments}; setBuffer(def_k_InsideGradientS3, def_k_s3g_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_outputs_gr, getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_positions, cPositions.GetIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_probability, cProbability.getOutputIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_probability_gr, cProbability.getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_weights, cWeights.getOutputIndex()) setArgument(def_k_InsideGradientS3, def_k_s3g_window, (int)(iWindow)) setArgument(def_k_InsideGradientS3, def_k_s3g_total, (int)(Neurons())) kernelExecute(def_k_InsideGradientS3, global_work_offset, global_work_size) #ifdef _DEBUG if(!cWeights.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint dimension = MathMin(Neurons(), 64); uint global_work_size[2] = {dimension, 2}; uint local_work_size[2] = {dimension, 1}; setBuffer(def_k_WeightGradientS3, def_k_s3w_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_outputs_gr, getGradientIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_positions, cPositions.GetIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_weights_gr, cWeights.getGradientIndex()) setArgument(def_k_WeightGradientS3, def_k_s3w_window, (int)(iWindow)) setArgument(def_k_WeightGradientS3, def_k_s3w_total, (int)(Neurons())) kernelExecuteLoc(def_k_WeightGradientS3, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cWeights.getGradient().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronS3 feed-forward pass and writes the layer output | //| (feedForward). | //+------------------------------------------------------------------+ bool CNeuronS3::feedForward(CNeuronBaseOCL* NeuronOCL) { if(bTrain) { if(!cWeights.FeedForward(cOne.AsObject())) ReturnFalse; if(!cShufle.FeedForward(cOne.AsObject())) ReturnFalse; if(!cProbability.FeedForward(cShufle.AsObject())) ReturnFalse; } if(!feedForwardS3(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronS3 layer | //| (calcInputGradients). | //+------------------------------------------------------------------+ bool CNeuronS3::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!calcInputGradientsS3(NeuronOCL)) ReturnFalse; if(!cShufle.CalcHiddenGradients(cProbability.AsObject())) ReturnFalse; if(cWeights.Activation() != None) if(!DeActivation(cWeights.getOutput(), cWeights.getGradient(), cWeights.getGradient(), cWeights.Activation())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronS3 trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronS3::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cWeights.UpdateInputWeights(cOne.AsObject())) ReturnFalse; if(!cShufle.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronS3 state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronS3::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cShufle.Save(file_handle)) ReturnFalse; if(!cWeights.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow), INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSegments), INT_VALUE) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronS3 state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronS3::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cShufle.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWeights.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iSegments = uint(FileReadInteger(file_handle)); //--- if(cProbability.Neurons() != iSegments) { if(!cProbability.Init(0, 2, OpenCL, iSegments, optimization, iBatch)) ReturnFalse; cProbability.SetActivationFunction(None); } if(!cProbability.FeedForward(cShufle.AsObject())) ReturnFalse; if(cPositions.Total() != iSegments) { cPositions.BufferFree(); if(!cPositions.BufferInit(iSegments, 0) || !cPositions.BufferCreate(OpenCL)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronS3 GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronS3::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cShufle.SetOpenCL(OpenCL); cProbability.SetOpenCL(OpenCL); cWeights.SetOpenCL(OpenCL); cPositions.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Averages the CNeuronS3 weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronS3::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronS3 *Source = source; if(!cWeights.WeightsUpdate(Source.cWeights.AsObject(), tau)) ReturnFalse; if(!cShufle.WeightsUpdate(Source.cShufle.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTPMEncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronLSTMOCL::Init(numOutputs, myIndex, open_cl, hidden_size, optimization_type, batch)) ReturnFalse; if(!SetInputs(variables * lenth)) ReturnFalse; //--- uint windows[] = {variables, 6, 5, 4}; if(!cFeatureExtraction.Init(0, 0, OpenCL, windows, lenth, variables, ts_in_row, optimization, batch)) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 1, OpenCL, hidden_size * 2, optimization, batch)) ReturnFalse; if(!cConcatenated.Init(0, 2, OpenCL, variables * lenth, variables * lenth, hidden_size * 2, optimization, batch)) ReturnFalse; cConcatenated.SetActivationFunction(TANH); if(!cSoftMax.Init(0, 3, OpenCL, variables * lenth, optimization, batch)) ReturnFalse; cSoftMax.SetHeads(variables); if(!cAttentionOut.Init(0, 4, OpenCL, variables * lenth, optimization, batch)) ReturnFalse; //--- bTSinRow = ts_in_row; if(!bTSinRow) { if(!cTranspose.Init(0, 5, OpenCL, variables, lenth, optimization, iBatch)) ReturnFalse; } //--- if(!cTemp.BufferInit(variables * lenth, 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTPMEncoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- FEATURE EXTRACTION if(!cFeatureExtraction.FeedForward(NeuronOCL)) ReturnFalse; //--- Memory and Hidden if(!Concat(m_iHiddenState, m_iMemory, m_iHiddenState, m_iMemory, cMemAndHidden.getOutputIndex(), 1, 1, 0, 0, Neurons())) ReturnFalse; //--- Attention if(!cConcatenated.FeedForward(cFeatureExtraction.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cSoftMax.FeedForward(cConcatenated.AsObject())) ReturnFalse; CBufferFloat *map = cSoftMax.getOutput(); if(!bTSinRow) { if(!cTranspose.FeedForward(cSoftMax.AsObject())) ReturnFalse; map = cTranspose.getOutput(); } //--- if(!ElementMult(cFeatureExtraction.getOutput(), map, cAttentionOut.getOutput())) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- LSTM if(!CNeuronLSTMOCL::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTPMEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronLSTMOCL::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; //--- if(!ElementMultGrad(cFeatureExtraction.getOutput(), GetPointer(cTemp), (bTSinRow ? cSoftMax.getOutput() : cTranspose.getOutput()), (bTSinRow ? cSoftMax.getGradient() : cTranspose.getGradient()), cAttentionOut.getGradient(), NeuronOCL.Activation(), int(None))) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- if(bTSinRow) { if(!cSoftMax.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; } //--- if(!cConcatenated.CalcHiddenGradients((CObject*)cSoftMax.AsObject(), NULL) || !DeActivation(cConcatenated.getOutput(), cConcatenated.getGradient(), cConcatenated.getGradient(), cConcatenated.Activation())) ReturnFalse; if(!cFeatureExtraction.CalcHiddenGradients(cConcatenated.AsObject(), cMemAndHidden.getOutput(), cMemAndHidden.getGradient())) ReturnFalse; if(!DeActivation(cFeatureExtraction.getOutput(), GetPointer(cTemp), GetPointer(cTemp), NeuronOCL.Activation()) || !SumAndNormalize(cFeatureExtraction.getGradient(), GetPointer(cTemp), cFeatureExtraction.getGradient(), 1, false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cFeatureExtraction.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTPMEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronLSTMOCL::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; if(!cFeatureExtraction.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConcatenated.UpdateInputWeights(cFeatureExtraction.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTPMEncoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTPMEncoder::SetOpenCL(COpenCLMy * obj) { if(!OpenCL) cTemp.BufferFree(); //--- CNeuronLSTMOCL::SetOpenCL(obj); cFeatureExtraction.SetOpenCL(OpenCL); cMemAndHidden.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); if(!bTSinRow) cTranspose.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronTPMEncoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Save(const int file_handle) { if(!CNeuronLSTMOCL::Save(file_handle)) ReturnFalse; if(!cFeatureExtraction.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(bTSinRow)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTPMEncoder state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Load(const int file_handle) { if(!CNeuronLSTMOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureExtraction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSoftMax.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; bTSinRow = (bool)FileReadInteger(file_handle); //--- if(!cMemAndHidden.Init(0, 1, OpenCL, Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cAttentionOut.Init(0, 3, OpenCL, cSoftMax.Neurons(), optimization, iBatch)) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(cSoftMax.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTPM layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTPM::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronLSTMOCL::Init(numOutputs, myIndex, open_cl, hidden_size, optimization_type, batch)) ReturnFalse; if(!SetInputs(hidden_size)) ReturnFalse; //--- if(!cEncoder.Init(0, 0, OpenCL, variables, lenth, hidden_size, ts_in_row, optimization, iBatch)) ReturnFalse; //--- if(!cFeatureExtraction.Init(0, 1, OpenCL, variables, lenth, !ts_in_row, optimization, iBatch)) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 2, OpenCL, hidden_size * 2, optimization, iBatch)) ReturnFalse; if(!cConcatenated.Init(0, 3, OpenCL, hidden_size, hidden_size, hidden_size * 2, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(TANH); if(!cSoftMax.Init(0, 4, OpenCL, hidden_size, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(1); if(!cAttentionOut.Init(0, 5, OpenCL, hidden_size, optimization, iBatch)) ReturnFalse; if(!cAttAndFeature.Init(0, 6, OpenCL, hidden_size, hidden_size, variables * lenth, optimization, iBatch)) ReturnFalse; //--- if(!cTemp.BufferInit(variables * lenth, 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTPM feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronTPM::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Encoder if(!cEncoder.FeedForward(NeuronOCL)) ReturnFalse; //--- FEATURE EXTRACTION if(!cFeatureExtraction.FeedForward(NeuronOCL)) ReturnFalse; //--- Memory and Hidden if(!Concat(m_iHiddenState, m_iMemory, m_iHiddenState, m_iMemory, cMemAndHidden.getOutputIndex(), 1, 1, 0, 0, Neurons())) ReturnFalse; //--- Attention if(!cConcatenated.FeedForward(cEncoder.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cSoftMax.FeedForward(cConcatenated.AsObject())) ReturnFalse; //--- if(!ElementMult(cEncoder.getOutput(), cSoftMax.getOutput(), cAttentionOut.getOutput())) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- Attention and Features if(!cAttAndFeature.FeedForward(cAttentionOut.AsObject(), cFeatureExtraction.getOutput())) ReturnFalse; //--- LSTM if(!CNeuronLSTMOCL::feedForward(cAttAndFeature.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronTPM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTPM::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronLSTMOCL::calcInputGradients(cAttAndFeature.AsObject())) ReturnFalse; //--- if(!cAttentionOut.CalcHiddenGradients(cAttAndFeature.AsObject(), cFeatureExtraction.getOutput(), cFeatureExtraction.getGradient(), (ENUM_ACTIVATION)cFeatureExtraction.Activation() )) ReturnFalse; //--- if(!ElementMultGrad(cEncoder.getOutput(), cEncoder.getGradient(), cSoftMax.getOutput(), cSoftMax.getGradient(), cAttentionOut.getGradient(), cEncoder.Activation(), int(None))) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- if(!cConcatenated.CalcHiddenGradients(cSoftMax.AsObject(), NULL) || !DeActivation(cConcatenated.getOutput(), cConcatenated.getGradient(), cConcatenated.getGradient(), cConcatenated.Activation())) ReturnFalse; if(!cFeatureExtraction.CalcHiddenGradients(cConcatenated.AsObject(), cMemAndHidden.getOutput(), cMemAndHidden.getGradient())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cFeatureExtraction.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), GetPointer(cTemp), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cEncoder.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), GetPointer(cTemp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTPM trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronTPM::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronLSTMOCL::updateInputWeights(cAttAndFeature.AsObject())) ReturnFalse; if(!cEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeatureExtraction.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConcatenated.UpdateInputWeights(cEncoder.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cAttAndFeature.UpdateInputWeights(cAttentionOut.AsObject(), cFeatureExtraction.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTPM GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTPM::SetOpenCL(COpenCLMy * obj) { if(!OpenCL) cTemp.BufferFree(); //--- CNeuronLSTMOCL::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); cFeatureExtraction.SetOpenCL(OpenCL); cMemAndHidden.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cAttAndFeature.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronTPM state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTPM::Save(const int file_handle) { if(!CNeuronLSTMOCL::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cFeatureExtraction.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cAttAndFeature.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTPM state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTPM::Load(const int file_handle) { if(!CNeuronLSTMOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureExtraction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttAndFeature.AsObject())) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 2, OpenCL, Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cSoftMax.Init(0, 4, OpenCL, cConcatenated.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(1); if(!cAttentionOut.Init(0, 5, OpenCL, cSoftMax.Neurons(), optimization, iBatch)) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(cFeatureExtraction.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention out step of the CNeuronSTNNEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 1) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSTNNEncoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Feed Forward CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; inputs = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, inputs, 4 * iWindow, iWindow, None)) ReturnFalse; //--- Calculate Queries, Keys, Values CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, kv, iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSTNNEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = FF_Tensors.At(i * 6); CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), kv_g, inp, GetPointer(Temp), iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; inp = (i > 0 ? FF_Tensors.At(i * 6 - 4) : prevLayer.getOutput()); temp = (i > 0 ? FF_Tensors.At(i * 6 - 1) : prevLayer.getGradient()); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), inp, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out_grad = temp; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSTNNEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), inputs, (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 3), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), FF_Tensors.At(l * 6), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), FF_Tensors.At(l * 6), (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindow, 2 * iWindowKey * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 5), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow)) ReturnFalse; //--- inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSTNNDecoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!cEncoder.Init(0, 0, open_cl, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, window_kv, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention out step of the CNeuronSTNNDecoder layer. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 1) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSTNNDecoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!cEncoder.FeedForward(NeuronOCL, Context)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(cEncoder.AsObject(), Context)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSTNNDecoder layer. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(cEncoder.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEncoder.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSTNNDecoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!cEncoder.UpdateInputWeights(NeuronOCL, Context)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(cEncoder.AsObject(), Context)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSTNNDecoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSTNNDecoder state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSTNNDecoder weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronMLCrossAttentionMLKV::WeightsUpdate(source, tau)) ReturnFalse; if(!cEncoder.WeightsUpdate(((CNeuronSTNNDecoder*)source).cEncoder.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSTNNDecoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSTNNDecoder::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSparseTSF layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint sequence, uint variables, uint period, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; if(!acTranspose[0].Init(0, 0, OpenCL, sequence, variables, optimization, iBatch)) ReturnFalse; if(!cConvolution.Init(0, 1, OpenCL, 2 * period, period, period, sequence / period, variables, optimization, iBatch)) ReturnFalse; cConvolution.SetActivationFunction(None); if(!acTranspose[1].Init(0, 2, OpenCL, variables * sequence / period, period, optimization, iBatch)) ReturnFalse; if(!cForecast.Init(0, 3, OpenCL, sequence / period, sequence / period, forecast / period, variables, period, optimization, iBatch)) ReturnFalse; cForecast.SetActivationFunction(TANH); if(!acTranspose[2].Init(0, 4, OpenCL, period, variables * forecast / period, optimization, iBatch)) ReturnFalse; if(!acTranspose[3].Init(0, 5, OpenCL, variables, forecast, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(acTranspose[3].getOutput()) || !SetGradient(acTranspose[3].getGradient()) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSparseTSF feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!acTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cConvolution.FeedForward(acTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cConvolution.getOutput(), acTranspose[0].getOutput(), cConvolution.getOutput(), 1, false)) ReturnFalse; if(!acTranspose[1].FeedForward(cConvolution.AsObject())) ReturnFalse; if(!cForecast.FeedForward(acTranspose[1].AsObject())) ReturnFalse; if(!acTranspose[2].FeedForward(cForecast.AsObject())) ReturnFalse; if(!acTranspose[3].FeedForward(acTranspose[2].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronSparseTSF | //| layer. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!acTranspose[2].CalcHiddenGradients(acTranspose[3].AsObject())) ReturnFalse; if(!cForecast.CalcHiddenGradients(acTranspose[2].AsObject())) ReturnFalse; if(cForecast.Activation() != None && !DeActivation(cForecast.getOutput(), cForecast.getGradient(), cForecast.getGradient(), cForecast.Activation())) ReturnFalse; if(!acTranspose[1].CalcHiddenGradients(cForecast.AsObject())) ReturnFalse; if(!cConvolution.CalcHiddenGradients(acTranspose[1].AsObject())) ReturnFalse; if(!acTranspose[0].CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; if(!SumAndNormalize(cConvolution.getGradient(), acTranspose[0].getGradient(), acTranspose[0].getGradient(), 1, false)) ReturnFalse; if(!NeuronOCL || !NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None && !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSparseTSF trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cConvolution.UpdateInputWeights(acTranspose[0].AsObject())) ReturnFalse; if(!cForecast.UpdateInputWeights(acTranspose[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSparseTSF GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSparseTSF::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 4; i++) acTranspose[i].SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cForecast.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Averages the CNeuronSparseTSF weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSparseTSF *Source = source; if(!cConvolution.WeightsUpdate(Source.cConvolution.AsObject(), tau)) ReturnFalse; if(!cForecast.WeightsUpdate(Source.cForecast.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSparseTSF state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!acTranspose[i].Save(file_handle)) ReturnFalse;; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cForecast.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSparseTSF state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject())) ReturnFalse;; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cForecast.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTEMPOOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint sequence, uint variables, uint forecast, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { //--- base if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- constants iVariables = variables; iForecast = forecast; iSequence = MathMax(sequence, 1); //--- Calculate FFTsize uint size = iSequence; int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) < size) power++; iFFT = uint(MathPow(2, power)); //--- trend if(!cPLR.Init(0, 0, OpenCL, iVariables, iSequence, true, optimization, iBatch)) ReturnFalse; if(!cTrend.Init(0, 1, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- seasons if(!cInputSeasons.Init(0, 2, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[0].Init(0, 3, OpenCL, iSequence, iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[1].Init(0, 4, OpenCL, iVariables, iSequence, optimization, iBatch)) ReturnFalse; if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 5, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cNormFreqComplex.Init(0, 6, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cFreqAtteention.Init(0, 7, OpenCL, iFFT, 32, heads, iVariables, layers, optimization, batch)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 8, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.Init(0, 9, OpenCL, iFFT * iVariables, optimization, iBatch)) ReturnFalse; if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- Noise if(!cResidual.Init(0, 10, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- Forecast if(!cConcatInput.Init(0, 11, OpenCL, 3 * iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cNormalize.Init(0, 12, OpenCL, 3 * iSequence * iVariables, iBatch, optimization)) ReturnFalse; int window = MathMin(5, (int)iSequence - 1); int patches = (int)iSequence - window + 1; if(!cPatching.Init(0, 13, OpenCL, window, 1, 8, patches, 3 * iVariables, optimization, iBatch)) ReturnFalse; if(!acPE[0].Init(0, 14, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; int plr = cPLR.Neurons(); if(!cNormalizePLR.Init(0, 15, OpenCL, plr, iBatch, optimization)) ReturnFalse; plr = MathMax(plr / (3 * (int)iVariables), 1); if(!cPatchingPLR.Init(0, 16, OpenCL, 3, 3, 8, plr, iVariables, optimization, iBatch)) ReturnFalse; if(!acPE[1].Init(0, 17, OpenCL, plr, 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 18, OpenCL, 3 * 8 * iVariables, 3 * iVariables, MathMax(heads, 1), 8 * iVariables, MathMax(heads / 2, 1), patches, plr, MathMax(layers, 1), 2, optimization, iBatch)) ReturnFalse; if(!cTransposeAtt.Init(0, 19, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!acForecast[0].Init(0, 20, OpenCL, patches, patches, iForecast, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; acForecast[0].SetActivationFunction(LReLU); if(!acForecast[1].Init(0, 21, OpenCL, 8 * iForecast, 8 * iForecast, iForecast, 3 * iVariables, optimization, iBatch)) ReturnFalse; acForecast[1].SetActivationFunction(TANH); if(!cTransposeFrc.Init(0, 22, OpenCL, 3 * iVariables, iForecast, optimization, iBatch)) ReturnFalse; if(!cRevIn.Init(0, 23, OpenCL, 3 * iVariables * iForecast, 11, GetPointer(cNormalize))) ReturnFalse; if(!cSum.Init(0, 24, OpenCL, 3, 3, 1, iVariables, iForecast, optimization, iBatch)) ReturnFalse; cSum.SetActivationFunction(None); //--- SetActivationFunction(None); SetOutput(cSum.getOutput(), true); SetGradient(cSum.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronTEMPOOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex normalize step of the CNeuronTEMPOOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex un normalize step of the CNeuronTEMPOOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexUnNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalize, def_k_cn_inputs, cFreqAtteention.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_outputs, cUnNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!cFreqAtteention.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex normalize gradient step of the | //| CNeuronTEMPOOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex un normalize gradient step of the | //| CNeuronTEMPOOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexUnNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_inputs_gr, cFreqAtteention.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_outputs_gr, cUnNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalizeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cFreqAtteention.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cut trend and other step of the CNeuronTEMPOOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutTrendAndOther(CBufferFloat* inputs) { if(!inputs) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iSequence, iVariables}; setBuffer(def_k_CutTrendAndOther, def_k_ct_inputs, inputs.GetIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_plr, cPLR.getOutputIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_trend, cTrend.getOutputIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_other, cInputSeasons.getOutputIndex()) kernelExecute(def_k_CutTrendAndOther, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputSeasons.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cut trend and other gradient step of the | //| CNeuronTEMPOOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutTrendAndOtherGradient(CBufferFloat* inputs_gr) { if(!inputs_gr) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iSequence, iVariables}; setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_inputs_gr, inputs_gr.GetIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_plr, cPLR.getOutputIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_plr_gr, cPLR.getGradientIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_trend_gr, cTrend.getGradientIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_other_gr, cInputSeasons.getGradientIndex()) kernelExecute(def_k_CutTrendAndOtherGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cPLR.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cut one from another step of the CNeuronTEMPOOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutOneFromAnother(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSequence * iVariables}; setBuffer(def_k_CutOneFromAnother, def_k_ctofa_inputs, cInputSeasons.getOutputIndex()) setBuffer(def_k_CutOneFromAnother, def_k_ctofa_cut, cTranspose[1].getOutputIndex()) setBuffer(def_k_CutOneFromAnother, def_k_ctofa_other, cResidual.getOutputIndex()) kernelExecute(def_k_CutOneFromAnother, global_work_offset, global_work_size) #ifdef _DEBUG if(!cResidual.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cut one from another gradient step of the | //| CNeuronTEMPOOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutOneFromAnotherGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSequence * iVariables}; setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_inputs, cInputSeasons.getGradientIndex()) setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_cut, cOutputTimeSeriasRe.getGradientIndex()) setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_other, cResidual.getGradientIndex()) kernelExecute(def_k_CutOneFromAnother, global_work_offset, global_work_size) #ifdef _DEBUG if(!cResidual.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTEMPOOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- trend if(!cPLR.FeedForward(NeuronOCL)) ReturnFalse; if(!CutTrendAndOther(NeuronOCL.getOutput())) ReturnFalse; //--- seasons if(!cTranspose[0].FeedForward(cInputSeasons.AsObject())) ReturnFalse; if(!FFT(cTranspose[0].getOutput(), NULL, GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexNormalize()) ReturnFalse; if(!cFreqAtteention.FeedForward(cNormFreqComplex.AsObject())) ReturnFalse; if(!ComplexUnNormalize()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), true)) ReturnFalse; if(!DeConcat(cOutputTimeSeriasRe.getOutput(), cOutputTimeSeriasRe.getGradient(), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!cTranspose[1].FeedForward(cOutputTimeSeriasRe.AsObject())) ReturnFalse; //--- Noise if(!CutOneFromAnother()) ReturnFalse; //--- Forecast if(!Concat(cTrend.getOutput(), cTranspose[1].getOutput(), cResidual.getOutput(), cConcatInput.getOutput(), 1, 1, 1, 3 * iSequence * iVariables)) ReturnFalse; if(!cNormalize.FeedForward(cConcatInput.AsObject())) ReturnFalse; if(!cPatching.FeedForward(cNormalize.AsObject())) ReturnFalse; if(!acPE[0].FeedForward(cPatching.AsObject())) ReturnFalse; if(!cNormalizePLR.FeedForward(cPLR.AsObject())) ReturnFalse; if(!cPatchingPLR.FeedForward(cPatchingPLR.AsObject())) ReturnFalse; if(!acPE[1].FeedForward(cPatchingPLR.AsObject())) ReturnFalse; if(!cAttention.FeedForward(acPE[0].AsObject(), acPE[1].getOutput())) ReturnFalse; if(!cTransposeAtt.FeedForward(cAttention.AsObject())) ReturnFalse; if(!acForecast[0].FeedForward(cTransposeAtt.AsObject())) ReturnFalse; if(!acForecast[1].FeedForward(acForecast[0].AsObject())) ReturnFalse; if(!cTransposeFrc.FeedForward(acForecast[1].AsObject())) ReturnFalse; if(!cRevIn.FeedForward(cTransposeFrc.AsObject())) ReturnFalse; if(!cSum.FeedForward(cRevIn.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronTEMPOOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Devide to Trend, Seasons and Noise if(!cRevIn.CalcHiddenGradients(cSum.AsObject())) ReturnFalse; //--- Forecast gradient if(!cTransposeFrc.CalcHiddenGradients(cRevIn.AsObject())) ReturnFalse; if(!acForecast[1].CalcHiddenGradients(cTransposeFrc.AsObject())) ReturnFalse; if(acForecast[1].Activation() != None && !DeActivation(acForecast[1].getOutput(), acForecast[1].getGradient(), acForecast[1].getGradient(), acForecast[1].Activation()) ) ReturnFalse; if(!acForecast[0].CalcHiddenGradients(acForecast[1].AsObject())) ReturnFalse; //--- Attention gradient if(!cTransposeAtt.CalcHiddenGradients(acForecast[0].AsObject())) ReturnFalse; if(!cAttention.CalcHiddenGradients(cTransposeAtt.AsObject())) ReturnFalse; if(!acPE[0].CalcHiddenGradients(cAttention.AsObject(), acPE[1].getOutput(), acPE[1].getGradient(), (ENUM_ACTIVATION)acPE[1].Activation())) ReturnFalse; //--- Gradient to PLR if(!cPatchingPLR.CalcHiddenGradients(acPE[1].AsObject())) ReturnFalse; if(!cNormalizePLR.CalcHiddenGradients(cPatchingPLR.AsObject())) ReturnFalse; if(!cPLR.CalcHiddenGradients(cNormalizePLR.AsObject())) ReturnFalse; //--- Gradient to Concatenate buffer of Trend, Season and Noise if(!cPatching.CalcHiddenGradients(acPE[0].AsObject())) ReturnFalse; if(!cNormalize.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; if(!cConcatInput.CalcHiddenGradients(cNormalize.AsObject())) ReturnFalse; //--- DeConcatenate if(!DeConcat(cTrend.getGradient(), cOutputTimeSeriasRe.getGradient(), cResidual.getGradient(), cConcatInput.getGradient(), 1, 1, 1, 3 * iSequence * iVariables)) ReturnFalse; //--- Seasons if(!CutOneFromAnotherGradient()) ReturnFalse; if(!SumAndNormalize(cOutputTimeSeriasRe.getGradient(), cTranspose[1].getGradient(), cTranspose[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cOutputTimeSeriasRe.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!Concat(cOutputTimeSeriasRe.getGradient(), GetPointer(cZero), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!SumAndNormalize(GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!FFT(GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexUnNormalizeGradient()) ReturnFalse; if(!cNormFreqComplex.CalcHiddenGradients(cFreqAtteention.AsObject())) ReturnFalse; if(!ComplexNormalizeGradient()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cInputFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), true)) ReturnFalse; if(!DeConcat(cTranspose[0].getGradient(), GetPointer(cInputFreqIm), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!cInputSeasons.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cInputSeasons.getGradient(), cResidual.getGradient(), cInputSeasons.getGradient(), 1, 1, false, 0, 0, 1)) ReturnFalse; //--- trend if(!CutTrendAndOtherGradient(NeuronOCL.getGradient())) ReturnFalse; //--- input gradient if(!NeuronOCL.CalcHiddenGradients(cPLR.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cInputSeasons.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTEMPOOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- trend if(!cPLR.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- seasons if(!cFreqAtteention.UpdateInputWeights(cNormFreqComplex.AsObject())) ReturnFalse; //--- Forecast if(!cNormalize.UpdateInputWeights(cConcatInput.AsObject())) ReturnFalse; if(!cPatching.UpdateInputWeights(cNormalize.AsObject())) ReturnFalse; if(!acPE[0].UpdateInputWeights(cPatching.AsObject())) ReturnFalse; if(!cNormalizePLR.UpdateInputWeights(cPLR.AsObject())) ReturnFalse; if(!cPatchingPLR.UpdateInputWeights(cPatchingPLR.AsObject())) ReturnFalse; if(!acPE[1].UpdateInputWeights(cPatchingPLR.AsObject())) ReturnFalse; if(!cAttention.UpdateInputWeights(acPE[0].AsObject(), acPE[1].getOutput())) ReturnFalse; if(!acForecast[0].UpdateInputWeights(cTransposeAtt.AsObject())) ReturnFalse; if(!acForecast[1].UpdateInputWeights(acForecast[0].AsObject())) ReturnFalse; if(!cSum.UpdateInputWeights(cRevIn.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTEMPOOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- constants if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSequence)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iForecast)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFT)) < INT_VALUE) ReturnFalse; //--- trend if(!cPLR.Save(file_handle)) ReturnFalse; //--- seasons if(!cFreqAtteention.Save(file_handle)) ReturnFalse; //--- Forecast if(!cNormalize.Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!acPE[0].Save(file_handle)) ReturnFalse; if(!cNormalizePLR.Save(file_handle)) ReturnFalse; if(!cPatchingPLR.Save(file_handle)) ReturnFalse; if(!acPE[1].Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(!acForecast[0].Save(file_handle)) ReturnFalse; if(!acForecast[1].Save(file_handle)) ReturnFalse; if(!cSum.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTEMPOOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- constants if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSequence = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFT = (uint)FileReadInteger(file_handle); //--- trend if(!LoadInsideLayer(file_handle, cPLR.AsObject())) ReturnFalse; //--- seasons if(!LoadInsideLayer(file_handle, cFreqAtteention.AsObject())) ReturnFalse; //--- Forecast if(!LoadInsideLayer(file_handle, cNormalize.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acPE[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizePLR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingPLR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acPE[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acForecast[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acForecast[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSum.AsObject())) ReturnFalse; //--- ReInit Objects //--- trend if(!cTrend.Init(0, 1, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- seasons if(!cInputSeasons.Init(0, 2, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[0].Init(0, 3, OpenCL, iSequence, iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[1].Init(0, 4, OpenCL, iVariables, iSequence, optimization, iBatch)) ReturnFalse; cInputFreqRe.BufferFree(); if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cInputFreqIm.BufferFree(); if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 5, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; if(!cNormFreqComplex.Init(0, 6, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMeans.BufferFree(); if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; cVariances.BufferFree(); if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 8, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cOutputFreqRe.BufferFree(); if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cOutputFreqIm.BufferFree(); if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.Init(0, 9, OpenCL, iFFT * iVariables, optimization, iBatch)) ReturnFalse; cOutputTimeSeriasIm.BufferFree(); if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; cZero.BufferFree(); if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- Noise if(!cResidual.Init(0, 10, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- Forecast if(!cConcatInput.Init(0, 11, OpenCL, 3 * iSequence * iVariables, optimization, iBatch)) ReturnFalse; uint patches = cAttention.Neurons() / (3 * 8 * iVariables); if(!cTransposeAtt.Init(0, 19, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!cTransposeFrc.Init(0, 22, OpenCL, 3 * iVariables, iForecast, optimization, iBatch)) ReturnFalse; if(!cRevIn.Init(0, 23, OpenCL, 3 * iVariables * iForecast, 11, GetPointer(cNormalize))) ReturnFalse; //--- SetOutput(cSum.getOutput(), true); SetGradient(cSum.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTEMPOOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTEMPOOCL::SetOpenCL(COpenCLMy * obj) { //--- base CNeuronBaseOCL::SetOpenCL(obj); //--- trend cPLR.SetOpenCL(OpenCL); cTrend.SetOpenCL(OpenCL); //--- seasons cInputSeasons.SetOpenCL(OpenCL); cTranspose[0].SetOpenCL(OpenCL); cTranspose[1].SetOpenCL(OpenCL); cInputFreqRe.BufferCreate(OpenCL); cInputFreqIm.BufferCreate(OpenCL); cInputFreqComplex.SetOpenCL(OpenCL); cNormFreqComplex.SetOpenCL(OpenCL); cMeans.BufferCreate(OpenCL); cVariances.BufferCreate(OpenCL); cFreqAtteention.SetOpenCL(OpenCL); cUnNormFreqComplex.SetOpenCL(OpenCL); cOutputFreqRe.BufferCreate(OpenCL); cOutputFreqIm.BufferCreate(OpenCL); cOutputTimeSeriasRe.SetOpenCL(OpenCL); cOutputTimeSeriasIm.BufferCreate(OpenCL); cZero.BufferCreate(OpenCL); //--- Noise cResidual.SetOpenCL(OpenCL); //--- Forecast cConcatInput.SetOpenCL(OpenCL); cNormalize.SetOpenCL(OpenCL); cPatching.SetOpenCL(OpenCL); acPE[0].SetOpenCL(OpenCL); cNormalizePLR.SetOpenCL(OpenCL); cPatchingPLR.SetOpenCL(OpenCL); acPE[1].SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cTransposeAtt.SetOpenCL(OpenCL); acForecast[0].SetOpenCL(OpenCL); acForecast[1].SetOpenCL(OpenCL); cTransposeFrc.SetOpenCL(OpenCL); cRevIn.SetOpenCL(OpenCL); cSum.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Returns the weights of the CNeuronTEMPOOCL. | //+------------------------------------------------------------------+ CBufferFloat* CNeuronTEMPOOCL::getWeights(void) { CBufferFloat *result = CNeuronBaseOCL::getWeights(); if(!result) result = cSum.GetWeightsConv(); //--- return(result); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTrajLLMOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * forecast, optimization_type, batch)) ReturnFalse; //--- State Encoder if(!cStateRNN.Init(0, 0, OpenCL, window_key, units_count, optimization, iBatch) || !cStateRNN.SetInputs(window)) ReturnFalse; if(!cStateMLP[0].Init(0, 1, OpenCL, window_key, window_key, 4 * window_key, units_count, optimization, iBatch)) ReturnFalse; cStateMLP[0].SetActivationFunction(LReLU); if(!cStateMLP[1].Init(0, 2, OpenCL, 4 * window_key, 4 * window_key, window_key, units_count, optimization, iBatch)) ReturnFalse; //--- Variables Encoder if(!cTranspose.Init(0, 3, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; if(!cVariablesRNN.Init(0, 4, OpenCL, window_key, window, optimization, iBatch) || !cVariablesRNN.SetInputs(units_count)) ReturnFalse; if(!cVariablesMLP[0].Init(0, 5, OpenCL, window_key, window_key, 4 * window_key, window, optimization, iBatch)) ReturnFalse; cVariablesMLP[0].SetActivationFunction(LReLU); if(!cVariablesMLP[1].Init(0, 6, OpenCL, 4 * window_key, 4 * window_key, window_key, window, optimization, iBatch)) ReturnFalse; //--- Position Encoder if(!cStatePE.Init(0, 7, OpenCL, cStateMLP[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cVariablesPE.Init(0, 8, OpenCL, cVariablesMLP[1].Neurons(), optimization, iBatch)) ReturnFalse; //--- Context if(!cStateToState.Init(0, 9, OpenCL, window_key, window_key, heads, heads / 2, units_count, 2, 1, optimization, iBatch)) ReturnFalse; if(!cStateToVariable.Init(0, 10, OpenCL, window_key, window_key, heads, window_key, heads / 2, units_count, window, 2, 1, optimization, iBatch)) ReturnFalse; if(!cVariableToState.Init(0, 11, OpenCL, window_key, window_key, heads, window_key, heads / 2, window, units_count, 2, 1, optimization, iBatch)) ReturnFalse; if(!cContext.Init(0, 12, OpenCL, window_key * (units_count + window), optimization, iBatch)) ReturnFalse; if(!cContextMLP[0].Init(0, 13, OpenCL, window_key, window_key, 4 * window_key, window + units_count, optimization, iBatch)) ReturnFalse; cContextMLP[0].SetActivationFunction(LReLU); if(!cContextMLP[1].Init(0, 14, OpenCL, 4 * window_key, 4 * window_key, window_key, window + units_count, optimization, iBatch)) ReturnFalse; //--- if(!cHighLevelInteraction.Init(0, 15, OpenCL, window_key, window_key, heads, heads / 2, window + units_count, 4, 2, optimization, iBatch)) ReturnFalse; for(int i = 0; i < int(caMamba.Size()); i++) { if(!caMamba[i].Init(0, 16 + i, OpenCL, window_key, 2 * window_key, window + units_count, optimization, iBatch)) ReturnFalse; } if(!cLaneAware.Init(0, 19, OpenCL, window_key, window_key, heads, window_key, heads / 2, window, window + units_count, 2, 1, optimization, iBatch)) ReturnFalse; //--- if(!caForecastMLP[0].Init(0, 20, OpenCL, window_key, window_key, 4 * forecast, window, optimization, iBatch)) ReturnFalse; caForecastMLP[0].SetActivationFunction(LReLU); if(!caForecastMLP[1].Init(0, 21, OpenCL, 4 * forecast, 4 * forecast, forecast, window, optimization, iBatch)) ReturnFalse; caForecastMLP[1].SetActivationFunction(TANH); if(!cTransposeOut.Init(0, 22, OpenCL, window, forecast, optimization, iBatch)) ReturnFalse; //--- SetOutput(cTransposeOut.getOutput(), true); SetGradient(cTransposeOut.getGradient(), true); SetActivationFunction((ENUM_ACTIVATION)caForecastMLP[1].Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTrajLLMOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- State Encoder if(!cStateRNN.FeedForward(NeuronOCL)) ReturnFalse; if(!cStateMLP[0].FeedForward(cStateRNN.AsObject())) ReturnFalse; if(!cStateMLP[1].FeedForward(cStateMLP[0].AsObject())) ReturnFalse; //--- Variables Encoder if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cVariablesRNN.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cVariablesMLP[0].FeedForward(cVariablesRNN.AsObject())) ReturnFalse; if(!cVariablesMLP[1].FeedForward(cVariablesMLP[0].AsObject())) ReturnFalse; //--- Position Encoder if(!cStatePE.FeedForward(cStateMLP[1].AsObject())) ReturnFalse; if(!cVariablesPE.FeedForward(cVariablesMLP[1].AsObject())) ReturnFalse; //--- Context if(!cStateToState.FeedForward(cStatePE.AsObject())) ReturnFalse; if(!cStateToVariable.FeedForward(cStateToState.AsObject(), cVariablesPE.getOutput())) ReturnFalse; if(!cVariableToState.FeedForward(cVariablesPE.AsObject(), cStateToVariable.getOutput())) ReturnFalse; if(!Concat(cStateToVariable.getOutput(), cVariableToState.getOutput(), cContext.getOutput(), cStateToVariable.Neurons(), cVariableToState.Neurons(), 1)) ReturnFalse; if(!cContextMLP[0].FeedForward(cContext.AsObject())) ReturnFalse; if(!cContextMLP[1].FeedForward(cContextMLP[0].AsObject())) ReturnFalse; //--- Lane aware if(!cHighLevelInteraction.FeedForward(cContextMLP[1].AsObject())) ReturnFalse; if(!caMamba[0].FeedForward(cHighLevelInteraction.AsObject())) ReturnFalse; for(int i = 1; i < int(caMamba.Size()); i++) { if(!caMamba[i].FeedForward(caMamba[i - 1].AsObject())) ReturnFalse; } if(!cLaneAware.FeedForward(cVariablesPE.AsObject(), caMamba[caMamba.Size() - 1].getOutput())) ReturnFalse; //--- Forecast if(!caForecastMLP[0].FeedForward(cLaneAware.AsObject())) ReturnFalse; if(!caForecastMLP[1].FeedForward(caForecastMLP[0].AsObject())) ReturnFalse; if(!cTransposeOut.FeedForward(caForecastMLP[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTrajLLMOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Forecast if(!caForecastMLP[1].CalcHiddenGradients(cTransposeOut.AsObject())) ReturnFalse; if(!caForecastMLP[0].CalcHiddenGradients(caForecastMLP[1].AsObject())) ReturnFalse; //--- Lane aware if(!cLaneAware.CalcHiddenGradients(caForecastMLP[0].AsObject())) ReturnFalse; if(!cVariablesPE.CalcHiddenGradients(cLaneAware.AsObject(), caMamba[caMamba.Size() - 1].getOutput(), caMamba[caMamba.Size() - 1].getGradient(), (ENUM_ACTIVATION)caMamba[caMamba.Size() - 1].Activation())) ReturnFalse; //--- for(int i = int(caMamba.Size()) - 2; i >= 0; i--) if(!caMamba[i].CalcHiddenGradients(caMamba[i + 1].AsObject())) ReturnFalse; if(!cHighLevelInteraction.CalcHiddenGradients(caMamba[0].AsObject())) ReturnFalse; //--- Context if(!cContextMLP[1].CalcHiddenGradients(cHighLevelInteraction.AsObject())) ReturnFalse; if(!cContextMLP[0].CalcHiddenGradients(cContextMLP[1].AsObject())) ReturnFalse; if(!cContext.CalcHiddenGradients(cContextMLP[0].AsObject())) ReturnFalse; if(!DeConcat(cStatePE.getGradient(), cVariableToState.getGradient(), cContext.getGradient(), cStateToVariable.Neurons(), cVariableToState.Neurons(), 1)) ReturnFalse; if(!SumAndNormalize(cVariablesPE.getGradient(), cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), 1, false)) ReturnFalse; if(!cVariablesPE.CalcHiddenGradients(cVariableToState.AsObject(), cStateToVariable.getOutput(), cStateToVariable.getGradient(), (ENUM_ACTIVATION)cStateToVariable.Activation())) ReturnFalse; if(!SumAndNormalize(cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), cVariablesMLP[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cStateToVariable.getGradient(), cStatePE.getGradient(), cStateToVariable.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cStateToState.CalcHiddenGradients(cStateToVariable.AsObject(), cVariablesPE.getOutput(), cVariablesPE.getGradient(), (ENUM_ACTIVATION)cVariablesPE.Activation())) ReturnFalse; if(!SumAndNormalize(cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), cVariablesPE.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cStatePE.CalcHiddenGradients(cStateToState.AsObject())) ReturnFalse; //--- Position Encoder if(!cStateMLP[1].CalcHiddenGradients(cStatePE.AsObject())) ReturnFalse; if(!cVariablesMLP[1].CalcHiddenGradients(cVariablesPE.AsObject())) ReturnFalse; //--- Variables Encoder if(!cVariablesMLP[0].CalcHiddenGradients(cVariablesMLP[1].AsObject())) ReturnFalse; if(!cVariablesRNN.CalcHiddenGradients(cVariablesMLP[0].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cVariablesRNN.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), cTranspose.getGradient(), 1, false)) ReturnFalse; //--- State Encoder if(!cStateMLP[0].CalcHiddenGradients(cStateMLP[1].AsObject())) ReturnFalse; if(!cStateRNN.CalcHiddenGradients(cStateMLP[0].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cStateRNN.AsObject())) ReturnFalse; if(!SumAndNormalize(cTranspose.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTrajLLMOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- State Encoder if(!cStateRNN.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cStateMLP[0].UpdateInputWeights(cStateRNN.AsObject())) ReturnFalse; if(!cStateMLP[1].UpdateInputWeights(cStateMLP[0].AsObject())) ReturnFalse; //--- Variables Encoder if(!cVariablesRNN.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; if(!cVariablesMLP[0].UpdateInputWeights(cVariablesRNN.AsObject())) ReturnFalse; if(!cVariablesMLP[1].UpdateInputWeights(cVariablesMLP[0].AsObject())) ReturnFalse; //--- Position Encoder if(!cStatePE.UpdateInputWeights(cStateMLP[1].AsObject())) ReturnFalse; if(!cVariablesPE.UpdateInputWeights(cVariablesMLP[1].AsObject())) ReturnFalse; //--- Context if(!cStateToState.UpdateInputWeights(cStatePE.AsObject())) ReturnFalse; if(!cStateToVariable.UpdateInputWeights(cStateToState.AsObject(), cVariablesPE.getOutput())) ReturnFalse; if(!cVariableToState.UpdateInputWeights(cVariablesPE.AsObject(), cStateToVariable.getOutput())) ReturnFalse; if(!cContextMLP[0].UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cContextMLP[1].UpdateInputWeights(cContextMLP[0].AsObject())) ReturnFalse; //--- Lane aware if(!cHighLevelInteraction.UpdateInputWeights(cContextMLP[1].AsObject())) ReturnFalse; if(!caMamba[0].UpdateInputWeights(cHighLevelInteraction.AsObject())) ReturnFalse; for(int i = 1; i < int(caMamba.Size()); i++) { if(!caMamba[i].UpdateInputWeights(caMamba[i - 1].AsObject())) ReturnFalse; } if(!cLaneAware.UpdateInputWeights(cVariablesPE.AsObject(), caMamba[caMamba.Size() - 1].getOutput())) ReturnFalse; //--- Forecast if(!caForecastMLP[0].UpdateInputWeights(cLaneAware.AsObject())) ReturnFalse; if(!caForecastMLP[1].UpdateInputWeights(caForecastMLP[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTrajLLMOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- State Encoder if(!cStateRNN.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cStateMLP[i].Save(file_handle)) ReturnFalse; //--- Variables Encoder if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cVariablesRNN.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cVariablesMLP[i].Save(file_handle)) ReturnFalse; //--- Context Encoder if(!cStatePE.Save(file_handle)) ReturnFalse; if(!cVariablesPE.Save(file_handle)) ReturnFalse; if(!cStateToState.Save(file_handle)) ReturnFalse; if(!cVariableToState.Save(file_handle)) ReturnFalse; if(!cStateToVariable.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cContextMLP[i].Save(file_handle)) ReturnFalse; //--- if(!cHighLevelInteraction.Save(file_handle)) ReturnFalse; int total = int(caMamba.Size()); if(FileWriteInteger(file_handle, total) < INT_VALUE) ReturnFalse; for(int i = 0; i < total; i++) if(!caMamba[i].Save(file_handle)) ReturnFalse; if(!cLaneAware.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!caForecastMLP[i].Save(file_handle)) ReturnFalse; if(!cTransposeOut.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTrajLLMOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- State Encoder if(!LoadInsideLayer(file_handle, cStateRNN.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cStateMLP[i].AsObject())) ReturnFalse; //--- Variables Encoder if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariablesRNN.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cVariablesMLP[i].AsObject())) ReturnFalse; //--- Context Encoder if(!LoadInsideLayer(file_handle, cStatePE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariablesPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariableToState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToVariable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cContext.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cContextMLP[i].AsObject())) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cHighLevelInteraction.AsObject())) ReturnFalse; int total = FileReadInteger(file_handle); if(total != int(caMamba.Size())) { ReturnFalse; } for(int i = 0; i < total; i++) if(!LoadInsideLayer(file_handle, caMamba[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLaneAware.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, caForecastMLP[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTrajLLMOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTrajLLMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- State Encoder cStateRNN.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cStateMLP[i].SetOpenCL(OpenCL); //--- Variables Encoder cTranspose.SetOpenCL(OpenCL); cVariablesRNN.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cVariablesMLP[i].SetOpenCL(OpenCL); //--- Context Encoder cStatePE.SetOpenCL(OpenCL); cVariablesPE.SetOpenCL(OpenCL); cStateToState.SetOpenCL(OpenCL); cVariableToState.SetOpenCL(OpenCL); cStateToVariable.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cContextMLP[i].SetOpenCL(OpenCL); //--- cHighLevelInteraction.SetOpenCL(OpenCL); int total = int(caMamba.Size()); for(int i = 0; i < total; i++) caMamba[i].SetOpenCL(OpenCL); cLaneAware.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) caForecastMLP[i].SetOpenCL(OpenCL); cTransposeOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUniTraj layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * (units_count + forecast), optimization_type, batch)) ReturnFalse; //--- iVariables = window; fDropout = MathMax(MathMin(dropout, 1), 0); //--- if(!cHistoryMask.BufferInit(iVariables * units_count, 1) || !cHistoryMask.BufferCreate(OpenCL)) ReturnFalse; if(!cFutureMask.BufferInit(iVariables * forecast, 1) || !cFutureMask.BufferCreate(OpenCL)) ReturnFalse; if(!cData.Init(0, 0, OpenCL, 3 * iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cPE.Init(0, 1, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cEncoder.Init(0, 2, OpenCL, 3, window_key, heads, (heads + 1) / 2, iVariables, 1, 1, (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cDForw.Init(0, 3, OpenCL, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse;; if(!cDBakw.Init(0, 4, OpenCL, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cProjDForw.Init(0, 5, OpenCL, 1, 1, 3, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; cProjDForw.SetActivationFunction(SIGMOID); if(!cProjDBakw.Init(0, 6, OpenCL, 1, 1, 3, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; cProjDBakw.SetActivationFunction(SIGMOID); if(!cDataDForw.Init(0, 7, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cDataDBakw.Init(0, 8, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cConcatDataDForwBakw.Init(0, 9, OpenCL, 2 * cData.Neurons(), optimization, iBatch)) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) { if(!cSSM[i].Init(0, 10 + i, OpenCL, 6 * iVariables, 12 * iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; } uint id = 10 + cSSM.Size(); if(!cStat.Init(0, id, OpenCL, 6, 6, 12, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; id++; if(!cTranspStat.Init(0, id, OpenCL, iVariables * (units_count + forecast), 12, optimization, iBatch)) ReturnFalse; id++; if(!cVAE.Init(0, id, OpenCL, cTranspStat.Neurons() / 2, optimization, iBatch)) ReturnFalse; id++; if(!cTranspVAE.Init(0, id, OpenCL, cVAE.Neurons() / iVariables, iVariables, optimization, iBatch)) ReturnFalse; id++; uint w = cTranspVAE.Neurons() / iVariables; if(!cDecoder[0].Init(0, id, OpenCL, w, w, 2 * (units_count + forecast), iVariables, optimization, iBatch)) ReturnFalse; cDecoder[0].SetActivationFunction(LReLU); id++; if(!cDecoder[1].Init(0, id, OpenCL, 2 * (units_count + forecast), 2 * (units_count + forecast), (units_count + forecast), iVariables, optimization, iBatch)) ReturnFalse; cDecoder[1].SetActivationFunction(TANH); id++; if(!cTranspResult.Init(0, id, OpenCL, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cTranspResult.getOutput(), true) || !SetGradient(cTranspResult.getGradient(), true)) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)cDecoder[1].Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Prepares the buffers of the CNeuronUniTraj layer. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Prepare(const CBufferFloat* history, const CBufferFloat* future) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {MathMax(cHistoryMask.Total(), cFutureMask.Total()) / iVariables, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajPrepare, def_k_utp_h_mask, cHistoryMask.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_history, history.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_f_mask, cFutureMask.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_future, (!!future ? future.GetIndex() : cFutureMask.GetIndex())) setBuffer(def_k_UniTrajPrepare, def_k_utp_output, cData.getOutputIndex()) setArgument(def_k_UniTrajPrepare, def_k_utp_h_total, cHistoryMask.Total()) setArgument(def_k_UniTrajPrepare, def_k_utp_f_total, cFutureMask.Total()) kernelExecute(def_k_UniTrajPrepare, global_work_offset, global_work_size) #ifdef _DEBUG if(!cData.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Prepares the grad buffers of the CNeuronUniTraj layer. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::PrepareGrad(CBufferFloat* history_gr, CBufferFloat* future_gr) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {MathMax(cHistoryMask.Total(), cFutureMask.Total()) / iVariables, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_history_gr, history_gr.GetIndex()) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_future_gr, (!!future_gr ? future_gr.GetIndex() : cFutureMask.GetIndex())) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_output, cData.getOutputIndex()) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_output_gr, cData.getGradientIndex()) setArgument(def_k_UniTrajPrepareGrad, def_k_utpg_h_total, cHistoryMask.Total()) setArgument(def_k_UniTrajPrepareGrad, def_k_utpg_f_total, cFutureMask.Total()) kernelExecute(def_k_UniTrajPrepareGrad, global_work_offset, global_work_size) #ifdef _DEBUG if(!history_gr.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the bts step of the CNeuronUniTraj layer. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::BTS(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {2, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajBTS, def_k_utbts_concat_inp, cData.getOutputIndex()) setBuffer(def_k_UniTrajBTS, def_k_utbts_d_forw, cDForw.getOutputIndex()) setBuffer(def_k_UniTrajBTS, def_k_utbts_d_bakw, cDBakw.getOutputIndex()) setArgument(def_k_UniTrajBTS, def_k_utbts_total, int(cDForw.Neurons() / iVariables)) kernelExecute(def_k_UniTrajBTS, global_work_offset, global_work_size) #ifdef _DEBUG if(!cDBakw.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUniTraj feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!NeuronOCL) ReturnFalse; //--- Create History Mask int total = cHistoryMask.Total(); if(!cHistoryMask.BufferInit(total, 1)) ReturnFalse; if(bTrain) { for(int i = 0; i < int(total * fDropout); i++) cHistoryMask.Update(RND(total), 0); } if(!cHistoryMask.BufferWrite()) ReturnFalse; //--- Create Future Mask total = cFutureMask.Total(); if(!cFutureMask.BufferInit(total, (!SecondInput ? 0 : 1))) ReturnFalse; if(bTrain && !!SecondInput) { for(int i = 0; i < int(total * fDropout); i++) cFutureMask.Update(RND(total), 0); } if(!cFutureMask.BufferWrite()) ReturnFalse; //--- Prepare Data if(!Prepare(NeuronOCL.getOutput(), SecondInput)) ReturnFalse; //--- Encoder if(!cPE.FeedForward(cData.AsObject())) ReturnFalse; if(!cEncoder.FeedForward(cPE.AsObject())) ReturnFalse; //--- BTS if(!BTS()) ReturnFalse; if(!cProjDForw.FeedForward(cDForw.AsObject())) ReturnFalse; if(!cProjDBakw.FeedForward(cDBakw.AsObject())) ReturnFalse; if(!ElementMult(cEncoder.getOutput(), cProjDForw.getOutput(), cDataDForw.getOutput())) ReturnFalse; if(!ElementMult(cEncoder.getOutput(), cProjDBakw.getOutput(), cDataDBakw.getOutput())) ReturnFalse; if(!Concat(cDataDForw.getOutput(), cDataDBakw.getOutput(), cConcatDataDForwBakw.getOutput(), 3, 3, cData.Neurons() / 3)) ReturnFalse; //--- SSM if(!cSSM[0].FeedForward(cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 1; i < cSSM.Size(); i++) if(!cSSM[i].FeedForward(cSSM[i - 1].AsObject())) ReturnFalse; //--- VAE if(!cStat.FeedForward(cSSM[cSSM.Size() - 1].AsObject())) ReturnFalse; if(!cTranspStat.FeedForward(cStat.AsObject())) ReturnFalse; if(!cVAE.FeedForward(cTranspStat.AsObject())) ReturnFalse; //--- Decoder if(!cTranspVAE.FeedForward(cVAE.AsObject())) ReturnFalse; if(!cDecoder[0].FeedForward(cTranspVAE.AsObject())) ReturnFalse; if(!cDecoder[1].FeedForward(cDecoder[0].AsObject())) ReturnFalse; if(!cTranspResult.FeedForward(cDecoder[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronUniTraj | //| layer. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { //--- Decoder if(!cDecoder[1].CalcHiddenGradients(cTranspResult.AsObject())) ReturnFalse; if(!cDecoder[0].CalcHiddenGradients(cDecoder[1].AsObject())) ReturnFalse; if(!cTranspVAE.CalcHiddenGradients(cDecoder[0].AsObject())) ReturnFalse; if(!cVAE.CalcHiddenGradients(cTranspVAE.AsObject())) ReturnFalse; //--- VAE if(!cTranspStat.CalcHiddenGradients(cVAE.AsObject())) ReturnFalse; if(!cStat.CalcHiddenGradients(cTranspStat.AsObject())) ReturnFalse; if(!cSSM[cSSM.Size() - 1].CalcHiddenGradients(cStat.AsObject())) ReturnFalse; //--- SSM for(int i = (int)cSSM.Size() - 2; i >= 0; i--) if(!cSSM[i].CalcHiddenGradients(cSSM[i + 1].AsObject())) ReturnFalse; if(!cConcatDataDForwBakw.CalcHiddenGradients(cSSM[0].AsObject())) ReturnFalse; //--- BTS if(!DeConcat(cDataDForw.getGradient(), cDataDBakw.getGradient(), cConcatDataDForwBakw.getGradient(), 3, 3, cData.Neurons() / 3)) ReturnFalse; if(!ElementMultGrad(cEncoder.getOutput(), cEncoder.getGradient(), cProjDForw.getOutput(), cProjDForw.getGradient(), cDataDForw.getGradient(), cEncoder.Activation(), cProjDForw.Activation())) ReturnFalse; if(!ElementMultGrad(cEncoder.getOutput(), cPE.getGradient(), cProjDBakw.getOutput(), cProjDBakw.getGradient(), cDataDBakw.getGradient(), cEncoder.Activation(), cProjDBakw.Activation())) ReturnFalse; if(!SumAndNormalize(cEncoder.getGradient(), cPE.getGradient(), cEncoder.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cDForw.CalcHiddenGradients(cProjDForw.AsObject())) ReturnFalse; if(!cDBakw.CalcHiddenGradients(cProjDBakw.AsObject())) ReturnFalse; //--- Encoder if(!cPE.CalcHiddenGradients(cEncoder.AsObject())) ReturnFalse; if(!cData.CalcHiddenGradients(cPE.AsObject())) ReturnFalse; //--- Prepare Data if(!PrepareGrad(NeuronOCL.getGradient(), SecondGradient)) ReturnFalse; //--- Deactivation Gradient if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; } if(SecondActivation != None && !!SecondInput && !!SecondGradient) { if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUniTraj trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* second) { if(!NeuronOCL) ReturnFalse; //--- Encoder if(!cPE.UpdateInputWeights(cData.AsObject())) ReturnFalse; if(!cEncoder.UpdateInputWeights(cPE.AsObject())) ReturnFalse; //--- BTS if(!cProjDForw.UpdateInputWeights(cDForw.AsObject())) ReturnFalse; if(!cProjDBakw.UpdateInputWeights(cDBakw.AsObject())) ReturnFalse; //--- SSM if(!cSSM[0].UpdateInputWeights(cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 1; i < cSSM.Size(); i++) if(!cSSM[i].UpdateInputWeights(cSSM[i - 1].AsObject())) ReturnFalse; //--- VAE if(!cStat.UpdateInputWeights(cSSM[cSSM.Size() - 1].AsObject())) ReturnFalse; //--- Decoder if(!cDecoder[0].UpdateInputWeights(cTranspVAE.AsObject())) ReturnFalse; if(!cDecoder[1].UpdateInputWeights(cDecoder[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUniTraj state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteFloat(file_handle, fDropout) < sizeof(float)) ReturnFalse; //--- if(!cHistoryMask.Save(file_handle)) ReturnFalse; if(!cFutureMask.Save(file_handle)) ReturnFalse; if(!cData.Save(file_handle)) ReturnFalse; if(!cPE.Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cDForw.Save(file_handle)) ReturnFalse; if(!cDBakw.Save(file_handle)) ReturnFalse; if(!cProjDForw.Save(file_handle)) ReturnFalse; if(!cProjDBakw.Save(file_handle)) ReturnFalse; if(!cDataDForw.Save(file_handle)) ReturnFalse; if(!cDataDBakw.Save(file_handle)) ReturnFalse; if(!cConcatDataDForwBakw.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) if(!cSSM[i].Save(file_handle)) ReturnFalse; if(!cStat.Save(file_handle)) ReturnFalse; if(!cTranspStat.Save(file_handle)) ReturnFalse; if(!cVAE.Save(file_handle)) ReturnFalse; if(!cTranspVAE.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cDecoder[i].Save(file_handle)) ReturnFalse; if(!cTranspResult.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUniTraj state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; fDropout = FileReadFloat(file_handle); //--- if(!cHistoryMask.Load(file_handle)) ReturnFalse; if(!cFutureMask.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cData.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDataDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDataDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) if(!LoadInsideLayer(file_handle, cSSM[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStat.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspStat.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVAE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspVAE.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cDecoder[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspResult.AsObject())) ReturnFalse; //--- if(!SetOutput(cTranspResult.getOutput(), true) || !SetGradient(cTranspResult.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUniTraj GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUniTraj::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cHistoryMask.BufferCreate(OpenCL); cFutureMask.BufferCreate(OpenCL); cData.SetOpenCL(OpenCL); cPE.SetOpenCL(OpenCL); cEncoder.SetOpenCL(OpenCL); cDForw.SetOpenCL(OpenCL); cDBakw.SetOpenCL(OpenCL); cProjDForw.SetOpenCL(OpenCL); cProjDBakw.SetOpenCL(OpenCL); cDataDForw.SetOpenCL(OpenCL); cDataDBakw.SetOpenCL(OpenCL); cConcatDataDForwBakw.SetOpenCL(OpenCL); for(uint i = 0; i < cSSM.Size(); i++) cSSM[i].SetOpenCL(OpenCL); cStat.SetOpenCL(OpenCL); cTranspStat.SetOpenCL(OpenCL); cVAE.SetOpenCL(OpenCL); cTranspVAE.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cDecoder[i].SetOpenCL(OpenCL); cTranspResult.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHiVTAAEncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables; //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits * iVariables; //--- Size of KV tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariables; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //--- Size of weights' matrix of Q tenzor uint kv_weights = (iWindow * iHeadsKV + 1) * iWindowKey; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads * iVariables; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //--- Size of out tensore uint out = iWindow * iUnits * iVariables; //--- Size W0 tensor uint w0 = (iWindowKey * iHeads + 1) * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint gate = (2 * iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint self = (iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Concatenate temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Gate temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Self temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Out if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Gate Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(gate)) ReturnFalse; k = (float)(1 / sqrt(2 * iWindow + 1)); for(uint w = 0; w < gate; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Self temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(self)) ReturnFalse; k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < self; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Gate Momentum temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? gate : 2 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Self Momentum temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? self : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHiVTAAEncoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(10 * i - 6)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, k, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, v, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 10); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Concat out = FF_Tensors.At(i * 10 + 1); if(IsStopped() || !Concat(temp, inputs, out, iWindow, iWindow, iUnits)) ReturnFalse; //--- Gate if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), out, FF_Tensors.At(i * 10 + 2), 2 * iWindow, iWindow, SIGMOID)) ReturnFalse; //--- Self if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), inputs, FF_Tensors.At(i * 10 + 3), iWindow, iWindow, None)) ReturnFalse; //--- Out if(IsStopped() || !GateElementMult(FF_Tensors.At(i * 10 + 3), temp, FF_Tensors.At(i * 10 + 2), FF_Tensors.At(i * 10 + 4))) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronHiVTAAEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Gate out if(IsStopped() || !GateElementMultGrad(FF_Tensors.At(i * 10 + 3), FF_Tensors.At(i * 10 + 8), FF_Tensors.At(i * 10), FF_Tensors.At(i * 10 + 5), FF_Tensors.At(i * 10 + 2), FF_Tensors.At(i * 10 + 7), out_grad, None, None, SIGMOID)) ReturnFalse; //--- Passing gradient through Self CBufferFloat *inp = NULL; CBufferFloat *inp_gr = NULL; if(i == 0) { inp = prevLayer.getOutput(); inp_gr = prevLayer.getGradient(); } else { inp = FF_Tensors.At(i * 10 - 6); inp_gr = FF_Tensors.At(i * 10 - 1); } if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(i * 10 + 8), inp, inp_gr, iWindow, iWindow, None)) ReturnFalse; //--- Passing gradient through Gate if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 10 + 7), FF_Tensors.At(i * 10 + 1), FF_Tensors.At(i * 10 + 6), 2 * iWindow, iWindow, None)) ReturnFalse; //--- Concat if(IsStopped() || !DeConcat(FF_Tensors.At(i * 10 + 5), GetPointer(Temp), FF_Tensors.At(i * 10 + 6), iWindow, iWindow, iUnits)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), FF_Tensors.At(i * 10 + 5), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, GetPointer(Temp), iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = inp_gr; } //--- if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHiVTAAEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 10 + 5), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 10 + 7), FF_Tensors.At(l * 10 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), 2 * iWindow, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 10 + 8), inputs, (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 10 + 4); } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHiVTOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, uint num_traj, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(units_count < 2 || !CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * forecast, optimization_type, batch)) ReturnFalse; //--- iVariables = window; iHistory = units_count - 1; iForecast = forecast; iNumTraj = MathMax(num_traj, 1); //--- if(!cDataTAD.Init(0, 0, OpenCL, 2 * iVariables * iVariables * iHistory, optimization, iBatch)) ReturnFalse; if(!cEmbeddingTAD.Init(0, 1, OpenCL, 2 * iVariables, 2 * iVariables, window_key, iVariables * iHistory, 1, optimization, iBatch)) ReturnFalse; if(!cTransposeATD.Init(0, 2, OpenCL, iHistory, iVariables, window_key, optimization, iBatch)) ReturnFalse; if(!cAAEncoder.Init(0, 3, OpenCL, window_key, window_key, heads, (heads + 1) / 2, iVariables, 2, 1, iHistory, optimization, iBatch)) ReturnFalse; if(!cTransposeTAD.Init(0, 4, OpenCL, iVariables, iHistory, window_key, optimization, iBatch)) ReturnFalse; if(!cPosEmbeddingTAD.Init(0, 5, OpenCL, iVariables * iHistory * window_key, optimization, iBatch)) ReturnFalse; if(!cTemporalEncoder.Init(0, 6, OpenCL, window_key, window_key, heads, (heads + 1) / 2, iHistory, 2, 1, iVariables, optimization, iBatch)) ReturnFalse; if(!cLineEmbeddibg.Init(0, 7, OpenCL, 3, 1, 8, iHistory - 1, iVariables, optimization, iBatch)) ReturnFalse; if(!cPosLineEmbeddingTAD.Init(0, 8, OpenCL, cLineEmbeddibg.Neurons(), optimization, iBatch)) ReturnFalse; if(!cALEncoder.Init(0, 9, OpenCL, window_key, window_key, heads, 8, (heads + 1) / 2, iHistory, iHistory - 1, 2, 1, iVariables, iVariables, optimization, iBatch)) ReturnFalse; if(!cGlobalEncoder.Init(0, 10, OpenCL, window_key * iVariables, window_key * iVariables, heads, (heads + 1) / 2, iHistory, 4, 2, optimization, iBatch)) ReturnFalse; if(!cTransposeADT.Init(0, 11, OpenCL, iHistory, window_key * iVariables, optimization, iBatch)) ReturnFalse; if(!cDecoder[0].Init(0, 12, OpenCL, iHistory, iHistory, iForecast, window_key * iVariables, optimization, iBatch)) ReturnFalse; cDecoder[0].SetActivationFunction(SIGMOID); if(!cDecoder[1].Init(0, 13, OpenCL, iForecast * window_key, iForecast * window_key, iForecast * window_key, iVariables, optimization, iBatch)) ReturnFalse; cDecoder[1].SetActivationFunction(LReLU); if(!cDecoder[2].Init(0, 14, OpenCL, iForecast * window_key, iForecast * window_key, iForecast * iNumTraj, iVariables, optimization, iBatch)) ReturnFalse; cDecoder[2].SetActivationFunction(TANH); if(!cProbProj.Init(0, 15, OpenCL, iForecast * iNumTraj, iForecast * iNumTraj, iNumTraj, iVariables, optimization, iBatch)) ReturnFalse; if(!cProbability.Init(0, 16, OpenCL, iForecast * iNumTraj * iVariables, optimization, iBatch)) ReturnFalse; //--- Agent* Traj cProbability.SetHeads(iVariables); if(!cForecast.Init(0, 17, OpenCL, iForecast * iVariables, optimization, iBatch)) ReturnFalse; if(!cTransposeTA.Init(0, 18, OpenCL, iVariables, iForecast, optimization, iBatch)) ReturnFalse; //--- SetOutput(cTransposeTA.getOutput(), true); SetGradient(cTransposeTA.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Prepares the buffers of the CNeuronHiVTOCL layer. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Prepare(const CNeuronBaseOCL* history) { if(!history) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iHistory, iVariables}; setBuffer(def_k_HiVTPrepare, def_k_hivtp_data, history.getOutputIndex()) setBuffer(def_k_HiVTPrepare, def_k_hivtp_output, cDataTAD.getOutputIndex()) kernelExecute(def_k_HiVTPrepare, global_work_offset, global_work_size) #ifdef _DEBUG if(!cDataTAD.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHiVTOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!Prepare(NeuronOCL)) ReturnFalse; if(!cEmbeddingTAD.FeedForward(cDataTAD.AsObject())) ReturnFalse; if(!cTransposeATD.FeedForward(cEmbeddingTAD.AsObject())) ReturnFalse; if(!cAAEncoder.FeedForward(cTransposeATD.AsObject())) ReturnFalse; if(!cTransposeTAD.FeedForward(cAAEncoder.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.FeedForward(cTransposeTAD.AsObject())) ReturnFalse; if(!cTemporalEncoder.FeedForward(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cLineEmbeddibg.FeedForward(NeuronOCL)) ReturnFalse; if(!cPosLineEmbeddingTAD.FeedForward(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cALEncoder.FeedForward(cTemporalEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput())) ReturnFalse; if(!cGlobalEncoder.FeedForward(cALEncoder.AsObject())) ReturnFalse; if(!cTransposeADT.FeedForward(cGlobalEncoder.AsObject())) ReturnFalse; if(!cDecoder[0].FeedForward(cTransposeADT.AsObject())) ReturnFalse; if(!cDecoder[1].FeedForward(cDecoder[0].AsObject())) ReturnFalse; if(!cDecoder[2].FeedForward(cDecoder[1].AsObject())) ReturnFalse; if(!cProbProj.FeedForward(cDecoder[2].AsObject())) ReturnFalse; if(!cProbability.FeedForward(cProbProj.AsObject())) ReturnFalse; if(IsStopped() || !MatMul(cDecoder[2].getOutput(), cProbability.getOutput(), cForecast.getOutput(), iForecast, iNumTraj, 1, iVariables)) ReturnFalse; if(!cTransposeTA.FeedForward(cForecast.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronHiVTOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cForecast.CalcHiddenGradients(cTransposeTA.AsObject())) ReturnFalse; if(IsStopped() || !MatMulGrad(cDecoder[2].getOutput(), cDecoder[2].getGradient(), cProbability.getOutput(), cProbability.getGradient(), cForecast.getGradient(), iForecast, iNumTraj, 1, iVariables)) ReturnFalse; if(!cProbProj.CalcHiddenGradients(cProbability.AsObject())) ReturnFalse; if(!cDecoder[1].CalcHiddenGradients(cDecoder[2].AsObject())) ReturnFalse; if(!cDecoder[0].CalcHiddenGradients(cDecoder[1].AsObject())) ReturnFalse; if(!cTransposeADT.CalcHiddenGradients(cDecoder[0].AsObject())) ReturnFalse; if(!cGlobalEncoder.CalcHiddenGradients(cTransposeADT.AsObject())) ReturnFalse; if(!cALEncoder.CalcHiddenGradients(cGlobalEncoder.AsObject())) ReturnFalse; if(!cTemporalEncoder.CalcHiddenGradients(cALEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput(), cPosLineEmbeddingTAD.getGradient(), (ENUM_ACTIVATION)cPosLineEmbeddingTAD.Activation())) ReturnFalse; if(!cLineEmbeddibg.CalcHiddenGradients(cPosLineEmbeddingTAD.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.CalcHiddenGradients(cTemporalEncoder.AsObject())) ReturnFalse; if(!cTransposeTAD.CalcHiddenGradients(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cAAEncoder.CalcHiddenGradients(cTransposeTAD.AsObject())) ReturnFalse; if(!cTransposeATD.CalcHiddenGradients(cAAEncoder.AsObject())) ReturnFalse; if(!cEmbeddingTAD.CalcHiddenGradients(cTransposeATD.AsObject())) ReturnFalse; if(!cDataTAD.CalcHiddenGradients(cEmbeddingTAD.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHiVTOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cEmbeddingTAD.UpdateInputWeights(cDataTAD.AsObject())) ReturnFalse; if(!cAAEncoder.UpdateInputWeights(cTransposeATD.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.UpdateInputWeights(cTransposeTAD.AsObject())) ReturnFalse; if(!cTemporalEncoder.UpdateInputWeights(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cLineEmbeddibg.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPosLineEmbeddingTAD.UpdateInputWeights(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cALEncoder.UpdateInputWeights(cTemporalEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput())) ReturnFalse; if(!cGlobalEncoder.UpdateInputWeights(cALEncoder.AsObject())) ReturnFalse; if(!cDecoder[0].UpdateInputWeights(cTransposeADT.AsObject())) ReturnFalse; if(!cDecoder[1].UpdateInputWeights(cDecoder[0].AsObject())) ReturnFalse; if(!cDecoder[2].UpdateInputWeights(cDecoder[1].AsObject())) ReturnFalse; if(!cProbProj.UpdateInputWeights(cDecoder[2].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronHiVTOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHistory)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iForecast)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iNumTraj)) < INT_VALUE) ReturnFalse; //--- if(!cDataTAD.Save(file_handle)) ReturnFalse; if(!cEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cTransposeATD.Save(file_handle)) ReturnFalse; if(!cAAEncoder.Save(file_handle)) ReturnFalse; if(!cTransposeTAD.Save(file_handle)) ReturnFalse; if(!cPosEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cTemporalEncoder.Save(file_handle)) ReturnFalse; if(!cLineEmbeddibg.Save(file_handle)) ReturnFalse; if(!cPosLineEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cALEncoder.Save(file_handle)) ReturnFalse; if(!cGlobalEncoder.Save(file_handle)) ReturnFalse; if(!cTransposeADT.Save(file_handle)) ReturnFalse; if(!cDecoder[0].Save(file_handle)) ReturnFalse; if(!cDecoder[1].Save(file_handle)) ReturnFalse; if(!cDecoder[2].Save(file_handle)) ReturnFalse; if(!cProbProj.Save(file_handle)) ReturnFalse; if(!cProbability.Save(file_handle)) ReturnFalse; if(!cForecast.Save(file_handle)) ReturnFalse; if(!cTransposeTA.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHiVTOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iNumTraj = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cDataTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeATD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAAEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLineEmbeddibg.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPosLineEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cALEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGlobalEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeADT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[2].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProbProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProbability.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cForecast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeTA.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronHiVTOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronHiVTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDataTAD.SetOpenCL(OpenCL); cEmbeddingTAD.SetOpenCL(OpenCL); cTransposeATD.SetOpenCL(OpenCL); cAAEncoder.SetOpenCL(OpenCL); cTransposeTAD.SetOpenCL(OpenCL); cPosEmbeddingTAD.SetOpenCL(OpenCL); cTemporalEncoder.SetOpenCL(OpenCL); cLineEmbeddibg.SetOpenCL(OpenCL); cPosLineEmbeddingTAD.SetOpenCL(OpenCL); cALEncoder.SetOpenCL(OpenCL); cGlobalEncoder.SetOpenCL(OpenCL); cTransposeADT.SetOpenCL(OpenCL); cDecoder[0].SetOpenCL(OpenCL); cDecoder[1].SetOpenCL(OpenCL); cDecoder[2].SetOpenCL(OpenCL); cProbProj.SetOpenCL(OpenCL); cProbability.SetOpenCL(OpenCL); cForecast.SetOpenCL(OpenCL); cTransposeTA.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronPointNetOCL GPU resources. | //+------------------------------------------------------------------+ CNeuronPointNetOCL::~CNeuronPointNetOCL(void) { DeleteObj(cTNet1); DeleteObj(cTNet2); DeleteObj(cTurned1); DeleteObj(cTurned2); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPointNetOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, output, optimization_type, batch)) ReturnFalse; //--- Init T-Nets if(use_tnets) { if(!cTNet1) { cTNet1 = new CNeuronPointNetOCL(); if(!cTNet1) ReturnFalse; } if(!cTNet1.Init(0, 0, OpenCL, window, units_count, window * window, false, optimization, iBatch)) ReturnFalse; if(!cTurned1) { cTurned1 = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } if(!cTurned1.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; if(!cTNet2) { cTNet2 = new CNeuronPointNetOCL(); if(!cTNet2) ReturnFalse; } if(!cTNet2.Init(0, 2, OpenCL, 64, units_count, 64 * 64, false, optimization, iBatch)) ReturnFalse; if(!cTurned2) { cTurned2 = new CNeuronBaseOCL(); if(!cTurned2) ReturnFalse; } if(!cTurned2.Init(0, 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } //--- Init PreNet if(!cPreNet[0].Init(0, 0, OpenCL, window, window, 64, units_count, optimization, iBatch)) ReturnFalse; cPreNet[0].SetActivationFunction(None); if(!cPreNetNorm[0].Init(0, 1, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cPreNetNorm[0].SetActivationFunction(LReLU); if(!cPreNet[1].Init(0, 2, OpenCL, 64, 64, 64, units_count, optimization, iBatch)) ReturnFalse; cPreNet[1].SetActivationFunction(None); if(!cPreNetNorm[1].Init(0, 3, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cPreNetNorm[1].SetActivationFunction(None); //--- Init Feature Net if(!cFeatureNet[0].Init(0, 4, OpenCL, 64, 64, 64, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[0].SetActivationFunction(None); if(!cFeatureNetNorm[0].Init(0, 5, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNet[0].SetActivationFunction(LReLU); if(!cFeatureNet[1].Init(0, 6, OpenCL, 64, 64, 128, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[1].SetActivationFunction(None); if(!cFeatureNetNorm[1].Init(0, 7, OpenCL, 128 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[1].SetActivationFunction(LReLU); if(!cFeatureNet[2].Init(0, 8, OpenCL, 128, 128, 512, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[2].SetActivationFunction(None); if(!cFeatureNetNorm[2].Init(0, 9, OpenCL, 512 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[2].SetActivationFunction(None); if(!cTranspose.Init(0, 10, OpenCL, units_count, 512, optimization, iBatch)) ReturnFalse; if(!cMaxPool.Init(512, 11, OpenCL, units_count, units_count, 512, optimization, iBatch)) ReturnFalse; //--- Init Final MLP if(!cFinalMLP[0].Init(256, 12, OpenCL, 512, optimization, iBatch)) ReturnFalse; cFinalMLP[0].SetActivationFunction(LReLU); if(!cFinalMLP[1].Init(output, 13, OpenCL, 256, optimization, iBatch)) ReturnFalse; cFinalMLP[1].SetActivationFunction(LReLU); //--- SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPointNetOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- PreNet if(!cTNet1) { if(!cPreNet[0].FeedForward(NeuronOCL)) ReturnFalse; } else { if(!cTurned1) ReturnFalse; if(!cTNet1.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNet1.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNet1.getOutput(), cTurned1.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!cPreNet[0].FeedForward(cTurned1.AsObject())) ReturnFalse; } if(!cPreNetNorm[0].FeedForward(cPreNet[0].AsObject())) ReturnFalse; if(!cPreNet[1].FeedForward(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cPreNetNorm[1].FeedForward(cPreNet[1].AsObject())) ReturnFalse; //--- Feature Net if(!cTNet2) { if(!cFeatureNet[0].FeedForward(cPreNetNorm[1].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTNet2.FeedForward(cPreNetNorm[1].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet2.Neurons()); if(IsStopped() || !MatMul(cPreNetNorm[1].getOutput(), cTNet2.getOutput(), cTurned2.getOutput(), cPreNetNorm[1].Neurons() / window, window, window)) ReturnFalse; if(!cFeatureNet[0].FeedForward(cTurned2.AsObject())) ReturnFalse; } if(!cFeatureNetNorm[0].FeedForward(cFeatureNet[0].AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = 1; i < total; i++) { if(!cFeatureNet[i].FeedForward(cFeatureNetNorm[i - 1].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i].FeedForward(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cTranspose.FeedForward(cFeatureNetNorm[total - 1].AsObject())) ReturnFalse; if(!cMaxPool.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cFinalMLP[0].FeedForward(cMaxPool.AsObject())) ReturnFalse; if(!cFinalMLP[1].FeedForward(cFinalMLP[0].AsObject())) ReturnFalse; if(!CNeuronBaseOCL::feedForward(cFinalMLP[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the orthoganal loss step of the CNeuronPointNetOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::OrthoganalLoss(CNeuronBaseOCL* NeuronOCL, bool add = false) { if(!NeuronOCL) ReturnFalse; //--- uint dimension = (uint)MathSqrt(NeuronOCL.Neurons()); uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {dimension, dimension}; uint local_work_size[2] = {1, dimension}; int kernel = def_k_OrthoganalLoss; setBuffer(kernel, def_k_ol_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ol_grad, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_ol_add, (int)add) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPointNetOCL layer. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronBaseOCL::calcInputGradients(cFinalMLP[1].AsObject())) ReturnFalse; if(!cFinalMLP[0].CalcHiddenGradients(cFinalMLP[1].AsObject())) ReturnFalse; //--- Feature Net if(!cMaxPool.CalcHiddenGradients(cFinalMLP[0].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cMaxPool.AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = total - 1; i > 0; i--) { if(!cFeatureNet[i].CalcHiddenGradients(cFeatureNetNorm[i].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i - 1].CalcHiddenGradients(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFeatureNet[0].CalcHiddenGradients(cFeatureNetNorm[0].AsObject())) ReturnFalse; if(!cTNet2) { if(!cPreNetNorm[1].CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTurned2.CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet2.Neurons()); if(IsStopped() || !MatMulGrad(cPreNetNorm[1].getOutput(), cPreNet[1].getGradient(), cTNet2.getOutput(), cTNet2.getGradient(), cTurned2.getGradient(), cPreNetNorm[1].Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNet2.AsObject(), true)) ReturnFalse; if(!cPreNetNorm[1].CalcHiddenGradients((CObject*)cTNet2)) ReturnFalse; if(!SumAndNormalize(cPreNetNorm[1].getGradient(), cPreNet[1].getGradient(), cPreNetNorm[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- PreNet if(!cPreNet[1].CalcHiddenGradients(cPreNetNorm[1].AsObject())) ReturnFalse; if(!cPreNetNorm[0].CalcHiddenGradients(cPreNet[1].AsObject())) ReturnFalse; if(!cPreNet[0].CalcHiddenGradients(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cTNet1) { if(!NeuronOCL.CalcHiddenGradients(cPreNet[0].AsObject())) ReturnFalse; } else { if(!cTurned1) ReturnFalse; if(!cTurned1.CalcHiddenGradients(cPreNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet1.Neurons()); if(IsStopped() || !MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTNet1.getOutput(), cTNet1.getGradient(), cTurned1.getGradient(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNet1, true)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); NeuronOCL.SetGradient(cTurned1.getGradient(), false); cTurned1.SetGradient(temp, false); //--- if(!NeuronOCL.CalcHiddenGradients(cTNet1.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cTurned1.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPointNetOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- PreNet if(!cTNet1) { if(!cPreNet[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; } else { if(!cTNet1.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPreNet[0].UpdateInputWeights(cTurned1.AsObject())) ReturnFalse; } if(!cPreNetNorm[0].UpdateInputWeights(cPreNet[0].AsObject())) ReturnFalse; if(!cPreNet[1].UpdateInputWeights(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cPreNetNorm[1].UpdateInputWeights(cPreNet[1].AsObject())) ReturnFalse; //--- Feature Net if(!cTNet2) { if(!cFeatureNet[0].UpdateInputWeights(cPreNetNorm[1].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTNet2.UpdateInputWeights(cPreNetNorm[1].AsObject())) ReturnFalse; if(!cFeatureNet[0].UpdateInputWeights(cTurned2.AsObject())) ReturnFalse; } if(!cFeatureNetNorm[0].UpdateInputWeights(cFeatureNet[0].AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = 1; i < total; i++) { if(!cFeatureNet[i].UpdateInputWeights(cFeatureNetNorm[i - 1].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i].UpdateInputWeights(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFinalMLP[0].UpdateInputWeights(cMaxPool.AsObject())) ReturnFalse; if(!cFinalMLP[1].UpdateInputWeights(cFinalMLP[0].AsObject())) ReturnFalse; if(!CNeuronBaseOCL::updateInputWeights(cFinalMLP[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPointNetOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!(!cTNet1 || !cTNet2)) { if(FileWriteInteger(file_handle, 1) < INT_VALUE) ReturnFalse; int units = int(cTurned1.Neurons() / MathSqrt(cTNet1.Neurons())); if(FileWriteInteger(file_handle, units) < INT_VALUE) ReturnFalse; if(!cTNet1.Save(file_handle)) ReturnFalse; if(!cTNet2.Save(file_handle)) ReturnFalse; } else if(FileWriteInteger(file_handle, 0) < INT_VALUE) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!cPreNet[i].Save(file_handle)) ReturnFalse; if(!cPreNetNorm[i].Save(file_handle)) ReturnFalse; } for(int i = 0; i < 3; i++) { if(!cFeatureNet[i].Save(file_handle)) ReturnFalse; if(!cFeatureNetNorm[i].Save(file_handle)) ReturnFalse; } if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cMaxPool.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFinalMLP[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPointNetOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; bool use_tnets = bool(FileReadInteger(file_handle)); if(use_tnets) { int units = FileReadInteger(file_handle); if(!cTNet1) { cTNet1 = new CNeuronPointNetOCL(); if(!cTNet1) ReturnFalse; } if(!cTNet1.Init(0, 0, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNet1)) ReturnFalse; if(!cTurned1) { cTurned1 = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } int dimension = (int)MathSqrt(cTNet1.Neurons()); if(!cTurned1.Init(0, 1, OpenCL, dimension * units, optimization, iBatch)) ReturnFalse; //--- if(!cTNet2) { cTNet2 = new CNeuronPointNetOCL(); if(!cTNet2) ReturnFalse; } if(!cTNet2.Init(0, 2, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNet2)) ReturnFalse; if(!cTurned2) { cTurned2 = new CNeuronBaseOCL(); if(!cTurned2) ReturnFalse; } dimension = (int)MathSqrt(cTNet2.Neurons()); if(!cTurned2.Init(0, 3, OpenCL, dimension * units, optimization, iBatch)) ReturnFalse; } //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cPreNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPreNetNorm[i].AsObject())) ReturnFalse; } for(int i = 0; i < 3; i++) { if(!LoadInsideLayer(file_handle, cFeatureNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureNetNorm[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMaxPool.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cFinalMLP[i].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPointNetOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPointNetOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!cTNet1) cTNet1.SetOpenCL(OpenCL); if(!!cTurned1) cTurned1.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) { cPreNet[i].SetOpenCL(OpenCL); cPreNetNorm[i].SetOpenCL(OpenCL); } if(!!cTNet2) cTNet2.SetOpenCL(OpenCL); if(!!cTurned2) cTurned2.SetOpenCL(OpenCL); for(int i = 0; i < 3; i++) { cFeatureNet[i].SetOpenCL(OpenCL); cFeatureNetNorm[i].SetOpenCL(OpenCL); } cTranspose.SetOpenCL(OpenCL); cMaxPool.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cFinalMLP[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPointNet2Local layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint window_out, float radius, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, 128, 128, window_out, units_count, 1, optimization_type, batch)) ReturnFalse; //--- fRadius = MathMax(0.1f, radius); iUnits = units_count; //--- cDistance.BufferFree(); if(!cDistance.BufferInit(iUnits * iUnits, 0) || !cDistance.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cFeatureNet[0].Init(0, 0, OpenCL, window, window, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[0].Init(0, 1, OpenCL, 64 * iUnits, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[0].SetActivationFunction(LReLU); if(!cFeatureNet[1].Init(0, 2, OpenCL, 64, 64, 128, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[1].Init(0, 3, OpenCL, 128 * iUnits, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[1].SetActivationFunction(LReLU); if(!cFeatureNet[2].Init(0, 4, OpenCL, 128, 128, 256, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[2].Init(0, 5, OpenCL, 256 * iUnits, iBatch, optimization)) cFeatureNetNorm[2].SetActivationFunction(None); //--- if(!cLocalMaxPool.Init(0, 6, OpenCL, cFeatureNetNorm[2].Neurons(), optimization, iBatch)) ReturnFalse; if(!cFinalMLP.Init(0, 7, OpenCL, 256, 256, 128, iUnits, 1, optimization, iBatch)) ReturnFalse; cFinalMLP.SetActivationFunction(LReLU); //--- SetActivationFunction(None); return(true); } //+------------------------------------------------------------------+ //| Executes the calc distance step of the CNeuronPointNet2Local | //| layer. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::CalcDistance(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- return(CNeuronBaseOCL::CalcDistance(NeuronOCL.getOutput(), GetPointer(cDistance), iWindow, iUnits)); } //+------------------------------------------------------------------+ //| Executes the local max pool step of the CNeuronPointNet2Local | //| layer. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::LocalMaxPool(void) { //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_FeedForwardLocalMax; setBuffer(kernel, def_k_fflm_matrix_i, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getOutputIndex()) setBuffer(kernel, def_k_fflm_distance, cDistance.GetIndex()) setBuffer(kernel, def_k_fflm_matrix_o, cLocalMaxPool.getOutputIndex()) setArgument(kernel, def_k_fflm_radius, fRadius) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cLocalMaxPool.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the local max pool grad step of the | //| CNeuronPointNet2Local layer. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::LocalMaxPoolGrad(void) { //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_CalcInputGradientLocalMax; setBuffer(kernel, def_k_cglm_matrix_i, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getOutputIndex()) setBuffer(kernel, def_k_cglm_matrix_ig, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getGradientIndex()) setBuffer(kernel, def_k_cglm_distance, cDistance.GetIndex()) setBuffer(kernel, def_k_cglm_matrix_o, cLocalMaxPool.getOutputIndex()) setBuffer(kernel, def_k_cglm_matrix_g, cLocalMaxPool.getGradientIndex()) setArgument(kernel, def_k_cglm_radius, fRadius) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPointNet2Local feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!CalcDistance(NeuronOCL)) ReturnFalse; CNeuronBaseOCL *temp = NeuronOCL; uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].FeedForward(temp)) ReturnFalse; if(!cFeatureNetNorm[i].FeedForward(cFeatureNet[i].AsObject())) ReturnFalse; temp = cFeatureNetNorm[i].AsObject(); } if(!LocalMaxPool()) ReturnFalse; if(!cFinalMLP.FeedForward(cLocalMaxPool.AsObject())) ReturnFalse; if(!CNeuronConvOCL::feedForward(cFinalMLP.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPointNet2Local layer. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cFinalMLP.AsObject())) ReturnFalse; if(!cLocalMaxPool.CalcHiddenGradients(cFinalMLP.AsObject())) ReturnFalse; if(!LocalMaxPoolGrad()) ReturnFalse; int total = (int)cFeatureNet.Size() - 1; for(int i = total; i > 0; i--) { if(!cFeatureNet[i].CalcHiddenGradients(cFeatureNetNorm[i].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i - 1].CalcHiddenGradients(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFeatureNet[0].CalcHiddenGradients(cFeatureNetNorm[0].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPointNet2Local trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *temp = NeuronOCL; uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].UpdateInputWeights(temp)) ReturnFalse; if(!cFeatureNetNorm[i].UpdateInputWeights(cFeatureNet[i].AsObject())) ReturnFalse; temp = cFeatureNetNorm[i].AsObject(); } if(!cFinalMLP.UpdateInputWeights(cLocalMaxPool.AsObject())) ReturnFalse; if(!CNeuronConvOCL::updateInputWeights(cFinalMLP.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPointNet2Local state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteFloat(file_handle, fRadius) < sizeof(float)) ReturnFalse; if(FileWriteInteger(file_handle, iUnits) < INT_VALUE) ReturnFalse; //--- Save objects uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].Save(file_handle)) ReturnFalse; if(!cFeatureNetNorm[i].Save(file_handle)) ReturnFalse; } if(!cFinalMLP.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPointNet2Local state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- Load constants fRadius = FileReadFloat(file_handle); iUnits = FileReadInteger(file_handle); //--- Load objects uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!LoadInsideLayer(file_handle, cFeatureNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureNetNorm[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cFinalMLP.AsObject())) ReturnFalse; //--- cDistance.BufferFree(); if(!cDistance.BufferInit(iUnits * iUnits, 0) || !cDistance.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cLocalMaxPool.Init(0, 6, OpenCL, cFeatureNetNorm[2].Neurons(), optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPointNet2Local GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronPointNet2Local::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { cFeatureNet[i].SetOpenCL(OpenCL); cFeatureNetNorm[i].SetOpenCL(OpenCL); } cFinalMLP.SetOpenCL(OpenCL); cDistance.BufferCreate(OpenCL); cLocalMaxPool.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronPointNet2OCL GPU resources. | //+------------------------------------------------------------------+ CNeuronPointNet2OCL::~CNeuronPointNet2OCL(void) { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPointNet2OCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronPointNetOCL::Init(numOutputs, myIndex, open_cl, 64, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; //--- Init T-Nets if(use_tnets) { if(!cTNetG) { cTNetG = new CNeuronPointNetOCL(); if(!cTNetG) ReturnFalse; } if(!cTNetG.Init(0, 0, OpenCL, window, units_count, window * window, false, optimization, iBatch)) ReturnFalse; if(!cTurnedG) { cTurnedG = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } if(!cTurnedG.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; } else { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //--- if(!caLocalPointNet[0].Init(0, 0, OpenCL, window, units_count, 64, 0.1f, optimization, iBatch)) ReturnFalse; if(!caLocalPointNet[1].Init(0, 0, OpenCL, 64, units_count, 64, 0.3f, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPointNet2OCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet if(!cTNetG) { if(!caLocalPointNet[0].FeedForward(NeuronOCL)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[0].FeedForward(cTurnedG.AsObject())) ReturnFalse; } if(!caLocalPointNet[1].FeedForward(caLocalPointNet[0].AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(caLocalPointNet[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPointNet2OCL layer. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronPointNetOCL::calcInputGradients(caLocalPointNet[1].AsObject())) ReturnFalse; //--- Local Net if(!caLocalPointNet[0].CalcHiddenGradients(caLocalPointNet[1].AsObject())) ReturnFalse; if(!cTNetG) { if(!NeuronOCL.CalcHiddenGradients(caLocalPointNet[0].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); NeuronOCL.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!NeuronOCL.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cTurnedG.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPointNet2OCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet if(!cTNetG) { if(!caLocalPointNet[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caLocalPointNet[0].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } if(!caLocalPointNet[1].UpdateInputWeights(caLocalPointNet[0].AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(caLocalPointNet[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPointNet2OCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPointNet2OCL::SetOpenCL(COpenCLMy * obj) { CNeuronPointNetOCL::SetOpenCL(obj); if(!!cTNetG) { cTNetG.SetOpenCL(OpenCL); cTurnedG.SetOpenCL(OpenCL); } caLocalPointNet[0].SetOpenCL(OpenCL); caLocalPointNet[1].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronPointNet2OCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Save(const int file_handle) { if(!CNeuronPointNetOCL::Save(file_handle)) ReturnFalse; if(!caLocalPointNet[0].Save(file_handle)) ReturnFalse; if(!caLocalPointNet[1].Save(file_handle)) ReturnFalse; bool use_tnet = bool(!!cTNetG); if(FileWriteInteger(file_handle, int(use_tnet)) < INT_VALUE) ReturnFalse; if(use_tnet) { if(!cTNetG.Save(file_handle)) ReturnFalse; if(!cTurnedG.Save(file_handle)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPointNet2OCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Load(const int file_handle) { if(!CNeuronPointNetOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPointNet[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPointNet[1].AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; bool use_tnet = bool(FileReadInteger(file_handle)); if(use_tnet) { if(!cTNetG) { cTNetG = new CNeuronPointNetOCL(); if(!cTNetG) ReturnFalse; } if(!cTNetG.Init(0, 0, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNetG)) ReturnFalse; if(!cTurnedG) { cTurnedG = new CNeuronBaseOCL(); if(!cTurnedG) ReturnFalse; } if(!cTurnedG.Init(0, 1, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTurnedG)) ReturnFalse; } else { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPointFormer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Init(0, i * 5, OpenCL, 64, 16, 4, units_count, 2, optimization, iBatch)) ReturnFalse; if(!caLocalGlobalAttention[i].Init(0, i * 5 + 1, OpenCL, 64, 16, 4, 64, 2, units_count, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caGlobalAttention[i].Init(0, i * 5 + 2, OpenCL, 64, 16, 4, 2, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caLocalPE[i].Init(0, i * 5 + 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; if(!caGlobalPE[i].Init(0, i * 5 + 4, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } caLocalAttention[0].Sparse(0.1f); caLocalAttention[1].Sparse(0.3f); //--- if(!cConcatenate.Init(0, 10, OpenCL, 128 * units_count, optimization, iBatch)) ReturnFalse; if(!cScale.Init(0, 11, OpenCL, 128, 128, 64, units_count, 1, optimization, iBatch)) ReturnFalse; //--- DeleteObj(cbTemp); cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPointFormer feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].FeedForward(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(inputs)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[i].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].FeedForward(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].FeedForward(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].FeedForward(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!Concat(caGlobalAttention[0].getOutput(), caGlobalAttention[1].getOutput(), cConcatenate.getOutput(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; if(!cScale.FeedForward(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(cScale.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPointFormer layer. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPointNetOCL::calcInputGradients(cScale.AsObject())) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cScale.AsObject())) ReturnFalse; if(!DeConcat(caGlobalAttention[0].getGradient(), caGlobalAttention[1].getGradient(), cConcatenate.getGradient(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; //--- CNeuronBaseOCL *inputs = caGlobalAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { //--- Global Attention if(!caLocalGlobalAttention[i].CalcHiddenGradients(caGlobalAttention[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalPE[i].CalcHiddenGradients(caLocalGlobalAttention[i].AsObject(), caGlobalPE[i].getOutput(), caGlobalPE[i].getGradient(), (ENUM_ACTIVATION)caGlobalPE[i].Activation())) ReturnFalse; //--- Position Encoder if(!caLocalAttention[i].CalcHiddenGradients(caLocalPE[i].AsObject())) ReturnFalse; if(!caLocalPointNet[i].CalcHiddenGradients(caGlobalPE[i].AsObject())) ReturnFalse; //--- Local Attention CBufferFloat *temp = caLocalPointNet[i].getGradient(); caLocalPointNet[i].SetGradient(cbTemp, false); cbTemp = temp; if(!caLocalPointNet[i].CalcHiddenGradients(caLocalAttention[i].AsObject())) ReturnFalse; if(!SumAndNormalize(caLocalPointNet[i].getGradient(), cbTemp, caLocalPointNet[i].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) { temp = inputs.getGradient(); inputs.SetGradient(cbTemp, false); cbTemp = temp; } //--- Local Net if(!cTNetG || i > 0) { if(!inputs.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(inputs.getOutput(), inputs.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), inputs.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = inputs.getGradient(); inputs.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!inputs.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormalize(inputs.getGradient(), cTurnedG.getGradient(), inputs.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) { if(!SumAndNormalize(inputs.getGradient(), cbTemp, inputs.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; inputs = caGlobalAttention[i - 1].AsObject(); } else inputs = NeuronOCL; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPointFormer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].UpdateInputWeights(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(inputs)) ReturnFalse; if(!caLocalPointNet[i].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].UpdateInputWeights(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].UpdateInputWeights(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].UpdateInputWeights(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!cScale.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(cScale.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPointFormer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalPE[i].Save(file_handle)) ReturnFalse; if(!caGlobalPE[i].Save(file_handle)) ReturnFalse; } if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPointFormer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Load(const int file_handle) { DeleteObj(cbTemp); //--- if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, caLocalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPE[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalPE[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; //--- cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPointFormer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPointFormer::SetOpenCL(COpenCLMy * obj) { CNeuronPointNet2OCL::SetOpenCL(obj); //--- for(int i = 0; i < 2; i++) { caLocalAttention[i].SetOpenCL(OpenCL); caLocalGlobalAttention[i].SetOpenCL(OpenCL); caGlobalAttention[i].SetOpenCL(OpenCL); caLocalPE[i].SetOpenCL(OpenCL); caGlobalPE[i].SetOpenCL(OpenCL); } cConcatenate.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHyperDet layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Init(0, i * 5, OpenCL, 64, 16, 4, units_count, 2, optimization, iBatch)) ReturnFalse; if(!caLocalGlobalAttention[i].Init(0, i * 5 + 1, OpenCL, 64, 16, 4, 64, 2, units_count, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caGlobalAttention[i].Init(0, i * 5 + 2, OpenCL, 64, 16, 4, 2, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caLocalPE[i].Init(0, i * 5 + 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; if(!caGlobalPE[i].Init(0, i * 5 + 4, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } caLocalAttention[0].Sparse(0.1f); caLocalAttention[1].Sparse(0.3f); //--- if(!cConcatenate.Init(0, 10, OpenCL, 128 * units_count, optimization, iBatch)) ReturnFalse; if(!cScale.Init(0, 11, OpenCL, 128, 128, 64, units_count, 1, optimization, iBatch)) ReturnFalse; //--- DeleteObj(cbTemp); cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHyperDet feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].FeedForward(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(inputs)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[i].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].FeedForward(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].FeedForward(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].FeedForward(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!Concat(caGlobalAttention[0].getOutput(), caGlobalAttention[1].getOutput(), cConcatenate.getOutput(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; if(!cScale.FeedForward(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(cScale.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronHyperDet | //| layer. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPointNetOCL::calcInputGradients(cScale.AsObject())) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cScale.AsObject())) ReturnFalse; if(!DeConcat(caGlobalAttention[0].getGradient(), caGlobalAttention[1].getGradient(), cConcatenate.getGradient(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; //--- CNeuronBaseOCL *inputs = caGlobalAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { //--- Global Attention if(!caLocalGlobalAttention[i].CalcHiddenGradients(caGlobalAttention[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalPE[i].CalcHiddenGradients(caLocalGlobalAttention[i].AsObject(), caGlobalPE[i].getOutput(), caGlobalPE[i].getGradient(), (ENUM_ACTIVATION)caGlobalPE[i].Activation())) ReturnFalse; //--- Position Encoder if(!caLocalAttention[i].CalcHiddenGradients(caLocalPE[i].AsObject())) ReturnFalse; if(!caLocalPointNet[i].CalcHiddenGradients(caGlobalPE[i].AsObject())) ReturnFalse; //--- Local Attention CBufferFloat *temp = caLocalPointNet[i].getGradient(); caLocalPointNet[i].SetGradient(cbTemp, false); cbTemp = temp; if(!caLocalPointNet[i].CalcHiddenGradients(caLocalAttention[i].AsObject())) ReturnFalse; if(!SumAndNormalize(caLocalPointNet[i].getGradient(), cbTemp, caLocalPointNet[i].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) { temp = inputs.getGradient(); inputs.SetGradient(cbTemp, false); cbTemp = temp; } //--- Local Net if(!cTNetG || i > 0) { if(!inputs.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(inputs.getOutput(), inputs.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), inputs.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = inputs.getGradient(); inputs.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!inputs.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormalize(inputs.getGradient(), cTurnedG.getGradient(), inputs.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) if(!SumAndNormalize(inputs.getGradient(), cbTemp, inputs.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; inputs = NeuronOCL; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHyperDet trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].UpdateInputWeights(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(inputs)) ReturnFalse; if(!caLocalPointNet[i].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].UpdateInputWeights(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].UpdateInputWeights(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].UpdateInputWeights(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!cScale.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(cScale.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronHyperDet state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalPE[i].Save(file_handle)) ReturnFalse; if(!caGlobalPE[i].Save(file_handle)) ReturnFalse; } if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHyperDet state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Load(const int file_handle) { DeleteObj(cbTemp); //--- if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, caLocalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPE[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalPE[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; //--- cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronHyperDet GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronHyperDet::SetOpenCL(COpenCLMy * obj) { CNeuronPointNet2OCL::SetOpenCL(obj); //--- for(int i = 0; i < 2; i++) { caLocalAttention[i].SetOpenCL(OpenCL); caLocalGlobalAttention[i].SetOpenCL(OpenCL); caGlobalAttention[i].SetOpenCL(OpenCL); caLocalPE[i].SetOpenCL(OpenCL); caGlobalPE[i].SetOpenCL(OpenCL); } cConcatenate.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSEFormer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, uint center_points, uint center_window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; //--- iUnits = units_count; iPoints = MathMax(center_points, 9); //--- cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cCenterPoints.SetOpenCL(OpenCL); cFinalAttention.SetOpenCL(OpenCL); //--- Init center points CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iPoints * center_window * 2, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; if(!cCenterPoints.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base.Init(0, 1, OpenCL, iPoints * center_window * 2, optimization, iBatch)) ReturnFalse; if(!cCenterPoints.Add(base)) ReturnFalse; //--- Inside layers for(int i = 0; i < 2; i++) { //--- Interpolation CNeuronMVCrossAttentionMLKV *cross = new CNeuronMVCrossAttentionMLKV(); if(!cross || !cross.Init(0, i * 12 + 2, OpenCL, center_window, 32, 4, 64, 2, iPoints, iUnits, 2, 2, 2, 1, optimization, iBatch)) ReturnFalse; if(!cCenterPoints.Add(cross)) ReturnFalse; //--- Query CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 3, OpenCL, 64, 64, 64, iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 4, OpenCL, center_window, center_window, 32, iPoints, 2, optimization, iBatch)) ReturnFalse; if(!cKey.Add(conv)) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 5, OpenCL, center_window, center_window, 32, 1, iPoints * 2, optimization, iBatch)) ReturnFalse; if(!cValue.Add(conv)) ReturnFalse; //--- Key-Value base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 6, OpenCL, iPoints * 2 * 32 * 2, optimization, iBatch)) ReturnFalse; if(!cKeyValue.Add(base)) ReturnFalse; //--- Score int s = int(iUnits * iPoints * 4); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- MH Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 7, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!cMHAttentionOut.Add(base)) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 8, OpenCL, 64, 64, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 9, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 10, OpenCL, 64, 64, 256, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 11, OpenCL, 256, 64, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 12, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient(), true)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Final Attention CNeuronMLMHSceneConditionAttention *att = new CNeuronMLMHSceneConditionAttention(); if(!att || !att.Init(0, i * 12 + 13, OpenCL, 64, 16, 4, 2, iUnits, 2, 1, optimization, iBatch)) ReturnFalse; if(!cFinalAttention.Add(att)) ReturnFalse; } //--- if(!SEOut.Init(0, 26, OpenCL, 64, 64, 4, 16, 4, iUnits, iUnits, 4, 1, optimization, iBatch)) ReturnFalse; //--- uint buf_size = MathMax(MathMax(64, window) * iUnits, center_window * iPoints); if(!cbTemp.BufferInit(buf_size, 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention out step of the CNeuronSEFormer layer. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::AttentionOut(CBufferFloat* q, CBufferFloat* kv, int scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iPoints, 4}; uint local_work_size[3] = {1, iPoints, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, 16) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (2 * 2)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention inside gradients step of the | //| CNeuronSEFormer layer. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, int scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, 16, 4}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iPoints) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (2 * 2)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSEFormer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- CNeuronBaseOCL *neuron = NULL, *q = NULL, *k = NULL, *v = NULL, *kv = NULL; //--- Init Points if(bTrain) { neuron = cCenterPoints[1]; if(!neuron || !neuron.FeedForward(cCenterPoints[0])) ReturnFalse; } //--- Inside Layers for(int l = 0; l < 2; l++) { //--- Segmentation Inputs if(l > 0 || !cTNetG) { if(!caLocalPointNet[l].FeedForward((l == 0 ? NeuronOCL : GetPointer(caLocalPointNet[l - 1])))) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[0].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Interpolate center points neuron = cCenterPoints[l + 2]; if(!neuron || !neuron.FeedForward(cCenterPoints[l + 1], caLocalPointNet[l].getOutput())) ReturnFalse; //--- Structure-Embedding Attention q = cQuery[l]; k = cKey[l]; v = cValue[l]; kv = cKeyValue[l]; //--- Query if(!q || !q.FeedForward(GetPointer(caLocalPointNet[l]))) ReturnFalse; //--- Key if(!k || !k.FeedForward(cCenterPoints[l + 2])) ReturnFalse; //--- Value if(!v || !v.FeedForward(cCenterPoints[l + 2])) ReturnFalse; if(!kv || !Concat(k.getOutput(), v.getOutput(), kv.getOutput(), 32 * 2, 32 * 2, iPoints)) ReturnFalse; //--- Multi-Head Attention neuron = cMHAttentionOut[l]; if(!neuron || !AttentionOut(q.getOutput(), kv.getOutput(), cScores[l], neuron.getOutput())) ReturnFalse; //--- Scale neuron = cAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[l])) ReturnFalse; //--- Residual q = cResidual[l * 2]; if(!q || !SumAndNormalize(caLocalPointNet[l].getOutput(), neuron.getOutput(), q.getOutput(), 64, true, 0, 0, 0, 1)) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.FeedForward(q)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cFeedForward[l * 2])) ReturnFalse; //--- Residual k = cResidual[l * 2 + 1]; if(!k || !SumAndNormalize(q.getOutput(), neuron.getOutput(), k.getOutput(), 64, true, 0, 0, 0, 1)) ReturnFalse; //--- Final Attention neuron = cFinalAttention[l]; if(!neuron || !neuron.FeedForward(k)) ReturnFalse; } //--- Cross scale attention if(!SEOut.FeedForward(cFinalAttention[0], neuron.getOutput())) ReturnFalse; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::feedForward(SEOut.AsObject())) ReturnFalse; //--- result return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronSEFormer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *neuron = NULL, *q = NULL, *k = NULL, *v = NULL, *kv = NULL; CBufferFloat *buf = NULL; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::calcInputGradients(SEOut.AsObject())) ReturnFalse; //--- Cross scale attention neuron = cFinalAttention[0]; q = cFinalAttention[1]; if(!neuron.CalcHiddenGradients(SEOut.AsObject(), q.getOutput(), q.getGradient(), (ENUM_ACTIVATION)q.Activation())) ReturnFalse; //--- Inside Layers for(int l = 1; l >= 0; l--) { //--- Final Attention neuron = cResidual[l * 2 + 1]; if(!neuron || !neuron.CalcHiddenGradients(cFinalAttention[l])) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; neuron = cResidual[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual q = cResidual[l * 2 + 1]; k = neuron; neuron = cAttentionOut[l]; if(!neuron || !SumAndNormalize(q.getGradient(), k.getGradient(), neuron.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Scale neuron = cMHAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cAttentionOut[l])) ReturnFalse; //--- MH Attention q = cQuery[l]; kv = cKeyValue[l]; k = cKey[l]; v = cValue[l]; if(!AttentionInsideGradients(q.getOutput(), q.getGradient(), kv.getOutput(), kv.getGradient(), cScores[l], neuron.getGradient())) ReturnFalse; if(!DeConcat(k.getGradient(), v.getGradient(), kv.getGradient(), 32 * 2, 32 * 2, iPoints)) ReturnFalse; if(l == 0) { buf = caLocalPointNet[l].getGradient(); if(!caLocalPointNet[l].SetGradient(GetPointer(cbTemp), false)) ReturnFalse; } if(!caLocalPointNet[l].CalcHiddenGradients(q, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormalize(buf, GetPointer(cbTemp), buf, 64, false, 0, 0, 0, 1)) ReturnFalse; if(!caLocalPointNet[l].SetGradient(buf, false)) ReturnFalse; } neuron = cAttentionOut[l]; //--- Residual if(!SumAndNormalize(caLocalPointNet[l].getGradient(), neuron.getGradient(), caLocalPointNet[l].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Interpolate Center points neuron = cCenterPoints[l + 2]; if(!neuron) ReturnFalse; buf = neuron.getGradient(); if(!neuron.SetGradient(GetPointer(cbTemp), false)) ReturnFalse; if(!neuron.CalcHiddenGradients(k, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormalize(buf, GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(GetPointer(cbTemp), GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 0.5f)) ReturnFalse; } if(!neuron.CalcHiddenGradients(v, NULL)) ReturnFalse; if(!SumAndNormalize(buf, GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!neuron.SetGradient(buf, false)) ReturnFalse; neuron = cCenterPoints[l + 1]; if(!neuron.CalcHiddenGradients(cCenterPoints[l + 2], caLocalPointNet[l].getOutput(), GetPointer(cbTemp), (ENUM_ACTIVATION)caLocalPointNet[l].Activation())) ReturnFalse; if(!SumAndNormalize(caLocalPointNet[l].getGradient(), GetPointer(cbTemp), caLocalPointNet[l].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Local Net neuron = (l > 0 ? GetPointer(caLocalPointNet[l - 1]) : NeuronOCL); if(l > 0 || !cTNetG) { if(!neuron.CalcHiddenGradients(caLocalPointNet[l].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[l].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(neuron.getOutput(), neuron.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), neuron.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = neuron.getGradient(); neuron.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!neuron.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormalize(neuron.getGradient(), cTurnedG.getGradient(), neuron.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSEFormer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- CNeuronBaseOCL *q = NULL, *k = NULL, *v = NULL, *kv = NULL; //--- Init Points CNeuronBaseOCL *neuron = cCenterPoints[1]; if(!neuron || !neuron.UpdateInputWeights(cCenterPoints[0])) ReturnFalse; //--- Inside Layers for(int l = 0; l < 2; l++) { //--- Segmentation Inputs if(l > 0 || !cTNetG) { if(!caLocalPointNet[l].UpdateInputWeights((l == 0 ? NeuronOCL : GetPointer(caLocalPointNet[l - 1])))) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caLocalPointNet[0].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Interpolate center points neuron = cCenterPoints[l + 2]; if(!neuron || !neuron.UpdateInputWeights(cCenterPoints[l + 1], caLocalPointNet[l].getOutput())) ReturnFalse; //--- Structure-Embedding Attention q = cQuery[l]; k = cKey[l]; v = cValue[l]; //--- Query if(!q || !q.UpdateInputWeights(GetPointer(caLocalPointNet[l]))) ReturnFalse; //--- Key if(!k || !k.UpdateInputWeights(cCenterPoints[l + 2])) ReturnFalse; //--- Value if(!v || !v.UpdateInputWeights(cCenterPoints[l + 2])) ReturnFalse; //--- Scale neuron = cAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[l])) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.UpdateInputWeights(cResidual[l * 2])) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; //--- Final Attention neuron = cFinalAttention[l]; if(!neuron || !neuron.UpdateInputWeights(cResidual[l * 2 + 1])) ReturnFalse; } //--- Cross scale attention if(!SEOut.UpdateInputWeights(cFinalAttention[0], neuron.getOutput())) ReturnFalse; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::updateInputWeights(SEOut.AsObject())) ReturnFalse; //--- result return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSEFormer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPoints) < INT_VALUE) ReturnFalse; //--- Save objects if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cKeyValue.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; if(!cCenterPoints.Save(file_handle)) ReturnFalse; if(!cFinalAttention.Save(file_handle)) ReturnFalse; if(!SEOut.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSEFormer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Load(const int file_handle) { if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- Save constants iUnits = (uint)FileReadInteger(file_handle); iPoints = (uint)FileReadInteger(file_handle); //--- Save objects if(!cQuery.Load(file_handle)) ReturnFalse; if(!cKey.Load(file_handle)) ReturnFalse; if(!cValue.Load(file_handle)) ReturnFalse; if(!cKeyValue.Load(file_handle)) ReturnFalse; if(!cMHAttentionOut.Load(file_handle)) ReturnFalse; if(!cAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; if(!cCenterPoints.Load(file_handle)) ReturnFalse; if(!cFinalAttention.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, SEOut.AsObject())) ReturnFalse; //--- Score int s = int(iUnits * iPoints * 4); for(int i = 0; i < 2; i++) { OpenCL.BufferFree(cScores[i]); if(!cScores.Update(i, OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE)) || cScores[i] < 0) ReturnFalse; } //--- CNeuronBaseOCL *ff = NULL, *residual = NULL; for(int i = 1; i < cFeedForward.Total(); i += 2) { ff = cFeedForward[i]; residual = cResidual[i]; if(!ff || !residual || !residual.SetGradient(ff.getGradient())) ReturnFalse; } //--- int buf_size = MathMax(((CNeuronBaseOCL*)cCenterPoints[2]).Neurons(), caLocalPointNet[0].Neurons()); cbTemp.BufferFree(); if(!cbTemp.BufferInit(buf_size, 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSEFormer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSEFormer::SetOpenCL(COpenCLMy * obj) { bool create_score = false; if(!!OpenCL && obj != OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); create_score = true; } //--- CNeuronPointNet2OCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cCenterPoints.SetOpenCL(OpenCL); cFinalAttention.SetOpenCL(OpenCL); SEOut.SetOpenCL(OpenCL); //--- Score int s = int(iUnits * iPoints * 4); for(int i = 0; i < 2 && create_score; i++) cScores.Update(i, OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE)); //--- cbTemp.BufferFree(); cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultitaskStockformer layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint neurons_out, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, neurons_out, optimization_type, batch)) ReturnFalse; //--- Decoupling Flow uint index = 0; uint wave_window = MathMin(24, units_count); if(!cDecouplingFlow.Init(0, index, OpenCL, wave_window, 2, units_count, filters, window, optimization, iBatch)) ReturnFalse; cDecouplingFlow.SetActivationFunction(None); //--- Dual-Frequency Spatiotemporal Encoder uint wave_units_out = cDecouplingFlow.GetUnits(); index++; if(!cLowFreqSignal.Init(0, index, OpenCL, cDecouplingFlow.Neurons() / 2, optimization, iBatch)) ReturnFalse; cLowFreqSignal.SetActivationFunction(None); index++; if(!cHighFreqSignal.Init(0, index, OpenCL, cDecouplingFlow.Neurons() / 2, optimization, iBatch)) ReturnFalse; cHighFreqSignal.SetActivationFunction(None); index++; if(!cTemporalAttention.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, heads, layers, optimization, iBatch)) ReturnFalse; cTemporalAttention.SetActivationFunction(None); index++; if(!cDilatedCasualConvolution.Init(0, index, OpenCL, 2, 2, filters, wave_units_out, window, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowFreqPE.Init(0, index, OpenCL, cTemporalAttention.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cHighFreqPE.Init(0, index, OpenCL, cDilatedCasualConvolution.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cLowFreqGraphAttention.Init(0, index, OpenCL, filters, 3, wave_units_out * window, optimization, iBatch)) ReturnFalse; index++; if(!cHighFreqGraphAttention.Init(0, index, OpenCL, filters, 3, cDilatedCasualConvolution.Neurons() / filters, optimization, iBatch)) ReturnFalse; index++; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, heads, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowHighFreqFusionDecoder.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, filters, cDilatedCasualConvolution.Neurons() / filters, heads, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowHigh.Init(0, index, OpenCL, cLowFreqFusionDecoder.Neurons(), optimization, iBatch)) ReturnFalse; CBufferFloat *grad = cLowFreqFusionDecoder.getGradient(); if(!grad || !cLowHigh.SetGradient(grad, true) || !cLowHighFreqFusionDecoder.SetGradient(grad, true)) ReturnFalse; index++; if(!cProjection.Init(Neurons(), index, OpenCL, filters, filters, 3, wave_units_out, window, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultitaskStockformer feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Decoupling Flow if(!cDecouplingFlow.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cLowFreqSignal.getOutput(), cHighFreqSignal.getOutput(), cDecouplingFlow.getOutput(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetUnits()*cDecouplingFlow.GetVariables())) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cTemporalAttention.FeedForward(cLowFreqSignal.AsObject())) ReturnFalse; if(!cLowFreqPE.FeedForward(cTemporalAttention.AsObject())) ReturnFalse; if(!cLowFreqGraphAttention.FeedForward(cLowFreqPE.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cDilatedCasualConvolution.FeedForward(cHighFreqSignal.AsObject())) ReturnFalse; if(!cHighFreqPE.FeedForward(cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!cHighFreqGraphAttention.FeedForward(cHighFreqPE.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.FeedForward(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cLowHighFreqFusionDecoder.FeedForward(cLowFreqGraphAttention.AsObject(), cHighFreqGraphAttention.getOutput())) ReturnFalse; if(!SumAndNormalize(cLowFreqFusionDecoder.getOutput(), cLowHighFreqFusionDecoder.getOutput(), cLowHigh.getOutput(), cLowFreqFusionDecoder.GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- if(!cProjection.FeedForward(cLowHigh.AsObject())) ReturnFalse; //--- return(CNeuronBaseOCL::feedForward(cProjection.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultitaskStockformer layer. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronBaseOCL::calcInputGradients(cProjection.AsObject())) ReturnFalse; if(!cLowHigh.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqGraphAttention.CalcHiddenGradients(cLowFreqFusionDecoder.AsObject())) ReturnFalse; CBufferFloat *grad = cLowFreqGraphAttention.getGradient(); if(!cLowFreqGraphAttention.SetGradient(cLowFreqGraphAttention.getPrevOutput(), false) || !cLowFreqGraphAttention.CalcHiddenGradients(cLowHighFreqFusionDecoder.AsObject(), cHighFreqGraphAttention.getOutput(), cHighFreqGraphAttention.getGradient(), (ENUM_ACTIVATION)cHighFreqGraphAttention.Activation()) || !SumAndNormalize(grad, cLowFreqGraphAttention.getGradient(), grad, 1, false, 0, 0, 0, 1) || !cLowFreqGraphAttention.SetGradient(grad, false)) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cLowFreqPE.CalcHiddenGradients(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cTemporalAttention.CalcHiddenGradients(cLowFreqPE.AsObject())) ReturnFalse; if(!cLowFreqSignal.CalcHiddenGradients(cTemporalAttention.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cHighFreqPE.CalcHiddenGradients(cHighFreqGraphAttention.AsObject())) ReturnFalse; if(!cDilatedCasualConvolution.CalcHiddenGradients(cHighFreqPE.AsObject())) ReturnFalse; if(!cHighFreqSignal.CalcHiddenGradients(cDilatedCasualConvolution.AsObject())) ReturnFalse; //--- Decoupling Flow if(!Concat(cLowFreqSignal.getGradient(), cHighFreqSignal.getGradient(), cDecouplingFlow.getGradient(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetUnits()*cDecouplingFlow.GetVariables())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cDecouplingFlow.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultitaskStockformer trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- if(!CNeuronBaseOCL::updateInputWeights(cProjection.AsObject())) ReturnFalse; if(!cProjection.UpdateInputWeights(cLowHigh.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.UpdateInputWeights(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cLowHighFreqFusionDecoder.UpdateInputWeights(cLowFreqGraphAttention.AsObject(), cHighFreqGraphAttention.getOutput())) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cLowFreqPE.UpdateInputWeights(cTemporalAttention.AsObject())) ReturnFalse; if(!cTemporalAttention.UpdateInputWeights(cLowFreqSignal.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cHighFreqPE.UpdateInputWeights(cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!cDilatedCasualConvolution.UpdateInputWeights(cHighFreqSignal.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultitaskStockformer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cDecouplingFlow.Save(file_handle)) ReturnFalse; if(!cLowFreqSignal.Save(file_handle)) ReturnFalse; if(!cHighFreqSignal.Save(file_handle)) ReturnFalse; if(!cTemporalAttention.Save(file_handle)) ReturnFalse; if(!cDilatedCasualConvolution.Save(file_handle)) ReturnFalse; if(!cLowFreqPE.Save(file_handle)) ReturnFalse; if(!cHighFreqPE.Save(file_handle)) ReturnFalse; if(!cLowFreqGraphAttention.Save(file_handle)) ReturnFalse; if(!cHighFreqGraphAttention.Save(file_handle)) ReturnFalse; if(!cLowFreqFusionDecoder.Save(file_handle)) ReturnFalse; if(!cLowHighFreqFusionDecoder.Save(file_handle)) ReturnFalse; if(!cLowHigh.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultitaskStockformer state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cDecouplingFlow.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqSignal.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqSignal.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqGraphAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqFusionDecoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowHighFreqFusionDecoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowHigh.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; //--- CBufferFloat *grad = cLowFreqFusionDecoder.getGradient(); if(!grad || !cLowHigh.SetGradient(grad, true) || !cLowHighFreqFusionDecoder.SetGradient(grad, true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultitaskStockformer GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultitaskStockformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDecouplingFlow.SetOpenCL(OpenCL); cLowFreqSignal.SetOpenCL(OpenCL); cHighFreqSignal.SetOpenCL(OpenCL); cTemporalAttention.SetOpenCL(OpenCL); cDilatedCasualConvolution.SetOpenCL(OpenCL); cLowFreqPE.SetOpenCL(OpenCL); cHighFreqPE.SetOpenCL(OpenCL); cLowFreqGraphAttention.SetOpenCL(OpenCL); cHighFreqGraphAttention.SetOpenCL(OpenCL); cLowFreqFusionDecoder.SetOpenCL(OpenCL); cLowHighFreqFusionDecoder.SetOpenCL(OpenCL); cLowHigh.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHidformer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHidformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, nactions, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; uint half = (caTSAgents.Size() + 1) / 2; for(uint i = 0; i < half; i++) { index++; if(!caTSAgents[i].Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } for(uint i = half; i < caTSAgents.Size(); i++) { index++; if(!caTSAgents[i].Init(0, index, OpenCL, units_count, window_key, window, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } half = (caFreqAgents.Size() + 1) / 2; for(uint i = 0; i < half; i++) { index++; if(!caFreqAgents[i].Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } for(uint i = half; i < caFreqAgents.Size(); i++) { index++; if(!caFreqAgents[i].Init(0, index, OpenCL, units_count, window_key, window, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } index++; if(!cHyperAgent.Init(0, index, OpenCL, window, window_key, units_count, heads, layers, caTSAgents.Size() + caFreqAgents.Size(), stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cConcatenated.Init(0, index, OpenCL, (caTSAgents.Size() + caFreqAgents.Size())*nactions, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHidformer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronHidformer::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- uint total_ts = caTSAgents.Size(); uint half = (total_ts + 1) / 2; for(uint i = 0; i < half; i++) { if(!caTSAgents[i].FeedForward(NeuronOCL)) ReturnFalse; } for(uint i = half; i < total_ts; i++) { if(!caTSAgents[i].FeedForward(cTranspose.AsObject())) ReturnFalse; } //--- uint total_fr = caFreqAgents.Size(); half = (total_fr + 1) / 2; for(uint i = 0; i < half; i++) { if(!caFreqAgents[i].FeedForward(NeuronOCL)) ReturnFalse; } for(uint i = half; i < total_fr; i++) { if(!caFreqAgents[i].FeedForward(cTranspose.AsObject())) ReturnFalse; } if(!cHyperAgent.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Concat(caTSAgents[0].getOutput(), caTSAgents[1].getOutput(), caTSAgents[2].getOutput(), caTSAgents[3].getOutput(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1) || !Concat(cConcatenated.getPrevOutput(), caFreqAgents[0].getOutput(), caFreqAgents[1].getOutput(), cConcatenated.getOutput(), 4 * Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; if(!MatMul(cHyperAgent.getOutput(), cConcatenated.getOutput(), Output, 1, total_fr + total_ts, Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronHidformer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronHidformer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint total_ts = caTSAgents.Size(); uint total_fr = caFreqAgents.Size(); if(!MatMulGrad(cHyperAgent.getOutput(), cHyperAgent.getGradient(), cConcatenated.getOutput(), cConcatenated.getGradient(), Gradient, 1, total_ts + total_fr, Neurons(), 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cHyperAgent.AsObject())) ReturnFalse; //--- if(!DeConcat(cConcatenated.getPrevOutput(), caFreqAgents[0].getGradient(), caFreqAgents[1].getGradient(), cConcatenated.getGradient(), 4 * Neurons(), Neurons(), Neurons(), 1) || !DeConcat(caTSAgents[0].getGradient(), caTSAgents[1].getGradient(), caTSAgents[2].getGradient(), caTSAgents[3].getGradient(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!temp || !NeuronOCL.SetGradient(cTranspose.getPrevOutput(), false)) ReturnFalse; uint half = (total_ts + 1) / 2; for(uint i = 0; i < half; i++) { if(!NeuronOCL.CalcHiddenGradients(caTSAgents[i].AsObject())) ReturnFalse; if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } for(uint i = half; i < total_ts; i++) { if(!cTranspose.CalcHiddenGradients(caTSAgents[i].AsObject()) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } half = (total_fr + 1) / 2; for(uint i = 0; i < half; i++) { if(!NeuronOCL.CalcHiddenGradients(caFreqAgents[i].AsObject())) ReturnFalse; if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } for(uint i = half; i < total_fr; i++) { if(!cTranspose.CalcHiddenGradients(caFreqAgents[i].AsObject()) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHidformer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHidformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cHyperAgent.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- uint total = caTSAgents.Size(); uint half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caTSAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caTSAgents[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- total = caFreqAgents.Size(); half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caFreqAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caFreqAgents[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronHidformer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHidformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!caTSAgents[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!caFreqAgents[i].Save(file_handle)) ReturnFalse; if(!cHyperAgent.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHidformer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHidformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caTSAgents[i].AsObject())) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqAgents[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHyperAgent.AsObject())) ReturnFalse; //--- if(!cConcatenated.Init(0, 8, OpenCL, (caTSAgents.Size() + caFreqAgents.Size())*Neurons(), optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronHidformer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronHidformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(obj); for(uint i = 0; i < caTSAgents.Size(); i++) caTSAgents[i].SetOpenCL(obj); for(uint i = 0; i < caFreqAgents.Size(); i++) caFreqAgents[i].SetOpenCL(obj); cHyperAgent.SetOpenCL(obj); cConcatenated.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| Clears the CNeuronHidformer execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronHidformer::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cTranspose.Clear()) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!caTSAgents[i].Clear()) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!caFreqAgents[i].Clear()) ReturnFalse; if(!cHyperAgent.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronChimera layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronChimera::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_out * window_out, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; for(int i = 0; i < 2; i++) { if(!caSSM[i].Init(0, index, OpenCL, window_in, (i + 1)*window_out, units_in, units_out, optimization, iBatch)) ReturnFalse; index++; } if(!cDiscretization.Init(0, index, OpenCL, 2 * window_out, 2 * window_out, window_out, units_out, 1, optimization, iBatch)) ReturnFalse; cDiscretization.SetActivationFunction(None); //--- Residual cResidual.Clear(); cResidual.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, window_in, optimization, iBatch) || !cResidual.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, units_out, window_in, 1, optimization, iBatch) || !cResidual.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_in, units_out, optimization, iBatch) || !cResidual.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, window_out, units_out, 1, optimization, iBatch) || !cResidual.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- if(!SetGradient(caSSM[0].getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronChimera feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronChimera::feedForward(CNeuronBaseOCL* NeuronOCL) { for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].FeedForward(NeuronOCL)) ReturnFalse; } if(!cDiscretization.FeedForward(caSSM[1].AsObject())) ReturnFalse; //--- CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- if(!SumAndNormalize(caSSM[0].getOutput(), cDiscretization.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormalize(Output, current.getOutput(), Output, cDiscretization.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronChimera | //| layer. | //+------------------------------------------------------------------+ bool CNeuronChimera::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cDiscretization.getOutput(), cDiscretization.getGradient(), Gradient, cDiscretization.Activation())) ReturnFalse; if(!caSSM[1].CalcHiddenGradients(cDiscretization.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *residual = cResidual[-1]; if(!residual) ReturnFalse; if(!DeActivation(residual.getOutput(), residual.getGradient(), Gradient, residual.Activation())) ReturnFalse; for(int i = cResidual.Total() - 2; i >= 0; i--) { residual = cResidual[i]; if(!residual || !residual.CalcHiddenGradients(cResidual[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(caSSM[0].AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(residual.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(caSSM[1].AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients((CObject*)residual) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronChimera trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronChimera::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cDiscretization.UpdateInputWeights(caSSM[1].AsObject())) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; } //--- CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.UpdateInputWeights(inp)) ReturnFalse; inp = current; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronChimera state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronChimera::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!caSSM[i].Save(file_handle)) ReturnFalse; if(!cDiscretization.Save(file_handle)) ReturnFalse; //--- if(!cResidual.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronChimera state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronChimera::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!LoadInsideLayer(file_handle, caSSM[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDiscretization.AsObject())) ReturnFalse; //--- if(!cResidual.Load(file_handle)) ReturnFalse; //--- if(!SetGradient(caSSM[0].getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronChimera GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronChimera::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < caSSM.Size(); i++) caSSM[i].SetOpenCL(OpenCL); cDiscretization.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Clears the CNeuronChimera execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronChimera::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!caSSM[i].Clear()) ReturnFalse; if(!cDiscretization.Clear()) ReturnFalse; //--- for(int i = 0; i < cResidual.Total(); i++) if(!((CNeuronBaseOCL*)cResidual[i]).Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMoT layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMoT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, window, units_count, 4, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cNodesTokenizer.Init(0, index, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cNodesTokenizer.SetActivationFunction(SoftPlus); //--- index++; if(!cEdgesTokenizer.Init(0, index, OpenCL, 2 * iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cEdgesTokenizer.SetActivationFunction(SoftPlus); //--- index++; if(!cSubGraphsTokenizer.Init(0, index, OpenCL, 3 * iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cSubGraphsTokenizer.SetActivationFunction(SoftPlus); //--- cUnitarSubGraphsTokenizer.Clear(); cUnitarSubGraphsTokenizer.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, iUnits, iWindow, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 3, 1, 1, iUnits, iWindow, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- index++; if(!cConcatenate.Init(0, index, OpenCL, 4 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatenate.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMoT feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMoT::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cNodesTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!cEdgesTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!cSubGraphsTokenizer.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL, *current = NULL; for(int i = 0; i < cUnitarSubGraphsTokenizer.Total(); i++) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- if(!Concat(cNodesTokenizer.getOutputIndex(), cEdgesTokenizer.getOutputIndex(), cSubGraphsTokenizer.getOutputIndex(), current.getOutputIndex(), cConcatenate.getOutputIndex(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- return(CNeuronMHAttentionPooling::feedForward(cConcatenate.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMoT | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMoT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcatenate.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *current = cUnitarSubGraphsTokenizer[-1]; if(!current || !DeConcat(cNodesTokenizer.getGradient(), cEdgesTokenizer.getGradient(), cSubGraphsTokenizer.getGradient(), current.getGradient(), cConcatenate.getGradient(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- if(current.Activation() != None && !DeActivation(current.getOutput(), current.getGradient(), current.getGradient(), current.Activation())) ReturnFalse; for(int i = cUnitarSubGraphsTokenizer.Total() - 2; i >= 0; i--) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.CalcHiddenGradients(cUnitarSubGraphsTokenizer[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(current.AsObject())) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(current.getPrevOutput(), false)) ReturnFalse; //--- if(cNodesTokenizer.Activation() != None && !DeActivation(cNodesTokenizer.getOutput(), cNodesTokenizer.getGradient(), cNodesTokenizer.getGradient(), cNodesTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNodesTokenizer.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(cEdgesTokenizer.Activation() != None && !DeActivation(cEdgesTokenizer.getOutput(), cEdgesTokenizer.getGradient(), cEdgesTokenizer.getGradient(), cEdgesTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEdgesTokenizer.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(cSubGraphsTokenizer.Activation() != None && !DeActivation(cSubGraphsTokenizer.getOutput(), cSubGraphsTokenizer.getGradient(), cSubGraphsTokenizer.getGradient(), cSubGraphsTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSubGraphsTokenizer.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMoT trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMoT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMHAttentionPooling::updateInputWeights(cConcatenate.AsObject())) ReturnFalse; //--- if(!cNodesTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cEdgesTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSubGraphsTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL, *current = NULL; for(int i = 0; i < cUnitarSubGraphsTokenizer.Total(); i++) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMoT state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoT::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cNodesTokenizer.Save(file_handle)) ReturnFalse; if(!cEdgesTokenizer.Save(file_handle)) ReturnFalse; if(!cSubGraphsTokenizer.Save(file_handle)) ReturnFalse; if(!cUnitarSubGraphsTokenizer.Save(file_handle)) ReturnFalse; if(!cConcatenate.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMoT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoT::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cNodesTokenizer.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEdgesTokenizer.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSubGraphsTokenizer.AsObject())) ReturnFalse; if(!cUnitarSubGraphsTokenizer.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMoT GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMoT::SetOpenCL(COpenCLMy * obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cNodesTokenizer.SetOpenCL(OpenCL); cEdgesTokenizer.SetOpenCL(OpenCL); cSubGraphsTokenizer.SetOpenCL(OpenCL); cUnitarSubGraphsTokenizer.SetOpenCL(OpenCL); cConcatenate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronChimeraPlus layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronChimera::Init(numOutputs, myIndex, open_cl, window_in, window_out, units_in, units_out, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cSSMPlus.Init(0, index, OpenCL, window_in, window_out, units_in, 2 * units_out, optimization, iBatch)) ReturnFalse; //--- CNeuronTransposeOCL *transp = NULL; CNeuronConvOCL *conv = NULL; cDiscretizationPlus.Clear(); cDiscretizationPlus.SetOpenCL(OpenCL); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, 2 * units_out, window_out, optimization, iBatch) || !cDiscretizationPlus.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 2 * units_out, 2 * units_out, units_out, window_out, 1, optimization, iBatch) || !cDiscretizationPlus.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_out, units_out, optimization, iBatch) || !cDiscretizationPlus.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronChimeraPlus feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::feedForward(CNeuronBaseOCL* NeuronOCL) { for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].FeedForward(NeuronOCL)) ReturnFalse; } if(!cSSMPlus.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!cDiscretization.FeedForward(caSSM[1].AsObject())) ReturnFalse; CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- inp = NeuronOCL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- inp = cDiscretizationPlus[-1]; if(!SumAndNormalize(caSSM[0].getOutput(), cDiscretization.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormalize(Output, inp.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormalize(Output, current.getOutput(), Output, cDiscretization.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronChimeraPlus layer. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronChimera::calcInputGradients(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *current = cDiscretizationPlus[-1]; if(!current || !DeActivation(current.getOutput(), current.getGradient(), Gradient, current.Activation())) ReturnFalse; for(int i = cDiscretizationPlus.Total() - 2; i >= 0; i--) { current = cDiscretizationPlus[i]; if(!current || !current.CalcHiddenGradients(cDiscretizationPlus[i + 1])) ReturnFalse; } if(!cSSMPlus.CalcHiddenGradients(current.AsObject())) ReturnFalse; //--- current = cResidual[0]; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(current.getGradient(), false) || !NeuronOCL.CalcHiddenGradients(cSSMPlus.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronChimeraPlus trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronChimera::updateInputWeights(NeuronOCL)) ReturnFalse; if(!cSSMPlus.UpdateInputWeights(NeuronOCL)) ReturnFalse; CNeuronBaseOCL *current = NULL, *inp = cSSMPlus.AsObject(); for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.UpdateInputWeights(inp.AsObject())) ReturnFalse; inp = current; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronChimeraPlus state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Save(const int file_handle) { if(!CNeuronChimera::Save(file_handle)) ReturnFalse; if(!cSSMPlus.Save(file_handle)) ReturnFalse; if(!cDiscretizationPlus.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronChimeraPlus state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Load(const int file_handle) { if(!CNeuronChimera::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSSMPlus.AsObject())) ReturnFalse; if(!cDiscretizationPlus.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronChimeraPlus GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronChimeraPlus::SetOpenCL(COpenCLMy * obj) { CNeuronChimera::SetOpenCL(obj); cSSMPlus.SetOpenCL(OpenCL); cDiscretizationPlus.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Clears the CNeuronChimeraPlus execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Clear(void) { if(!CNeuronChimera::Clear()) ReturnFalse; if(!cSSMPlus.Clear()) ReturnFalse; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.Clear()) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHypridDecoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronHidformer::Init(numOutputs, myIndex, open_cl, window_key, window_key, nactions, heads, layers, stack_size, nactions, optimization_type, batch)) ReturnFalse; if(!cChimera.Init(0, 0, OpenCL, window, window_key, units_count, nactions, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHypridDecoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cChimera.FeedForward(NeuronOCL)) ReturnFalse; return(CNeuronHidformer::feedForward(cChimera.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronHypridDecoder layer. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronHidformer::calcInputGradients(cChimera.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cChimera.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHypridDecoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronHidformer::updateInputWeights(cChimera.AsObject())) ReturnFalse; //--- if(!cChimera.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronHypridDecoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Save(const int file_handle) { if(!CNeuronHidformer::Save(file_handle)) ReturnFalse; if(!cChimera.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHypridDecoder state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Load(const int file_handle) { if(!CNeuronHidformer::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cChimera.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronHypridDecoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronHypridDecoder::SetOpenCL(COpenCLMy * obj) { CNeuronHidformer::SetOpenCL(obj); cChimera.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Clears the CNeuronHypridDecoder execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Clear(void) { if(!CNeuronHidformer::Clear()) ReturnFalse; if(!cChimera.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAttraos layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAttraos::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; if(!cOne.Init(0, index, OpenCL, units_count, optimization, iBatch)) ReturnFalse; if(!cOne.getOutput().Fill(1)) ReturnFalse; cOne.SetActivationFunction(None); //--- index++; if(!cA.Init(0, index, OpenCL, 1, 1, window * window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cA.SetActivationFunction(MinusSoftPlus); CBufferFloat *w = cA.GetWeightsConv(); if(!w || !w.Fill(0)) ReturnFalse; //--- index++; if(!cD.Init(0, index, OpenCL, 1, 1, window, units_count, 1, optimization, iBatch)) ReturnFalse; cD.SetActivationFunction(None); w = cD.GetWeightsConv(); if(!w || !w.Fill(1)) ReturnFalse; //--- index++; if(!cX_norm.Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cX_norm.SetActivationFunction(None); index++; if(!cX_proj.Init(0, index, OpenCL, window, window, 4 * window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cX_proj.SetActivationFunction(None); //--- index++; if(!cDelta.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cDelta.SetActivationFunction(None); index++; if(!cB.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); index++; if(!cDelta_proj.Init(0, index, OpenCL, window_key, window_key, window, units_count, 1, optimization, iBatch)) ReturnFalse; cDelta_proj.SetActivationFunction(SoftPlus); //--- index++; if(!cDeltaA.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaA.SetActivationFunction(None); index++; if(!cDeltaB.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaB.SetActivationFunction(None); index++; if(!cDeltaBX.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaBX.SetActivationFunction(None); index++; if(!cDeltaH.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaH.SetActivationFunction(None); index++; if(!cHS.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cHS.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the p scan step of the CNeuronAttraos layer. | //+------------------------------------------------------------------+ bool CNeuronAttraos::PScan(void) { if(!OpenCL) ReturnFalse; uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {units, window * window_key}; uint local_work_size[2] = {units, 1}; //--- uint kernel = def_k_PScan; setBuffer(kernel, def_k_psc_A, cDeltaA.getOutputIndex()) setBuffer(kernel, def_k_psc_X, cDeltaBX.getOutputIndex()) setBuffer(kernel, def_k_psc_H, cDeltaH.getOutputIndex()) setBuffer(kernel, def_k_psc_X_out, cHS.getOutputIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cHS.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the p scan calc gradient step of the CNeuronAttraos | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAttraos::PScanCalcGradient(void) { if(!OpenCL) ReturnFalse; uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {units, window * window_key}; uint local_work_size[2] = {units, 1}; //--- uint kernel = def_k_PScan_CalcHiddenGradient; setBuffer(kernel, def_k_pscgr_A, cDeltaA.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_A, cDeltaA.getGradientIndex()) setBuffer(kernel, def_k_pscgr_X, cDeltaBX.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_X, cDeltaBX.getGradientIndex()) setBuffer(kernel, def_k_pscgr_H, cDeltaH.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_H, cDeltaH.getGradientIndex()) setBuffer(kernel, def_k_pscgr_grad_X_out, cHS.getGradientIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cHS.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAttraos feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronAttraos::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- //--- (Units, Window, WindowKey) if(!cA.FeedForward(cOne.AsObject())) ReturnFalse; //--- (Units, Window)) if(!cD.FeedForward(cOne.AsObject())) ReturnFalse; //--- if(!NeuronOCL || !SumAndNormalize(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cX_norm.getOutput(), window, true, 0, 0, 0, 0.5f)) ReturnFalse; //--- (Units, 4*WindowKey) if(!cX_proj.FeedForward(cX_norm.AsObject())) ReturnFalse; //--- if(!DeConcat(cDelta.getOutput(), cB.getOutput(), cC.getOutput(), cH.getOutput(), cX_proj.getOutput(), //--- 4*(Units, WindowKey) window_key, window_key, window_key, window_key, units)) ReturnFalse; //--- (Units, Window) if(!cDelta_proj.FeedForward(cDelta.AsObject())) ReturnFalse; if(!DiagMatMul(cDelta_proj.getOutput(), cA.getOutput(), cDeltaA.getOutput(), //--- (Units, Window, WindowKey) window, window_key, units, SoftPlus)) ReturnFalse; if(!MatMul(cDelta_proj.getOutput(), cB.getOutput(), cDeltaB.getOutput(), //--- (Units, Window, WindowKey) window, 1, window_key, units)) ReturnFalse; if(!DiagMatMul(cX_norm.getOutput(), cDeltaB.getOutput(), cDeltaBX.getOutput(), //--- (Units, Window, WindowKey) window, window_key, units, None)) ReturnFalse; if(!MatMul(cDelta_proj.getOutput(), cH.getOutput(), cDeltaH.getOutput(), //--- (Units, Window, WindowKey) window, 1, window_key, units)) ReturnFalse; if(!PScan()) ReturnFalse; //--- (Units, Window, 1) if(!MatMul(cHS.getOutput(), cC.getOutput(), Output, window, window_key, 1, units)) ReturnFalse; //--- (Units, Window)) if(!ElementMult(cD.getOutput(), cX_norm.getOutput(), PrevOutput)) ReturnFalse; //--- (Units, Window)) if(!SumAndNormalize(Output, PrevOutput, Output, window, false, 0, 0, 0, 1)) ReturnFalse; //--- (Units, Window)) if(!SumAndNormalize(Output, NeuronOCL.getOutput(), Output, window, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronAttraos | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAttraos::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- if(!ElementMultGrad(cD.getOutput(), cD.getGradient(), cX_norm.getOutput(), cX_norm.getPrevOutput(), //--- (Units, Window)) Gradient, cD.Activation(), None)) ReturnFalse; if(!MatMulGrad(cHS.getOutput(), cHS.getGradient(), cC.getOutput(), cC.getGradient(), Gradient, //--- (Units, Window, 1) window, window_key, 1, units)) ReturnFalse; if(cHS.Activation() != None) { if(!DeActivation(cHS.getOutput(), cHS.getGradient(), cHS.getGradient(), cHS.Activation())) ReturnFalse; } if(cC.Activation() != None) { if(!DeActivation(cC.getOutput(), cC.getGradient(), cC.getGradient(), cC.Activation())) ReturnFalse; } //--- if(!PScanCalcGradient()) ReturnFalse; //--- if(!MatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getGradient(), cH.getOutput(), cH.getGradient(), //--- (Units, Window, WindowKey) cDeltaH.getGradient(), window, 1, window_key, units)) ReturnFalse; if(!DiagMatMulGrad(cX_norm.getOutput(), cX_norm.getGradient(), cDeltaB.getOutput(), cDeltaB.getGradient(), //--- (Units, Window, WindowKey) cDeltaBX.getGradient(), window, window_key, units)) ReturnFalse; if(!SumAndNormalize(cX_norm.getGradient(), cX_norm.getPrevOutput(), cX_norm.getPrevOutput(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!MatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getPrevOutput(), cB.getOutput(), cB.getGradient(), //--- (Units, Window, WindowKey) cDeltaB.getGradient(), window, 1, window_key, units)) ReturnFalse; if(!SumAndNormalize(cDelta_proj.getGradient(), cDelta_proj.getPrevOutput(), cDelta_proj.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!DeActivation(cDeltaA.getOutput(), cDeltaA.getGradient(), cDeltaA.getGradient(), SoftPlus)) ReturnFalse; if(!DiagMatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getPrevOutput(), cA.getOutput(), cA.getGradient(), //--- (Units, Window, WindowKey) cDeltaA.getGradient(), window, window_key, units)) ReturnFalse; if(!SumAndNormalize(cDelta_proj.getGradient(), cDelta_proj.getPrevOutput(), cDelta_proj.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(cDelta_proj.Activation() != None) { if(!DeActivation(cDelta_proj.getOutput(), cDelta_proj.getGradient(), cDelta_proj.getGradient(), cDelta_proj.Activation())) ReturnFalse; } //--- if(!cDelta.CalcHiddenGradients(cDelta_proj.AsObject())) ReturnFalse; //--- if(!Concat(cDelta.getGradient(), cB.getGradient(), cC.getGradient(), cH.getGradient(), cX_proj.getGradient(), //--- 4*(Units, WindowKey) window_key, window_key, window_key, window_key, units)) ReturnFalse; //--- if(!cX_norm.CalcHiddenGradients(cX_proj.AsObject())) ReturnFalse; if(!SumAndNormalize(cX_norm.getGradient(), cX_norm.getPrevOutput(), cX_norm.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cX_norm.getGradient(), Gradient, cX_norm.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cX_norm.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAttraos trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronAttraos::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- //--- (Units, Window, WindowKey) if(!cA.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- (Units, Window)) if(!cD.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- (Units, 4*WindowKey) if(!cX_proj.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- (Units, Window) if(!cDelta_proj.UpdateInputWeights(cDelta.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronAttraos state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttraos::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cA.Save(file_handle)) ReturnFalse; if(!cD.Save(file_handle)) ReturnFalse; if(!cX_proj.Save(file_handle)) ReturnFalse; if(!cDelta_proj.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAttraos state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttraos::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cX_proj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_proj.AsObject())) ReturnFalse; //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units_count = cD.GetUnits(); //--- if(!cOne.Init(0, 0, OpenCL, units_count, optimization, iBatch)) ReturnFalse; if(!cOne.getOutput().Fill(1)) ReturnFalse; cOne.SetActivationFunction(None); int index = 3; if(!cX_norm.Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cX_norm.SetActivationFunction(None); index += 2; if(!cDelta.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cDelta.SetActivationFunction(None); index++; if(!cB.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); index += 2; if(!cDeltaA.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaA.SetActivationFunction(None); index++; if(!cDeltaB.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaB.SetActivationFunction(None); index++; if(!cDeltaBX.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaBX.SetActivationFunction(None); index++; if(!cDeltaH.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaH.SetActivationFunction(None); index++; if(!cHS.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cHS.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronAttraos GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronAttraos::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cX_proj.SetOpenCL(OpenCL); cDelta.SetOpenCL(OpenCL); cB.SetOpenCL(OpenCL); cC.SetOpenCL(OpenCL); cD.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cDelta_proj.SetOpenCL(OpenCL); cDeltaA.SetOpenCL(OpenCL); cDeltaB.SetOpenCL(OpenCL); cDeltaBX.SetOpenCL(OpenCL); cDeltaH.SetOpenCL(OpenCL); cHS.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMoE layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMoE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count, optimization_type, batch)) ReturnFalse; int index = 0; if(!cGates.Init(0, index, OpenCL, window, units_count, experts, top_k, optimization, iBatch)) ReturnFalse; //--- index++; if(!cXProj.Init(0, index, OpenCL, window, window, window_out, units_count, optimization, iBatch)) ReturnFalse; cXProj.SetActivationFunction(None); //--- cExperts.Clear(); cExperts.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeRCDOCL *transp = NULL; //--- index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, window_out * experts, units_count, 1, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, units_count, experts, window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_out, window_out, window_out, units_count, experts, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, experts, units_count, window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMoE feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMoE::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cGates.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL; int total = cExperts.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *neuron = cExperts[i]; if(!neuron || !neuron.FeedForward(prev)) ReturnFalse; prev = neuron; } if(!MatMul(cGates.getOutput(), prev.getOutput(), getOutput(), 1, cGates.GetGates(), Neurons() / cGates.GetUnits(), cGates.GetUnits())) ReturnFalse; //--- if(!cXProj.FeedForward(NeuronOCL) || !SumAndNormalize(cXProj.getOutput(), Output, Output, cXProj.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMoE | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMoE::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = cExperts[-1]; if(!MatMulGrad(cGates.getOutput(), cGates.getGradient(), neuron.getOutput(), neuron.getGradient(), getGradient(), 1, cGates.GetGates(), Neurons() / cGates.GetUnits(), cGates.GetUnits())) ReturnFalse; //--- if(cGates.Activation() != None) if(!DeActivation(cGates.getOutput(), cGates.getGradient(), cGates.getGradient(), cGates.Activation())) ReturnFalse; if(neuron.Activation() != None) if(!DeActivation(neuron.getOutput(), neuron.getGradient(), neuron.getGradient(), neuron.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cGates.AsObject())) ReturnFalse; //--- for(int i = cExperts.Total() - 2; i >= 0; i--) { neuron = cExperts[i]; if(!neuron || !neuron.CalcHiddenGradients(cExperts[i + 1])) ReturnFalse; } CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(neuron.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- if(!DeActivation(cXProj.getOutput(), cXProj.getGradient(), Gradient, cXProj.Activation()) || !NeuronOCL.CalcHiddenGradients(cXProj.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMoE trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMoE::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cGates.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cXProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL; int total = cExperts.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *neuron = cExperts[i]; if(!neuron || !neuron.UpdateInputWeights(prev)) ReturnFalse; prev = neuron; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMoE state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cGates.Save(file_handle)) ReturnFalse; if(!cXProj.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMoE state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGates.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cXProj.AsObject())) ReturnFalse; if(!cExperts.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMoE GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMoE::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cGates.SetOpenCL(OpenCL); cXProj.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronChanelMask layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * units_count, optimization_type, batch)) ReturnFalse; //--- Save constants iUnits = units_count; activation = None; //--- Calculate FFT dimension int power = int(MathLog(window) / M_LN2); if(MathPow(2, power) != window) power++; iFFTdimension = uint(MathPow(2, power)); //--- if(!cbFFTReal.BufferInit(iFFTdimension * iUnits, 0) || !cbFFTReal.BufferCreate(OpenCL)) ReturnFalse; if(!cbFFTImag.BufferInit(iFFTdimension * iUnits, 0) || !cbFFTImag.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronChanelMask layer. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits}; int kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / iUnits)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / iUnits)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Applies the masking pass of the CNeuronChanelMask layer. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Mask(void) { uint global_work_offset[2] = {0, 0 }; uint global_work_size[2] = {iUnits, iUnits}; uint local_work_size[2] = {1, iUnits}; //--- int kernel = def_k_MaskByDistance; setBuffer(kernel, def_k_maskDist_buf_real, cbFFTReal.GetIndex()) setBuffer(kernel, def_k_maskDist_buf_imag, cbFFTImag.GetIndex()) setBuffer(kernel, def_k_maskDist_mask, getOutputIndex()) setArgument(kernel, def_k_maskDist_dimension, int(iFFTdimension)) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronChanelMask feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!FFT(NeuronOCL.getOutput(), NULL, GetPointer(cbFFTReal), GetPointer(cbFFTImag), false)) ReturnFalse; //--- return(Mask()); } //+------------------------------------------------------------------+ //| Saves the CNeuronChanelMask state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTdimension)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronChanelMask state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFTdimension = (uint)FileReadInteger(file_handle); //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronChanelMask GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronChanelMask::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cbFFTReal.BufferFree(); cbFFTImag.BufferFree(); cbFFTReal.BufferInit(iFFTdimension * iUnits, 0); cbFFTImag.BufferInit(iFFTdimension * iUnits, 0); cbFFTReal.BufferCreate(OpenCL); cbFFTImag.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDUET layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronDUET::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint units_out, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, window, units_out, optimization_type, batch)) ReturnFalse; //--- iWindowKey = MathMax(window_key, 1); iHeads = MathMax(heads, 1); //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cExperts.Init(0, index, OpenCL, units_count, units_out, window, experts, top_k, optimization, iBatch)) ReturnFalse; index++; if(!cMask.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQKV.Init(0, index, OpenCL, units_out, units_out, iHeads * iWindowKey * 3, window, 1, optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cbScores.BufferInit(cMask.Neurons()*iHeads, 0) || !cbScores.BufferCreate(OpenCL)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cPooling.Init(0, index, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, units_out, window, 1, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- index++; if(!cFeedForward.Init(0, index, OpenCL, units_out, 4 * units_out, window, 1, heads, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention out step of the CNeuronDUET layer. | //+------------------------------------------------------------------+ bool CNeuronDUET::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount/*Q units*/, iCount/*K units*/, iHeads}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); int kernel = def_k_MaskAttention; setBuffer(kernel, def_k_maskatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskatt_scores, cbScores.GetIndex()) setBuffer(kernel, def_k_maskatt_out, cMHAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_maskatt_masks, cMask.getOutputIndex()) setArgument(kernel, def_k_maskatt_dimension, (int)iWindowKey) setArgument(kernel, def_k_maskatt_heads_kv, (int)iHeads) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the attention inside gradients step of the CNeuronDUET | //| layer. | //+------------------------------------------------------------------+ bool CNeuronDUET::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iWindowKey, iHeads}; ResetLastError(); int kernel = def_k_MaskAttentionGradients; setBuffer(kernel, def_k_maskattgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_maskattgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_maskattgr_scores, cbScores.GetIndex()) setBuffer(kernel, def_k_maskattgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_maskattgr_kunits, (int)iCount) setArgument(kernel, def_k_maskattgr_heads_kv, (int)iHeads) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDUET feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronDUET::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cExperts.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cMask.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- if(!cQKV.FeedForward(cExperts.AsObject())) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), iWindowKey, 2 * iWindowKey, cQKV.GetUnits())) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(cExperts.getOutput(), cPooling.getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward.FeedForward(cResidual.AsObject())) ReturnFalse; //--- return(CNeuronTransposeOCL::feedForward(cFeedForward.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronDUET | //| layer. | //+------------------------------------------------------------------+ bool CNeuronDUET::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cFeedForward.AsObject())) ReturnFalse; if(!cPooling.CalcHiddenGradients(cFeedForward.AsObject())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), iWindowKey, 2 * iWindowKey, iCount)) ReturnFalse; if(cQKV.Activation() != None) if(!DeActivation(cQKV.getOutput(), cQKV.getGradient(), cQKV.getGradient(), cQKV.Activation())) ReturnFalse; if(!cExperts.CalcHiddenGradients(cQKV.AsObject()) || !DeActivation(cExperts.getOutput(), cExperts.getPrevOutput(), cPooling.getGradient(), cExperts.Activation()) || !SumAndNormalize(cExperts.getGradient(), cExperts.getPrevOutput(), cExperts.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cExperts.AsObject())) ReturnFalse; //--- return(prevLayer.CalcHiddenGradients(cTranspose.AsObject())); } //+------------------------------------------------------------------+ //| Updates the CNeuronDUET trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronDUET::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cExperts.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- if(!cQKV.UpdateInputWeights(cExperts.AsObject())) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cFeedForward.UpdateInputWeights(cResidual.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronDUET state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDUET::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iHeads) < INT_VALUE) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDUET state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronDUET::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cExperts.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward.AsObject())) ReturnFalse; //--- int index = 4; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cbScores.BufferInit(cMask.Neurons()*iHeads, 0) || !cbScores.BufferCreate(OpenCL)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDUET GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronDUET::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); cQKV.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cbScores.BufferCreate(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Releases the MultiWindows convolution window table. | //+------------------------------------------------------------------+ CNeuronMultiWindowsConvOCL::~CNeuronMultiWindowsConvOCL(void) { //--- Release only the live integer table that belongs to this context. if(CheckPointer(OpenCL) != POINTER_INVALID && iWindow != (uint)INVALID_HANDLE && OpenCL.GetBufferSize((int)iWindow) == aiWindows.Size() * sizeof(int)) OpenCL.BufferFree((int)iWindow); //--- Invalidate the raw context-local slot before inherited destruction. iWindow = (uint)INVALID_HANDLE; } //+------------------------------------------------------------------+ //| Initializes MultiWindows convolution and its window table. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint &windows[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Release the previous table through its current context before geometry changes. if(CheckPointer(OpenCL) != POINTER_INVALID && iWindow != (uint)INVALID_HANDLE && OpenCL.GetBufferSize((int)iWindow) == aiWindows.Size() * sizeof(int)) OpenCL.BufferFree((int)iWindow); iWindow = (uint)INVALID_HANDLE; //--- Validate and retain the requested positive window geometry. if(windows.Size() <= 0) ReturnFalseEx("positive window count"); if(ArrayCopy(aiWindows, windows, 0, 0, windows.Size()) < int(windows.Size())) ReturnFalseEx("window geometry copy"); int window = 0; for(uint i = 0; i < aiWindows.Size(); i++) { window += aiWindows[i] + 1; if(aiWindows[i] <= 0) ReturnFalse; } window--; //--- Initialize inherited owners before publishing a replacement raw table. bool result = CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count * aiWindows.Size() * variables, 1, ADAM, batch); iWindow = (uint)INVALID_HANDLE; if(!result) ReturnFalseEx("inherited convolution initialization"); //--- Create and publish the context-local integer table only after inherited success. iVariables = variables; int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalseEx("window table create"); iWindow = (uint)temp; //--- Finalize after the replacement table has a live owner. return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiWindowsConvOCL feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {Neurons() / iVariables, iVariables}; ResetLastError(); #ifdef _DEBUG if(NeuronOCL.Neurons() != OpenCL.GetBufferSize(NeuronOCL.getOutputIndex()) / sizeof(float)) ReturnFalse; if(Neurons() != OpenCL.GetBufferSize(getOutputIndex()) / sizeof(float)) ReturnFalse; if(WeightsConv.Total() != OpenCL.GetBufferSize(WeightsConv.GetIndex()) / sizeof(float)) ReturnFalse; if(aiWindows.Size() != OpenCL.GetBufferSize(iWindow) / sizeof(float)) ReturnFalse; if(!Output.BufferRead()) ReturnFalse; #endif int kernel = def_k_FeedForwardMultWinConv; setBuffer(kernel, def_k_ffmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmwc_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmwc_windows_in, iWindow) setArgument(kernel, def_k_ffmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmwc_window_out, iWindowOut) setArgument(kernel, def_k_ffmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_ffmwc_activation, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiWindowsConvOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {NeuronOCL.Neurons() / iVariables, iVariables}; ResetLastError(); int kernel = def_k_CalcHiddenGradientMultWinConv; setBuffer(kernel, def_k_chgmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgmwc_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgmwc_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_chgmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgmwc_windows_in, iWindow) setArgument(kernel, def_k_chgmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgmwc_window_out, iWindowOut) setArgument(kernel, def_k_chgmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_chgmwc_activation, NeuronOCL.Activation()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiWindowsConvOCL trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {WeightsConv.Total(), iVariables}; uint local_work_size[2] = {1, iVariables}; ResetLastError(); int kernel = def_k_UpdateWeightsMultWinConvAdam; setBuffer(kernel, def_k_uwmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmwc_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmwc_windows_in, iWindow) setBuffer(kernel, def_k_uwmwc_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmwc_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_uwmwc_window_out, iWindowOut) setArgument(kernel, def_k_uwmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_uwmwc_l, lr) setArgument(kernel, def_k_uwmwc_b1, b1) setArgument(kernel, def_k_uwmwc_b2, b2) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!SecondMomentumConv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiWindowsConvOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; uint total = aiWindows.Size(); if(FileWriteInteger(file_handle, (int)total, INT_VALUE) < INT_VALUE) ReturnFalse; for(int i = 0; i < (int)total; i++) if(FileWriteInteger(file_handle, aiWindows[i]) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores MultiWindows convolution and its window table. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Load(const int file_handle) { //--- Release the previous live table before parent deserialization changes its handle. if(CheckPointer(OpenCL) != POINTER_INVALID && iWindow != (uint)INVALID_HANDLE && OpenCL.GetBufferSize((int)iWindow) == aiWindows.Size() * sizeof(int)) OpenCL.BufferFree((int)iWindow); iWindow = (uint)INVALID_HANDLE; //--- Restore inherited convolution owners, never accepting its historical raw handle. bool result = CNeuronConvOCL::Load(file_handle); iWindow = (uint)INVALID_HANDLE; if(!result) ReturnFalseEx("inherited convolution load"); //--- Read and validate the persisted positive window table. ArrayFree(aiWindows); const ulong file_size = FileSize(file_handle); const ulong count_position = FileTell(file_handle); if(FileIsEnding(file_handle) || count_position > file_size || file_size - count_position < (ulong)INT_VALUE) ReturnFalseEx("window count read"); int total = FileReadInteger(file_handle); const ulong values_position = FileTell(file_handle); if(total <= 0 || values_position > file_size || (ulong)total > (file_size - values_position) / (ulong)INT_VALUE) ReturnFalseEx("window count"); if(ArrayResize(aiWindows, total) != total) ReturnFalseEx("window table resize"); for(int i = 0; i < total; i++) { aiWindows[i] = FileReadInteger(file_handle); if(aiWindows[i] <= 0) ReturnFalseEx("positive window value"); } //--- Verify the serialized table against the inherited convolution step. long sum = 0; for(int i = 0; i < total; i++) { sum += (long)aiWindows[i] + 1; if(sum > (long)UINT_MAX) ReturnFalseEx("window sum"); } if(iStep != uint(sum - 1)) ReturnFalseEx("window step"); //--- Allocate and publish the replacement table only after complete validation. int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalseEx("window table create"); iWindow = (uint)temp; //--- Finalize only after the fully validated table is published. return(true); } //+------------------------------------------------------------------+ //| Checks MultiWindows convolution transfer capability. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::SupportsOpenCLChecked(void) const { //--- Validate the exact type, persistent table and current raw owner. if(Type() != defNeuronMultiWindowsConvOCL || aiWindows.Size() <= 0 || CheckPointer(OpenCL) == POINTER_INVALID || iWindow == (uint)INVALID_HANDLE || OpenCL.GetBufferSize((int)iWindow) != aiWindows.Size() * sizeof(int) || CheckPointer(WeightsConv) == POINTER_INVALID) return(false); for(uint i = 0; i < aiWindows.Size(); i++) if(aiWindows[i] <= 0) return(false); //--- Require the optimizer owner set selected by the persisted profile. if(optimization == SGD) return(CheckPointer(DeltaWeightsConv) != POINTER_INVALID); //--- Finalize the non-SGD profile capability only with both moments present. return(CheckPointer(FirstMomentumConv) != POINTER_INVALID && CheckPointer(SecondMomentumConv) != POINTER_INVALID); } //+------------------------------------------------------------------+ //| Transfers MultiWindows convolution to the target context. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and current raw table before any mutation. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or window table"); //--- Let the inherited transfer reconcile float owners on the same context. if(obj == OpenCL) return(CNeuronConvOCL::SetOpenCLChecked(obj)); //--- Create the target table before the inherited transfer can publish the target. COpenCLMy *old_owner = OpenCL; const int old_window = (int)iWindow; const int candidate = obj.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(candidate == INVALID_HANDLE) ReturnFalseEx("candidate window table create"); //--- Transfer convolution and base owners while the source raw table remains live. if(!CNeuronConvOCL::SetOpenCLChecked(obj)) { if(!obj.BufferFree(candidate)) ReturnFalseEx("candidate window table cleanup"); ReturnFalseEx("inherited convolution context"); } //--- Release the old context-local slot before publishing the target candidate. if(!old_owner.BufferFree(old_window)) { iWindow = (uint)candidate; ReturnFalseEx("source window table release"); } iWindow = (uint)candidate; //--- Finalize the checked transfer with the raw table owned by the published context. return(true); } //+------------------------------------------------------------------+ //| Attaches or transfers MultiWindows convolution context. | //+------------------------------------------------------------------+ void CNeuronMultiWindowsConvOCL::SetOpenCL(COpenCLMy *obj) { //--- Attach fresh objects before Load supplies their serialized window geometry. if(CheckPointer(obj) == POINTER_INVALID) { PrintFormat("%s: invalid OpenCL attachment target", __FUNCTION__); return; } if(aiWindows.Size() <= 0) { CNeuronConvOCL::SetOpenCL(obj); return; } //--- Reuse checked transfer for initialized objects without recreating the raw table. if(!SetOpenCLChecked(obj)) PrintFormat("%s: MultiWindows checked OpenCL transfer failed", __FUNCTION__); } //+------------------------------------------------------------------+ //| Initializes the CNeuronComplexConvOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * window_out * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = (int)window; iStep = MathMax(step, 1); activation = None; iWindowOut = window_out; iVariables = variables; //--- if(CheckPointer(WeightsConv) == POINTER_INVALID) { WeightsConv = new CBufferFloat(); if(CheckPointer(WeightsConv) == POINTER_INVALID) ReturnFalse; } int count = (int)(2 * (iWindow + 1) * iWindowOut * iVariables); if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) { DeltaWeightsConv = new CBufferFloat(); if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) ReturnFalse; } if(!DeltaWeightsConv.BufferInit(count, 0.0)) ReturnFalse; if(!DeltaWeightsConv.BufferCreate(OpenCL)) ReturnFalse; } else { if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) { FirstMomentumConv = new CBufferFloat(); if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) { SecondMomentumConv = new CBufferFloat(); if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronComplexConvOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = Output.Total() / (2 * iWindowOut * iVariables); global_work_size[1] = iWindowOut; global_work_size[2] = iVariables; int kernel = def_k_FeedForwardComplexConv; setBuffer(kernel, def_k_ffc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffc_activation - 1, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronComplexConvOCL layer. | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / (2 * iVariables); global_work_size[1] = iVariables; int kernel = def_k_CalcHiddenGradientComplexConv; setBuffer(kernel, def_k_chgc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_g, Gradient.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_chgc_outputs, Neurons() / (2 * iVariables)) setArgument(kernel, def_k_chgc_step, (int)iStep) setArgument(kernel, def_k_chgc_window_in, (int)iWindow) setArgument(kernel, def_k_chgc_window_out, (int)iWindowOut) setArgument(kernel, def_k_chgc_activation, (int)NeuronOCL.Activation()) setArgument(kernel, def_k_chgc_shift_out, (int)0) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronComplexConvOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = { 0, 0 }; uint global_work_size[2] = { WeightsConv.Total() / (2 * iVariables), iVariables }; float lt = 0; int kernel = -1; switch(optimization) { case SGD: kernel = def_k_UpdateWeightsComplexConvMomentum; setBuffer(kernel, def_k_uwcm_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwcm_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwcm_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwcm_matrix_dw, DeltaWeightsConv.GetIndex()) setBuffer(kernel, def_k_uwcm_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_uwcm_learning_rates, lr) setArgument(kernel, def_k_uwcm_momentum, alpha) setBuffer(kernel, def_k_uwcm_window_in, (int)iWindow) setBuffer(kernel, def_k_uwcm_window_out, (int)iWindowOut) setBuffer(kernel, def_k_uwcm_step, (int)iStep) kernelExecute(kernel, global_work_offset, global_work_size) break; case ADAM: kernel = def_k_UpdateWeightsComplexConvAdam; setBuffer(kernel, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(kernel, def_k_uwca_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_uwca_l, lt) setArgument(kernel, def_k_uwca_b1, b1) setArgument(kernel, def_k_uwca_b2, b2) setArgument(kernel, def_k_uwca_window_in, (int)iWindow) setArgument(kernel, def_k_uwca_window_out, (int)iWindowOut) setArgument(kernel, def_k_uwca_step, (int)iStep) //--- kernelExecute(kernel, global_work_offset, global_work_size) t++; break; default: ReturnFalse; break; } #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronCATCH layer. | //+------------------------------------------------------------------+ bool CNeuronCATCH::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, uint variables, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; int kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / variables)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / variables)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_re.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCATCH layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCATCH::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint time_step, uint variables, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables, time_step, optimization_type, batch)) ReturnFalse; //--- Calculate FFT size int power = int(MathLog(time_step) / M_LN2); if(power <= 0) ReturnFalse; if(MathPow(2, power) != time_step) power++; uint FreqUinits = uint(MathPow(2, power)); //--- if(window <= 0 || step <= 0) ReturnFalse; int Segments = int((FreqUinits - int(window) + step - 1) / step); if(Segments <= 0) ReturnFalse; //--- uint index = 0; if(!cTranspose.Init(0, index, OpenCL, time_step, variables, optimization, iBatch)) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) { index++; if(!caFreqIn[i].Init(0, index, OpenCL, FreqUinits * variables, optimization, iBatch)) ReturnFalse; caFreqIn[i].SetActivationFunction(None); } index++; if(!cFreqConcat.Init(0, index, OpenCL, 2 * FreqUinits * variables, optimization, iBatch)) ReturnFalse; cFreqConcat.SetActivationFunction(None); //--- index++; if(!caProjection[0].Init(0, index, OpenCL, window, step, 2 * window, Segments, variables, optimization, iBatch)) ReturnFalse; caProjection[0].SetActivationFunction(LReLU); index++; if(!caProjection[1].Init(0, index, OpenCL, 2 * window, 2 * window, window_key, Segments, variables, optimization, iBatch)) ReturnFalse; caProjection[1].SetActivationFunction(TANH); //--- index++; if(!caChannelFusion[0].Init(0, index, OpenCL, window_key, window_key, heads, Segments, variables, optimization, iBatch)) ReturnFalse; index++; if(Segments % 2 == 0) { if(!caChannelFusion[1].Init(0, index, OpenCL, 2 * window_key, window_key, heads, Segments / 2, variables, optimization, iBatch)) ReturnFalse; } else if(!caChannelFusion[1].Init(0, index, OpenCL, window_key, window_key, heads, Segments, variables, optimization, iBatch)) ReturnFalse; //--- index++; if(!caLinearHead[0].Init(0, index, OpenCL, window_key, window_key, window, Segments, variables, optimization, iBatch)) ReturnFalse; caLinearHead[0].SetActivationFunction(LReLU); index++; if(!caLinearHead[1].Init(0, index, OpenCL, window * Segments, window * Segments, FreqUinits, variables, 1, optimization, iBatch)) ReturnFalse; caLinearHead[1].SetActivationFunction(None); //--- for(uint i = 0; i < caFreqOut.Size(); i++) { index++; if(!caFreqOut[i].Init(0, index, OpenCL, FreqUinits * variables, optimization, iBatch)) ReturnFalse; caFreqOut[i].SetActivationFunction(None); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCATCH feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronCATCH::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!FFT(cTranspose.getOutput(), NULL, caFreqIn[0].getOutput(), caFreqIn[1].getOutput(), iCount, false)) ReturnFalse; if(!Concat(caFreqIn[0].getOutput(), caFreqIn[1].getOutput(), cFreqConcat.getOutput(), 1, 1, caFreqIn[0].Neurons())) ReturnFalse; if(!ComplexNormalize(cFreqConcat.AsObject(), cFreqConcat.AsObject(), cFreqConcat.getPrevOutput(), cFreqConcat.getPrevOutput(), caFreqIn[0].Neurons() / iCount, iCount)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cFreqConcat.AsObject(); //---neuron.getOutput().BufferRead(); for(uint i = 0; i < caProjection.Size(); i++) { if(!caProjection[i].FeedForward(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); //---neuron.getOutput().BufferRead(); } //--- for(uint i = 0; i < caChannelFusion.Size(); i++) { if(!caChannelFusion[i].FeedForward(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); //---neuron.getOutput().BufferRead(); } //--- for(uint i = 0; i < caLinearHead.Size(); i++) { if(!caLinearHead[i].FeedForward(neuron)) ReturnFalse; neuron = caLinearHead[i].AsObject(); //---neuron.getOutput().BufferRead(); } //--- if(!DeConcat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), neuron.getOutput(), 1, 1, caFreqOut[0].Neurons())) ReturnFalse; if(!FFT(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caFreqOut[0].getPrevOutput(), caFreqOut[1].getPrevOutput(), iCount, true)) ReturnFalse; //--- if(!DeConcat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caFreqOut[0].getPrevOutput(), iWindow, caFreqOut[0].Neurons() / iCount - iWindow, iCount)) ReturnFalse; //--- if(!SumAndNormalize(caFreqOut[0].getOutput(), cTranspose.getOutput(), caFreqOut[0].getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(CNeuronTransposeOCL::feedForward(caFreqOut[0].AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronCATCH | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCATCH::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(caFreqOut[1].AsObject())) ReturnFalse; if(!SumAndNormalize(caFreqOut[1].getGradient(), caFreqOut[1].getGradient(), cTranspose.getPrevOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; if((caFreqOut[0].Neurons() - iWindow) > 0) if(!SumAndNormalize(caFreqOut[1].getOutput(), caFreqOut[1].getOutput(), caFreqOut[1].getOutput(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!Concat(caFreqOut[1].getGradient(), caFreqOut[1].getOutput(), caFreqOut[0].getGradient(), iWindow, caFreqOut[0].Neurons() - iWindow, iCount)) ReturnFalse; if(!SumAndNormalize(caFreqOut[1].getPrevOutput(), caFreqOut[1].getPrevOutput(), caFreqOut[1].getGradient(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; //--- if(!FFT(caFreqOut[0].getGradient(), caFreqOut[1].getGradient(), caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), iCount, false)) ReturnFalse; if(!Concat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caLinearHead[1].getGradient(), 1, 1, caFreqOut[0].Neurons())) ReturnFalse; //--- if(!caLinearHead[0].CalcHiddenGradients(caLinearHead[1].AsObject())) ReturnFalse; //--- CObject *neuron = caLinearHead[0].AsObject(); for(int i = int(caChannelFusion.Size()) - 1; i >= 0; i--) { if(!caChannelFusion[i].CalcHiddenGradients(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); } //--- for(int i = int(caProjection.Size()) - 1; i >= 0; i--) { if(!caProjection[i].CalcHiddenGradients(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); } //--- if(!cFreqConcat.CalcHiddenGradients(neuron)) ReturnFalse; //--- if(!DeConcat(caFreqIn[0].getGradient(), caFreqIn[1].getGradient(), cFreqConcat.getGradient(), 1, 1, caFreqIn[0].Neurons())) ReturnFalse; if(!FFT(caFreqIn[0].getGradient(), caFreqIn[1].getGradient(), caFreqIn[0].getPrevOutput(), caFreqIn[1].getPrevOutput(), iCount, false)) ReturnFalse; if(!DeConcat(cTranspose.getGradient(), caFreqIn[0].getGradient(), caFreqIn[0].getPrevOutput(), iWindow, caFreqIn[0].Neurons() / iCount - iWindow, iCount)) ReturnFalse; //--- if(!SumAndNormalize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCATCH trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronCATCH::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *neuron = cFreqConcat.AsObject(); for(uint i = 0; i < caProjection.Size(); i++) { if(!caProjection[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); } //--- for(uint i = 0; i < caChannelFusion.Size(); i++) { if(!caChannelFusion[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); } //--- for(uint i = 0; i < caLinearHead.Size(); i++) { if(!caLinearHead[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caLinearHead[i].AsObject(); } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronCATCH weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronCATCH::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronCATCH *Source = source; for(uint i = 0; i < caProjection.Size(); i++) if(!caProjection[i].WeightsUpdate(GetPointer(Source.caProjection[i]), tau)) ReturnFalse; //--- for(uint i = 0; i < caChannelFusion.Size(); i++) if(!caChannelFusion[i].WeightsUpdate(GetPointer(Source.caChannelFusion[i]), tau)) ReturnFalse; //--- for(uint i = 0; i < caLinearHead.Size(); i++) if(!caLinearHead[i].WeightsUpdate(GetPointer(Source.caLinearHead[i]), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCATCH state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCATCH::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) if(!caFreqIn[i].Save(file_handle)) ReturnFalse; if(!cFreqConcat.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caProjection.Size(); i++) if(!caProjection[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caChannelFusion.Size(); i++) if(!caChannelFusion[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caLinearHead.Size(); i++) if(!caLinearHead[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqOut.Size(); i++) if(!caFreqOut[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCATCH state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCATCH::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqIn[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFreqConcat.AsObject())) ReturnFalse; for(uint i = 0; i < caProjection.Size(); i++) if(!LoadInsideLayer(file_handle, caProjection[i].AsObject())) ReturnFalse; for(uint i = 0; i < caChannelFusion.Size(); i++) if(!LoadInsideLayer(file_handle, caChannelFusion[i].AsObject())) ReturnFalse; for(uint i = 0; i < caLinearHead.Size(); i++) if(!LoadInsideLayer(file_handle, caLinearHead[i].AsObject())) ReturnFalse; for(uint i = 0; i < caFreqOut.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqOut[i].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCATCH GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronCATCH::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); for(uint i = 0; i < caFreqIn.Size(); i++) caFreqIn[i].SetOpenCL(OpenCL); cFreqConcat.SetOpenCL(OpenCL); for(uint i = 0; i < caProjection.Size(); i++) caProjection[i].SetOpenCL(OpenCL); for(uint i = 0; i < caChannelFusion.Size(); i++) caChannelFusion[i].SetOpenCL(OpenCL); for(uint i = 0; i < caLinearHead.Size(); i++) caLinearHead[i].SetOpenCL(OpenCL); for(uint i = 0; i < caFreqOut.Size(); i++) caFreqOut[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Executes the complex normalize step of the CNeuronCATCH layer. | //+------------------------------------------------------------------+ bool CNeuronCATCH::ComplexNormalize(CNeuronBaseOCL* cInputFreqComplex, CNeuronBaseOCL* cNormFreqComplex, CBufferFloat * cMeans, CBufferFloat * cVariances, int dimension, int variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(dimension)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!cNormFreqComplex.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the complex normalize gradient step of the CNeuronCATCH | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCATCH::ComplexNormalizeGradient(CNeuronBaseOCL* cInputFreqComplex, CNeuronBaseOCL* cNormFreqComplex, CBufferFloat * cVariances, int dimension, int variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(dimension)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!cInputFreqComplex.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSkillsEncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint time_step, uint variables, uint skills, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, variables * skills, optimization_type, batch)) ReturnFalse; SetHeads(variables); //--- int index = 0; if(!cCrossObservAttention.Init(0, index, OpenCL, time_step, variables, window, step, window_key, heads, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, time_step, variables, optimization, iBatch)) ReturnFalse; //--- uint count = (time_step - (window - step) + (step - 1)) / step; if(count <= 1) { window = time_step; count = 1; } //--- index++; if(!cSkillsProjection[0].Init(0, index, OpenCL, window, step, (window_key + 1) / 2, count, variables, optimization, iBatch)) ReturnFalse; cSkillsProjection[0].SetActivationFunction(SoftPlus); index++; if(!cSkillsProjection[1].Init(0, index, OpenCL, (window_key + 1) / 2 * count, (window_key + 1) / 2 * count, skills, 1, variables, optimization, iBatch)) ReturnFalse; cSkillsProjection[1].SetActivationFunction(None); //--- index++; if(!cPrevSkillsConcat.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cPrevSkillsConcat.SetActivationFunction((ENUM_ACTIVATION)cSkillsProjection[1].Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSkillsEncoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cCrossObservAttention.FeedForward(NeuronOCL)) ReturnFalse; //---cCrossObservAttention.getOutput().BufferRead(); if(!cTranspose.FeedForward(cCrossObservAttention.AsObject())) ReturnFalse; if(!cSkillsProjection[0].FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cSkillsProjection[1].SwapOutputs() || !cSkillsProjection[1].FeedForward(cSkillsProjection[0].AsObject())) ReturnFalse; //--- return(CNeuronSoftMaxOCL::feedForward(cSkillsProjection[1].AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSkillsEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronSoftMaxOCL::calcInputGradients(cSkillsProjection[1].AsObject())) ReturnFalse; //--- if(!Concat(cSkillsProjection[1].getOutput(), cSkillsProjection[1].getPrevOutput(), cPrevSkillsConcat.getOutput(), cSkillsProjection[1].GetFilters(), cSkillsProjection[1].GetFilters(), iHeads)) ReturnFalse; if(!DiversityLoss(cPrevSkillsConcat.AsObject(), 2 * iHeads, cSkillsProjection[1].GetFilters(), false)) ReturnFalse; if(!DeConcat(cPrevSkillsConcat.getOutput(), cPrevSkillsConcat.getPrevOutput(), cPrevSkillsConcat.getGradient(), cSkillsProjection[1].GetFilters(), cSkillsProjection[1].GetFilters(), iHeads)) ReturnFalse; if(!SumAndNormalize(cSkillsProjection[1].getGradient(), cPrevSkillsConcat.getOutput(), cSkillsProjection[1].getGradient(), cSkillsProjection[1].GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cSkillsProjection[0].CalcHiddenGradients(cSkillsProjection[1].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cSkillsProjection[0].AsObject())) ReturnFalse; if(!cCrossObservAttention.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return(prevLayer.CalcHiddenGradients(cCrossObservAttention.AsObject())); } //+------------------------------------------------------------------+ //| Updates the CNeuronSkillsEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cCrossObservAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSkillsProjection[0].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; if(!cSkillsProjection[1].UpdateInputWeights(cSkillsProjection[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSkillsEncoder weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronSkillsEncoder *Source = source; if(!cCrossObservAttention.WeightsUpdate(GetPointer(Source.cCrossObservAttention), tau)) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!cSkillsProjection[i].WeightsUpdate(GetPointer(Source.cSkillsProjection[i]), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSkillsEncoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; if(!cCrossObservAttention.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!cSkillsProjection[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSkillsEncoder state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossObservAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!LoadInsideLayer(file_handle, cSkillsProjection[i].AsObject())) ReturnFalse; //--- if(!cPrevSkillsConcat.Init(0, 4, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cPrevSkillsConcat.SetActivationFunction((ENUM_ACTIVATION)cSkillsProjection[1].Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSkillsEncoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSkillsEncoder::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); cCrossObservAttention.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); for(uint i = 0; i < cSkillsProjection.Size(); i++) cSkillsProjection[i].SetOpenCL(OpenCL); cPrevSkillsConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTSPositionEncoder layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint & periods[], uint freqs, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * freqs * periods.Size(), optimization_type, batch)) ReturnFalse; //--- cPeriods.BufferFree(); if(!cPeriods.AssignArray(periods) || !cPeriods.BufferCreate(OpenCL)) ReturnFalse; //--- int index = 0; if(!cProjection.Init(0, index, OpenCL, window, window, 2 * freqs * cPeriods.Total(), units_count, 1, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); index++; if(!cNorm.Init(0, index, OpenCL, cProjection.Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); SetActivationFunction(None); if(!SetGradient(cNorm.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the add pe step of the CNeuronTSPositionEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::AddPE(CBufferFloat* time) { if(!time) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {GetUnits(), GetWindowOut() / (2 * cPeriods.Total()), cPeriods.Total()}; uint kernel = def_k_TSPositonEncoder; setBuffer(kernel, def_k_tspe_data, cNorm.getOutputIndex()) setBuffer(kernel, def_k_tspe_time, time.GetIndex()) setBuffer(kernel, def_k_tspe_output, getOutputIndex()) setBuffer(kernel, def_k_tspe_period, cPeriods.GetIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTSPositionEncoder feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(cProjection.AsObject())) ReturnFalse; //--- return(AddPE(SecondInput)); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTSPositionEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cProjection.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return(NeuronOCL.CalcHiddenGradients(cProjection.AsObject())); } //+------------------------------------------------------------------+ //| Updates the CNeuronTSPositionEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(cProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronTSPositionEncoder weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronTSPositionEncoder* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTSPositionEncoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cPeriods.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTSPositionEncoder state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cPeriods.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!SetGradient(cNorm.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTSPositionEncoder GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTSPositionEncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cPeriods.BufferFree(); cPeriods.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiWindowsConvWPadOCL layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(windows.Size() <= 0 || ArrayCopy(aiWindows, windows) < int(windows.Size())) ReturnFalse; int window = 0; for(uint i = 0; i < aiWindows.Size(); i++) window += aiWindows[i] + 1; window--; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, step, window_out, units_count * aiWindows.Size() * variables, 1, ADAM, batch)) ReturnFalse; //--- iVariables = variables; int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiWindowsConvWPadOCL feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = { GetUnits(), aiWindows.Size(), iVariables}; //--- uint inputs = NeuronOCL.Neurons() / iVariables; for(uint id = 0; id < global_work_size[0]; id++) { for(uint id_w = 0; id_w < global_work_size[1]; id_w++) { for(uint v = 0; id < global_work_size[2]; id++) { //--- int window_in = aiWindows[id_w]; int mid_win = window_in / 2; int shift_in = int(id * iStep) - mid_win; uint shift_in_var = v * inputs; uint shift_weight = 0; //--- for(uint w = 0; w < id_w; w++) shift_weight += (aiWindows[w] + 1) * iWindowOut; //--- for(uint w_out = 0; w_out < iWindowOut; w_out++) { float sum = WeightsConv[shift_weight + window_in]; //--- for(int w = 0; w < window_in; w++) if((shift_in + w) >= 0 && (shift_in + w) < int(inputs)) sum += NeuronOCL.getOutput()[shift_in_var + shift_in + w] * WeightsConv[shift_weight + w]; //--- uint shift_out = (v * global_work_size[0] + id) * iWindowOut + w_out; Output.Update(shift_out, sum); shift_weight += window_in + 1; } } } } //--- ResetLastError(); int kernel = def_k_FeedForwardMultWinConvWPad; setBuffer(kernel, def_k_ffmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmwconvwpad_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmwconvwpad_windows_in, iWindow) setArgument(kernel, def_k_ffmwconvwpad_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmwconvwpad_window_out, iWindowOut) setArgument(kernel, def_k_ffmwconvwpad_step, (int)iStep) setArgument(kernel, def_k_ffmwconvwpad_activation, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiWindowsConvWPadOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint filters = aiWindows.Size() * iWindowOut; uint global_work_offset[3] = {0}; uint global_work_size[3] = {NeuronOCL.Neurons() / iVariables, (uint)MathMin(filters, OpenCL.GetMaxLocalSize(1)), iVariables }; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); int kernel = def_k_CalcHiddenGradientMultWinConvWPad; setBuffer(kernel, def_k_hgmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_hgmwconvwpad_windows_in, iWindow) setArgument(kernel, def_k_hgmwconvwpad_outputs, GetUnits()) setArgument(kernel, def_k_hgmwconvwpad_window_out, (int)iWindowOut) setArgument(kernel, def_k_hgmwconvwpad_step, (int)iStep) setArgument(kernel, def_k_hgmwconvwpad_filters, (int)filters) setArgument(kernel, def_k_hgmwconvwpad_activation, NeuronOCL.Activation()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiWindowsConvWPadOCL trainable weights | //| from the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {WeightsConv.Total(), iVariables}; uint local_work_size[2] = {1, iVariables}; ResetLastError(); int kernel = def_k_UpdateWeightsMultWinConvAdamWPad; setBuffer(kernel, def_k_uwmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmwconvwpad_windows_in, iWindow) setBuffer(kernel, def_k_uwmwconvwpad_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwmwconvwpad_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmwconvwpad_outputs, GetUnits()) setArgument(kernel, def_k_uwmwconvwpad_window_out, iWindowOut) setArgument(kernel, def_k_uwmwconvwpad_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_uwmwconvwpad_l, lr) setArgument(kernel, def_k_uwmwconvwpad_b1, b1) setArgument(kernel, def_k_uwmwconvwpad_b2, b2) setArgument(kernel, def_k_uwmwconvwpad_step, (int)iStep) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiWindowsConvWPadOCL state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiWindows.Size()) < INT_VALUE) ReturnFalse; for(int i = 0; i < (int)aiWindows.Size(); i++) if(FileWriteInteger(file_handle, (int)aiWindows[i]) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiWindowsConvWPadOCL state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Load(const int file_handle) { ArrayFree(aiWindows); if(iWindow >= 0) OpenCL.BufferFree(iWindow); //--- if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; iWindow = INVALID_HANDLE; //--- int total = FileReadInteger(file_handle); if(ArrayResize(aiWindows, total) != total) ReturnFalse; for(int i = 0; i < total; i++) aiWindows[i] = FileReadInteger(file_handle); int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultiWindowsConvWPadOCL GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultiWindowsConvWPadOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && iWindow >= 0) OpenCL.BufferFree(iWindow); CNeuronConvOCL::SetOpenCL(obj); iWindow = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); } //+------------------------------------------------------------------+ //| Initializes the CNeuronConcatDiff layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint units_count, uint step, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * variables, optimization_type, batch)) ReturnFalse; if(step <= 0 || step >= units_count) ReturnFalse; //--- iUnits = units_count; iVariables = variables; iStep = step; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronConcatDiff feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL || !Output) ReturnFalse; if(getOutputIndex() < 0) ReturnFalse; //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iVariables}; const int kernel = def_k_ConcatDiff; setBuffer(kernel, def_k_concdiff_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_concdiff_output, getOutputIndex()) setArgument(kernel, def_k_concdiff_step, iStep) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {2 * iUnits}; const int kernel = def_k_Normalize; setBuffer(kernel, def_k_norm_buffer, getOutputIndex()) setArgument(kernel, def_k_norm_dimension, iVariables) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronConcatDiff layer. | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), getPrevOutput(), getGradient(), iVariables, iVariables, iUnits)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronConcatDiff state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!FileWriteInteger(file_handle, int(iUnits))) ReturnFalse; if(!FileWriteInteger(file_handle, int(iVariables))) ReturnFalse; if(!FileWriteInteger(file_handle, int(iStep))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronConcatDiff state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iStep = (uint)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMantisPatching layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables * embedding_size, patchs, optimization_type, batch)) ReturnFalse; int index = 0; if(!cToVarSeq.Init(0, index, OpenCL, count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cProjecting.Init(0, index, OpenCL, 3, 1, embedding_size, count - 2, variables, optimization, iBatch)) ReturnFalse; cProjecting.SetActivationFunction(SoftPlus); index++; if(!cToVarProjSeq.Init(0, index, OpenCL, variables, count - 2, embedding_size, optimization, iBatch)) ReturnFalse; index++; int patch_size = (int(count + patchs) - 3) / int(patchs); if(!cPatchingProj.Init(0, index, OpenCL, patch_size, patch_size, patch_filters, patchs, variables * embedding_size, optimization, iBatch)) ReturnFalse; cPatchingProj.SetActivationFunction(SoftPlus); index++; if(!cProof.Init(0, index, OpenCL, patch_filters, patch_filters, patchs * variables * embedding_size, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMantisPatching feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cToVarSeq.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjecting.FeedForward(cToVarSeq.AsObject())) ReturnFalse; if(!cToVarProjSeq.FeedForward(cProjecting.AsObject())) ReturnFalse; if(!cPatchingProj.FeedForward(cToVarProjSeq.AsObject())) ReturnFalse; if(!cProof.FeedForward(cPatchingProj.AsObject())) ReturnFalse; if(!CNeuronTransposeOCL::feedForward(cProof.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMantisPatching layer. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cProof.AsObject())) ReturnFalse; if(!cPatchingProj.CalcHiddenGradients(cProof.AsObject())) ReturnFalse; if(!cToVarProjSeq.CalcHiddenGradients(cPatchingProj.AsObject())) ReturnFalse; if(!cProjecting.CalcHiddenGradients(cToVarProjSeq.AsObject())) ReturnFalse; if(!cToVarSeq.CalcHiddenGradients(cProjecting.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cToVarSeq.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMantisPatching trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cProjecting.UpdateInputWeights(cToVarSeq.AsObject())) ReturnFalse; if(!cPatchingProj.UpdateInputWeights(cToVarProjSeq.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMantisPatching state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; if(!cToVarSeq.Save(file_handle)) ReturnFalse; if(!cProjecting.Save(file_handle)) ReturnFalse; if(!cToVarProjSeq.Save(file_handle)) ReturnFalse; if(!cPatchingProj.Save(file_handle)) ReturnFalse; if(!cProof.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMantisPatching state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjecting.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarProjSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProof.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMantisPatching GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMantisPatching::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cToVarSeq.SetOpenCL(obj); cProjecting.SetOpenCL(obj); cToVarProjSeq.SetOpenCL(obj); cPatchingProj.SetOpenCL(obj); cProof.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| Averages the CNeuronMantisPatching weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronTransposeOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMantisPatching *Source = source; if(!cProjecting.WeightsUpdate(Source.cProjecting.AsObject(), tau)) ReturnFalse; if(!cPatchingProj.WeightsUpdate(Source.cPatchingProj.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMantisAttentionUnit layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint token_size, uint window, uint window_key, uint heads, uint units_count, uint layers, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, token_size, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cClassToken[0].Init(token_size, index, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!cClassToken[0].getOutput().Fill(1)) ReturnFalse; index++; if(!cClassToken[1].Init(0, index, OpenCL, token_size, optimization, iBatch)) ReturnFalse; cClassToken[1].SetActivationFunction(SIGMOID); //--- index++; if(!cAttention.Init(0, index, OpenCL, token_size, window_key, heads * variables, window, heads, 1, units_count, layers, 1, 1, variables, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMantisAttentionUnit feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(bTrain) { if(!cClassToken[1].FeedForward(cClassToken[0].AsObject())) ReturnFalse; } if(!cAttention.FeedForward(cClassToken[1].AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!CNeuronSoftMaxOCL::feedForward(cAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMantisAttentionUnit layer. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSoftMaxOCL::calcInputGradients(cAttention.AsObject())) ReturnFalse; if(!cClassToken[1].CalcHiddenGradients(cAttention.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!cClassToken[0].CalcHiddenGradients(cClassToken[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMantisAttentionUnit trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cClassToken[1].UpdateInputWeights(cClassToken[0].AsObject())) ReturnFalse; if(!cAttention.UpdateInputWeights(cClassToken[1].AsObject(), NeuronOCL.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMantisAttentionUnit state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < cClassToken.Size(); i++) if(!cClassToken[i].Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMantisAttentionUnit state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < cClassToken.Size(); i++) if(!LoadInsideLayer(file_handle, cClassToken[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMantisAttentionUnit GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMantisAttentionUnit::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); for(uint i = 0; i < cClassToken.Size(); i++) cClassToken[i].SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Averages the CNeuronMantisAttentionUnit weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronSoftMaxOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMantisAttentionUnit *Source = source; for(uint i = 0; i < cClassToken.Size(); i++) if(!cClassToken[i].WeightsUpdate(Source.cClassToken[i].AsObject(), tau)) ReturnFalse; if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTimeFoundPatching layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch) { uint inside_emb = MathMax((embedding_size + 2) / 6, 8); if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, 3 * inside_emb, 3 * inside_emb, embedding_size, patchs * variables, 1, optimization_type, batch)) ReturnFalse; SetActivationFunction(SoftPlus); int index = 0; if(!cToVarSeq.Init(0, index, OpenCL, count, variables, optimization, iBatch)) ReturnFalse; index++; { uint windows[] = {3, 5, 7}; if(!cProjecting.Init(0, index, OpenCL, windows, 1, inside_emb, count, variables, optimization, iBatch)) ReturnFalse; cProjecting.SetActivationFunction(SoftPlus); } index++; if(!cToVarProjSeq.Init(0, index, OpenCL, variables, count, 3 * inside_emb, optimization, iBatch)) ReturnFalse; index++; { uint step = (count + patchs) / (patchs + 1); uint windows[] = {step, 3 * step / 2, 2 * step}; if(!cPatchingProj.Init(0, index, OpenCL, windows, step, patch_filters, patchs, 3 * inside_emb * variables, optimization, iBatch)) ReturnFalse; cPatchingProj.SetActivationFunction(SoftPlus); } index++; if(!cPatchsFeedForward.Init(0, index, OpenCL, 3 * patch_filters, 6 * patch_filters, patchs, 3 * inside_emb * variables, 3, optimization, iBatch)) ReturnFalse; index++; if(!cPatchingAgr.Init(0, index, OpenCL, 3 * patch_filters, 3 * patch_filters, patch_filters, patchs, 3 * inside_emb * variables, optimization, iBatch)) ReturnFalse; cPatchingAgr.SetActivationFunction(SoftPlus); index++; if(!cProof.Init(0, index, OpenCL, patch_filters, patch_filters, 3 * patchs * inside_emb * variables, optimization, iBatch)) ReturnFalse; index++; if(!cToPatchVarEmb.Init(0, index, OpenCL, variables * 3 * inside_emb, patchs, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTimeFoundPatching feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cToVarSeq.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjecting.FeedForward(cToVarSeq.AsObject())) ReturnFalse; if(!cToVarProjSeq.FeedForward(cProjecting.AsObject())) ReturnFalse; if(!cPatchingProj.FeedForward(cToVarProjSeq.AsObject())) ReturnFalse; if(!cPatchsFeedForward.FeedForward(cPatchingProj.AsObject())) ReturnFalse; if(!cPatchingAgr.FeedForward(cPatchsFeedForward.AsObject())) ReturnFalse; if(!cProof.FeedForward(cPatchingAgr.AsObject())) ReturnFalse; if(!cToPatchVarEmb.FeedForward(cProof.AsObject())) ReturnFalse; //--- return(CNeuronConvOCL::feedForward(cToPatchVarEmb.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTimeFoundPatching layer. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cToPatchVarEmb.AsObject())) ReturnFalse; if(!cProof.CalcHiddenGradients(cToPatchVarEmb.AsObject())) ReturnFalse; if(cPatchingAgr.Activation() != None) if(!DeActivation(cPatchingAgr.getOutput(), cPatchingAgr.getGradient(), cPatchingAgr.getGradient(), cPatchingAgr.Activation())) ReturnFalse; if(!cPatchingAgr.CalcHiddenGradients(cProof.AsObject())) ReturnFalse; if(!cPatchsFeedForward.CalcHiddenGradients(cPatchingAgr.AsObject())) ReturnFalse; if(!cPatchingProj.CalcHiddenGradients(cPatchsFeedForward.AsObject())) ReturnFalse; if(!cToVarProjSeq.CalcHiddenGradients(cPatchingProj.AsObject())) ReturnFalse; if(!cProjecting.CalcHiddenGradients(cToVarProjSeq.AsObject())) ReturnFalse; if(!cToVarSeq.CalcHiddenGradients(cProjecting.AsObject())) ReturnFalse; //--- return(NeuronOCL.CalcHiddenGradients(cToVarSeq.AsObject())); } //+------------------------------------------------------------------+ //| Updates the CNeuronTimeFoundPatching trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cProjecting.UpdateInputWeights(cToVarSeq.AsObject())) ReturnFalse; if(!cPatchingProj.UpdateInputWeights(cToVarProjSeq.AsObject())) ReturnFalse; if(!cPatchsFeedForward.UpdateInputWeights(cPatchingProj.AsObject())) ReturnFalse; if(!cPatchingAgr.UpdateInputWeights(cPatchsFeedForward.AsObject())) ReturnFalse; //--- return(CNeuronConvOCL::updateInputWeights(cToPatchVarEmb.AsObject())); } //+------------------------------------------------------------------+ //| Averages the CNeuronTimeFoundPatching weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronTimeFoundPatching *Source = source; if(!cProjecting.WeightsUpdate(Source.cProjecting.AsObject(), tau)) ReturnFalse; if(!cPatchingProj.WeightsUpdate(Source.cPatchingProj.AsObject(), tau)) ReturnFalse; if(!cPatchsFeedForward.WeightsUpdate(Source.cPatchsFeedForward.AsObject(), tau)) ReturnFalse; if(!cPatchingAgr.WeightsUpdate(Source.cPatchingAgr.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTimeFoundPatching state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cToVarSeq.Save(file_handle)) ReturnFalse; if(!cProjecting.Save(file_handle)) ReturnFalse; if(!cToVarProjSeq.Save(file_handle)) ReturnFalse; if(!cPatchingProj.Save(file_handle)) ReturnFalse; if(!cPatchsFeedForward.Save(file_handle)) ReturnFalse; if(!cPatchingAgr.Save(file_handle)) ReturnFalse; if(!cProof.Save(file_handle)) ReturnFalse; if(!cToPatchVarEmb.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTimeFoundPatching state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjecting.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarProjSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchsFeedForward.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingAgr.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProof.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToPatchVarEmb.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTimeFoundPatching GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTimeFoundPatching::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); cToVarSeq.SetOpenCL(OpenCL); cProjecting.SetOpenCL(OpenCL); cToVarProjSeq.SetOpenCL(OpenCL); cPatchingProj.SetOpenCL(OpenCL); cPatchsFeedForward.SetOpenCL(OpenCL); cPatchingAgr.SetOpenCL(OpenCL); cProof.SetOpenCL(OpenCL); cToPatchVarEmb.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTimeFoundTransformerUnit layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossDMHAttention::Init(numOutputs, myIndex, open_cl, window, window_key, variables, window, units_count * variables, heads, layers, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); if(!cSelfAttention.Init(0, 0, OpenCL, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, variables, optimization, iBatch)) ReturnFalse; cSelfAttention.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTimeFoundTransformerUnit feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronCrossDMHAttention::feedForward(NeuronOCL, cSelfAttention.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTimeFoundTransformerUnit layer. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; CBufferFloat *grad = prevLayer.getGradient(); if(!grad || !grad.Fill(0)) ReturnFalse; //--- if(!CNeuronCrossDMHAttention::calcInputGradients(prevLayer, cSelfAttention.getOutput(), cSelfAttention.getPrevOutput(), (ENUM_ACTIVATION)cSelfAttention.Activation())) ReturnFalse; if(!prevLayer.SetGradient(cSelfAttention.getPrevOutput(), false) || !prevLayer.CalcHiddenGradients(cSelfAttention.AsObject()) || !SumAndNormalize(grad, prevLayer.getGradient(), grad, GetWindow(), false, 0, 0, 0, 1) || !prevLayer.SetGradient(grad, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTimeFoundTransformerUnit trainable weights | //| from the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronCrossDMHAttention::updateInputWeights(NeuronOCL, cSelfAttention.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronTimeFoundTransformerUnit weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronCrossDMHAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTimeFoundTransformerUnit *Source = source; if(!cSelfAttention.WeightsUpdate(Source.cSelfAttention.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTimeFoundTransformerUnit state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Save(const int file_handle) { if(!CNeuronCrossDMHAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTimeFoundTransformerUnit state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Load(const int file_handle) { if(!CNeuronCrossDMHAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTimeFoundTransformerUnit GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTimeFoundTransformerUnit::SetOpenCL(COpenCLMy * obj) { CNeuronCrossDMHAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTimeMoESparseExperts layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, uint experts, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; if(!cExpertsIn.Init(0, index, OpenCL, window, window, window_out * experts, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cSharedIn.Init(0, index, OpenCL, window, window, window_out, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cExpertsOut.Init(0, index, OpenCL, window_out, experts, window, units_count, variables, optimization, iBatch)) ReturnFalse; cExpertsOut.SetActivationFunction(None); index++; if(!cSharedOut.Init(0, index, OpenCL, window_out, window_out, window, units_count, variables, optimization, iBatch)) ReturnFalse; cSharedOut.SetActivationFunction(None); index++; if(!cMasks.Init(0, index, OpenCL, window, units_count, experts, topK, optimization, iBatch)) ReturnFalse; index++; if(!cSharedGates.Init(0, index, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; cSharedGates.SetActivationFunction(SIGMOID); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTimeMoESparseExperts feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cExpertsIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cSharedIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cSharedGates.FeedForward(NeuronOCL)) ReturnFalse; if(!cMasks.FeedForward(NeuronOCL)) ReturnFalse; if(!cExpertsOut.FeedForward(cExpertsIn.AsObject(), cMasks.getOutput())) ReturnFalse; if(!cSharedOut.FeedForward(cSharedIn.AsObject())) ReturnFalse; if(!ElementMult(cSharedOut.getOutput(), cSharedGates.getOutput(), cSharedOut.getPrevOutput())) ReturnFalse; const int window = (int)cSharedGates.GetWindow(); if(!SumAndNormalize(cExpertsOut.getOutput(), cSharedOut.getPrevOutput(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), PrevOutput, Output, window, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTimeMoESparseExperts layer. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cExpertsOut.getOutput(), cExpertsOut.getGradient(), Gradient, cExpertsOut.Activation())) ReturnFalse; if(!ElementMultGrad(cSharedOut.getOutput(), cSharedOut.getGradient(), cSharedGates.getOutput(), cSharedGates.getGradient(), Gradient, cSharedOut.Activation(), cSharedGates.Activation())) ReturnFalse; //--- if(!cExpertsIn.CalcHiddenGradients(cExpertsOut.AsObject(), cMasks.getOutput(), cMasks.getGradient(), (ENUM_ACTIVATION)cMasks.Activation())) ReturnFalse; if(!cSharedIn.CalcHiddenGradients(cSharedOut.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cExpertsIn.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; const int window = (int)cSharedGates.GetWindow(); if(!SumAndNormalize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSharedIn.AsObject())) ReturnFalse; if(!SumAndNormalize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMasks.AsObject())) ReturnFalse; if(!SumAndNormalize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSharedGates.AsObject())) ReturnFalse; if(!SumAndNormalize(PrevOutput, NeuronOCL.getGradient(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTimeMoESparseExperts trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cExpertsIn.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSharedIn.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSharedGates.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMasks.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cExpertsOut.UpdateInputWeights(cExpertsIn.AsObject(), cMasks.getOutput())) ReturnFalse; if(!cSharedOut.UpdateInputWeights(cSharedIn.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronTimeMoESparseExperts weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTimeMoESparseExperts *Source = source; if(!cExpertsIn.WeightsUpdate(Source.cExpertsIn.AsObject(), tau)) ReturnFalse; if(!cSharedIn.WeightsUpdate(Source.cSharedIn.AsObject(), tau)) ReturnFalse; if(!cSharedGates.WeightsUpdate(Source.cSharedGates.AsObject(), tau)) ReturnFalse; if(!cMasks.WeightsUpdate(Source.cMasks.AsObject(), tau)) ReturnFalse; if(!cExpertsOut.WeightsUpdate(Source.cExpertsOut.AsObject(), tau)) ReturnFalse; if(!cSharedOut.WeightsUpdate(Source.cSharedOut.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTimeMoESparseExperts state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cExpertsIn.Save(file_handle)) ReturnFalse; if(!cSharedIn.Save(file_handle)) ReturnFalse; if(!cExpertsOut.Save(file_handle)) ReturnFalse; if(!cSharedOut.Save(file_handle)) ReturnFalse; if(!cMasks.Save(file_handle)) ReturnFalse; if(!cSharedGates.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTimeMoESparseExperts state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cExpertsIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cExpertsOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMasks.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedGates.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTimeMoESparseExperts GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTimeMoESparseExperts::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cExpertsIn.SetOpenCL(OpenCL); cSharedIn.SetOpenCL(OpenCL); cExpertsOut.SetOpenCL(OpenCL); cSharedOut.SetOpenCL(OpenCL); cMasks.SetOpenCL(OpenCL); cSharedGates.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronTimeMoESparseExperts layer. | //+------------------------------------------------------------------+ void CNeuronTimeMoESparseExperts::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cExpertsIn.TrainMode(bTrain); cSharedIn.TrainMode(bTrain); cExpertsOut.TrainMode(bTrain); cSharedOut.TrainMode(bTrain); cMasks.TrainMode(bTrain); cSharedGates.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTimeMoEAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronRelativeCrossAttention *cross = NULL; CNeuronTimeMoESparseExperts *MoE = NULL; bool use_self = units_count > 0; int layer = 0; for(uint i = 0; i < layers; i++) { if(use_self) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, layer, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } layer++; } cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, layer, OpenCL, window, window_key, units_count, heads, window_cross, units_cross, optimization, iBatch) || !cLayers.Add(cross) ) { DeleteObj(cross); ReturnFalse; } layer++; MoE = new CNeuronTimeMoESparseExperts(); if(!MoE || !MoE.Init(0, layer, OpenCL, window, experts_dimension, units_count, 1, experts, topK, optimization, iBatch) || !cLayers.Add(MoE) ) { DeleteObj(MoE); ReturnFalse; } layer++; } //--- SetOutput(MoE.getOutput(), true); SetGradient(MoE.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTimeMoEAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; return(CNeuronCrossDMHAttention::feedForward(NeuronOCL, NeuronOCL.getOutput())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTimeMoEAttention layer. | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; CBufferFloat *temp = prevLayer.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; if(!CNeuronCrossDMHAttention::calcInputGradients(prevLayer, prevLayer.getOutput(), prevLayer.getPrevOutput(), (ENUM_ACTIVATION) prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(temp, prevLayer.getPrevOutput(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTimeMoEAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- return(CNeuronCrossDMHAttention::updateInputWeights(NeuronOCL, NeuronOCL.getOutput())); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAdaptConv layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; //--- bMainFreq.BufferFree(); if(!bMainFreq.BufferInit(iVariables, 1) || !bMainFreq.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the fft step of the CNeuronAdaptConv layer. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::FFT(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* out_re, CBufferFloat* out_im, uint variables, bool reverse = false ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; uint kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / variables)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / variables)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out_im.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the periods finding step of the CNeuronAdaptConv layer. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::PeriodsFinding(CBufferFloat* inp_re, CBufferFloat* inp_im, CBufferFloat* windows, uint variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; uint kernel = def_k_MainFreq; setBuffer(kernel, def_k_mf_freq_r, inp_re.GetIndex()) setBuffer(kernel, def_k_mf_freq_im, inp_im.GetIndex()) setBuffer(kernel, def_k_mf_main_freq, windows.GetIndex()) setArgument(kernel, def_k_mf_dimension, (int)(inp_re.Total() / variables)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!windows.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the adaptive convolution step of the CNeuronAdaptConv | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::AdaptiveConvolution(CNeuronBaseOCL* NeuronOCL, CBufferFloat* main_freq) { uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {Output.Total() / (iWindowOut * iVariables), iWindowOut, iVariables}; uint kernel = def_k_FeedForwardAdaptConv; setBuffer(kernel, def_k_ffac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffac_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffac_main_freq, main_freq.GetIndex()) setArgument(kernel, def_k_ffac_window_in, iWindow) setArgument(kernel, def_k_ffac_inputs, (int)(NeuronOCL.Neurons())) setArgument(kernel, def_k_ffac_activation, (int)(activation)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAdaptConv feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!FFT(NeuronOCL.getOutput(), NULL, Output, PrevOutput, iVariables, false)) ReturnFalse; if(!PeriodsFinding(Output, PrevOutput, GetPointer(bMainFreq), iVariables)) ReturnFalse; //--- return(AdaptiveConvolution(NeuronOCL, GetPointer(bMainFreq))); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronAdaptConv | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {NeuronOCL.Neurons() / iVariables, iVariables}; uint kernel = def_k_CalcHiddenGradientAdaptConv; setBuffer(kernel, def_k_chgac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgac_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgac_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_chgac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgac_main_freq, bMainFreq.GetIndex()) setArgument(kernel, def_k_chgac_window_in, iWindow) setArgument(kernel, def_k_chgac_window_out, iWindowOut) setArgument(kernel, def_k_chgac_activation, (int)(NeuronOCL.Activation())) setArgument(kernel, def_k_chgac_outputs, (int)(Neurons())) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAdaptConv trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow + 1, iWindowOut, iVariables}; uint kernel = def_k_UpdateWeightsAdaptConvAdam; setBuffer(kernel, def_k_uwac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwac_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwac_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwac_matrix_v, SecondMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwac_main_freq, bMainFreq.GetIndex()) setArgument(kernel, def_k_uwac_outputs, (int)Neurons()) setArgument(kernel, def_k_uwac_inputs, int(NeuronOCL.Neurons())) setArgument(kernel, def_k_uwac_l, lr) setArgument(kernel, def_k_uwac_b1, b1) setArgument(kernel, def_k_uwac_b2, b2) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronAdaptConv state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!bMainFreq.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAdaptConv state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!bMainFreq.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronAdaptConv GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronAdaptConv::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); bMainFreq.BufferFree(); bMainFreq.BufferFree(); bMainFreq.BufferInit(iVariables, 1); bMainFreq.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronK2VAEEncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_cross, uint heads, uint layers, uint scenarios, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * scenarios, optimization_type, batch)) ReturnFalse; //--- Koopman int index = 0; if(!cKoopman.Init(0, index, OpenCL, window, 2 * window, units_cross, 1, experts, topK, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanRest.Init(0, index, OpenCL, window * (units_cross - 1), optimization, iBatch)) ReturnFalse; //--- index++; if(!cAuxiliaryNet.Init(0, index, OpenCL, window, window_key, 1, window, units_cross - 1, heads, layers, experts, experts_dimension, topK, optimization, iBatch)) ReturnFalse; //--- Kalman Filter index++; if(!cB.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cB.Identity(window, window)) ReturnFalse; index++; if(!cF.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cF.Identity(window, window)) ReturnFalse; index++; if(!cH.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cH.Identity(window, window)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cQ.Identity(window, window)) ReturnFalse; index++; if(!cR.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cR.Identity(window, window)) ReturnFalse; index++; if(!cP.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cP.getOutput().Fill(matrix::Identity(window, window))) ReturnFalse; //--- index++; if(!cFT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cHT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cQT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cRT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cPT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQ_QT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cR_RT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cXPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cF_P.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cPPred.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cP_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cH_P_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- mS = mSGrad = matrix::Zeros(window, window); //--- index++; if(!cSInv.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cK.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cKT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cYPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cDeltY.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cX.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_HT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- mKL = mP = mPGrad = mS; mNoise = mGrad = matrix::Zeros(scenarios, window); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronK2VAEEncoder feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Koopman if(!cKoopman.FeedForward(NeuronOCL)) ReturnFalse; //--- Pred / Rest if(!DeConcat(cKoopmanPred.getOutput(), cKoopmanRest.getPrevOutput(), cKoopman.getOutput(), cKoopman.GetWindow(), cKoopman.Neurons() - cKoopman.GetWindow(), 1)) ReturnFalse; if(!Different(NeuronOCL.getOutput(), cKoopmanRest.getPrevOutput(), cKoopmanRest.getOutput(), cKoopman.GetWindow())) ReturnFalse; //--- Rest Attention if(!cAuxiliaryNet.FeedForward(cKoopmanRest.AsObject())) ReturnFalse; //--- Kalman Filter if(bTrain) { if(!cB.FeedForward()) ReturnFalse; if(!cF.FeedForward()) ReturnFalse; if(!cH.FeedForward()) ReturnFalse; if(!cQ.FeedForward()) ReturnFalse; if(!cR.FeedForward()) ReturnFalse; if(!cFT.FeedForward(cF.AsObject())) ReturnFalse; if(!cHT.FeedForward(cH.AsObject())) ReturnFalse; if(!cQT.FeedForward(cQ.AsObject())) ReturnFalse; if(!cRT.FeedForward(cR.AsObject())) ReturnFalse; if(!MatMul(cQ.getOutput(), cQT.getOutput(), cQ_QT.getOutput(), cQT.GetCount(), cQT.GetWindow(), cQT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cR.getOutput(), cRT.getOutput(), cR_RT.getOutput(), cRT.GetCount(), cRT.GetWindow(), cRT.GetCount(), 1, true)) ReturnFalse; } //--- Prediction step if(!MatMul(NeuronOCL.getOutput(), cFT.getOutput(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cAuxiliaryNet.getOutput(), cB.getOutput(), cXPred.getPrevOutput(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cXPred.getGradient(), cXPred.getPrevOutput(), cXPred.getOutput(), cFT.GetCount(), false, 0, 0, 0, 1)) ReturnFalse; //--- cP.SwapOutputs(); if(!MatMul(cF.getOutput(), cP.getPrevOutput(), cF_P.getOutput(), cFT.GetCount(), cFT.GetWindow(), cP.Neurons() / cFT.GetWindow(), 1, true)) ReturnFalse; if(!MatMul(cF_P.getOutput(), cFT.getOutput(), cPPred.getOutput(), cFT.GetCount(), cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cPPred.getOutput(), cQ_QT.getOutput(), cPPred.getOutput(), cHT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- Update step if(!MatMul(cPPred.getOutput(), cHT.getOutput(), cP_HT.getOutput(), cPPred.Neurons() / cHT.GetWindow(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cH.getOutput(), cP_HT.getOutput(), cH_P_HT.getOutput(), cHT.GetCount(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cH_P_HT.getOutput(), cR_RT.getOutput(), cR_RT.getPrevOutput(), cRT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(cR_RT.getPrevOutput().GetData(mSGrad) <= 0) ReturnFalse; mS = mSGrad.Inv(); if(mS.Rows() == 0) { double max = MathAbs(mSGrad).Max(); if(max > 1) mSGrad = mSGrad / max; mSGrad = mSGrad + matrix::Identity(mSGrad.Rows(), mSGrad.Cols()); mS = mSGrad.Inv(); if(mS.Rows() == 0) { mSGrad = mSGrad + mSGrad.Transpose(); vector eigvals; matrix eigvecs; if(!mSGrad.Eig(eigvecs, eigvals)) ReturnFalse; if(eigvals.Size() > 0) { if(!eigvals.Clip(1e-6, DBL_MAX)) ReturnFalse; mSGrad = matrix::Zeros(eigvals.Size(), eigvals.Size()); mSGrad.Diag(eigvals); mSGrad = eigvecs.MatMul(mSGrad.MatMul(eigvecs.Transpose())); mSGrad = mSGrad + mSGrad.Transpose(); mS = mSGrad.Inv(); } if(mS.Rows() == 0) { mSGrad.Identity(); mS = mSGrad; } } } cSInv.getOutput().Fill(mS); if(!MatMul(cP_HT.getOutput(), cSInv.getOutput(), cK.getOutput(), cHT.GetCount(), (int)mS.Rows(), (int)mS.Cols(), 1, true)) ReturnFalse; //--- Measurement update if(!MatMul(cXPred.getOutput(), cHT.getOutput(), cYPred.getOutput(), 1, cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!Different(cKoopmanPred.getOutput(), cYPred.getOutput(), cDeltY.getOutput(), 1, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cDeltY.getOutput(), cK.getOutput(), cX.getPrevOutput(), 1, cDeltY.Neurons(), cX.Neurons(), 1, true)) ReturnFalse; if(!SumAndNormalize(cX.getPrevOutput(), cXPred.getOutput(), cX.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Joseph stabilized form for P if(!MatMul(cK.getOutput(), cH.getOutput(), cK_H.getOutput(), cKT.GetCount(), cKT.GetWindow(), cHT.GetWindow(), 1, true)) ReturnFalse; if(!IdentDifferent(cK_H.getOutput(), cIdifK_H.getOutput(), cHT.GetWindow(), 0, 0, 1)) ReturnFalse; if(!cIdifK_HT.FeedForward(cIdifK_H.AsObject())) ReturnFalse; if(!cKT.FeedForward(cK.AsObject())) ReturnFalse; if(!MatMul(cIdifK_H.getOutput(), cPPred.getOutput(), cIdifK_H.getPrevOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetWindow(), 1, true)) ReturnFalse; if(!MatMul(cIdifK_H.getPrevOutput(), cIdifK_HT.getOutput(), cP.getOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cK.getOutput(), cR_RT.getOutput(), cK.getPrevOutput(), cKT.GetCount(), cRT.GetCount(), cRT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cK.getPrevOutput(), cKT.getOutput(), cKT.getPrevOutput(), cKT.GetCount(), cKT.GetWindow(), cKT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cP.getOutput(), cKT.getPrevOutput(), cP.getOutput(), cPT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!cPT.FeedForward(cP.AsObject())) ReturnFalse; if(!SumAndNormalize(cP.getOutput(), cPT.getOutput(), cP.getOutput(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!SumAndNormalize(cX.getOutput(), cAuxiliaryNet.getOutput(), cX.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Sample Output if(!cP.getOutput().GetData(mPGrad)) ReturnFalse; if(mPGrad.HasNan() > 0) { mPGrad.Identity(); if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } mP = mPGrad.Inv(); if(mP.Rows() == 0) { double max = MathAbs(mPGrad).Max(); if(max > 1) mPGrad = mPGrad / max; mPGrad = mPGrad + matrix::Identity(mPGrad.Rows(), mPGrad.Cols()); mP = mPGrad.Inv(); if(mP.Rows() == 0) { mPGrad = mPGrad + mPGrad.Transpose(); vector eigvals; matrix eigvecs; if(!mPGrad.Eig(eigvecs, eigvals)) ReturnFalse; if(eigvals.Size() > 0) { if(!eigvals.Clip(1e-6, DBL_MAX)) ReturnFalse; mPGrad = matrix::Zeros(eigvals.Size(), eigvals.Size()); mPGrad.Diag(eigvals); mPGrad = eigvecs.MatMul(mPGrad.MatMul(eigvecs.Transpose())); mPGrad = mPGrad + mPGrad.Transpose(); mP = mPGrad.Inv(); } if(mP.Rows() == 0) { mPGrad.Identity(); if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; mP = mPGrad.Inv(); } else if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } else if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } mNoise.Random(-1, 1); matrix temp = mNoise.MatMul(mP); if(!PrevOutput.Fill(temp)) ReturnFalse; if(!SumVecMatrix(cX.getOutput(), PrevOutput, Output, (int)mNoise.Cols(), 1, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronK2VAEEncoder layer. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- From Output if(!SumVecMatrixGrad(cX.getGradient(), PrevOutput, Gradient, (int)mNoise.Cols(), 1, 0, 0, 0, 1)) ReturnFalse; if(!PrevOutput.GetData(mGrad)) ReturnFalse; mPGrad = mNoise.Transpose().MatMul(mGrad); mKL = mP.Transpose(); mKL = (mKL * (-1)).MatMul(mPGrad.MatMul(mKL)); //--- Joseph stabilized form for P if(!cPT.getGradient().Fill(mKL)) ReturnFalse; if(!cP.CalcHiddenGradients(cPT.AsObject())) ReturnFalse; if(!SumAndNormalize(cP.getGradient(), cPT.getGradient(), cP.getGradient(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- if(!MatMulGrad(cK.getPrevOutput(), cKT.getPrevOutput(), cKT.getOutput(), cKT.getGradient(), cP.getGradient(), cKT.GetCount(), cKT.GetWindow(), cKT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cK.getOutput(), cK.getPrevOutput(), cR_RT.getOutput(), cR_RT.getGradient(), cKT.getPrevOutput(), cKT.GetCount(), cRT.GetCount(), cRT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cIdifK_H.getPrevOutput(), cIdifK_HT.getPrevOutput(), cIdifK_HT.getOutput(), cIdifK_HT.getGradient(), cP.getGradient(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cIdifK_H.getOutput(), cIdifK_H.getPrevOutput(), cPPred.getOutput(), cPPred.getGradient(), cIdifK_HT.getPrevOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetWindow(), 1, true)) ReturnFalse; //--- if(!cK.CalcHiddenGradients(cKT.AsObject())) ReturnFalse; if(!SumAndNormalize(cK.getGradient(), cK.getPrevOutput(), cK.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Measurement update if(!cIdifK_H.CalcHiddenGradients(cIdifK_HT.AsObject())) ReturnFalse; if(!SumAndNormalize(cIdifK_H.getGradient(), cIdifK_H.getPrevOutput(), cIdifK_H.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!IdentDifferentGrad(cK_H.getGradient(), cIdifK_H.getGradient(), cHT.GetWindow(), 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cK.getOutput(), cK.getPrevOutput(), cH.getOutput(), cH.getGradient(), cK_H.getGradient(), cKT.GetCount(), cKT.GetWindow(), cHT.GetWindow(), 1, true)) ReturnFalse; //--- if(!MatMulGrad(cDeltY.getOutput(), cDeltY.getGradient(), cK.getOutput(), cK.getPrevOutput(), cX.getGradient(), 1, cDeltY.Neurons(), cX.Neurons(), 1, true)) ReturnFalse; if(!SumAndNormalize(cK.getGradient(), cK.getPrevOutput(), cK.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!DifferentGrad(cKoopmanPred.getGradient(), cYPred.getGradient(), cDeltY.getGradient(), 1, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cXPred.getOutput(), cXPred.getGradient(), cHT.getOutput(), cHT.getGradient(), cYPred.getGradient(), 1, cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cXPred.getGradient(), cX.getGradient(), cXPred.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Update step if(!MatMulGrad(cP_HT.getOutput(), cP_HT.getGradient(), cSInv.getOutput(), cSInv.getGradient(), cK.getGradient(), cHT.GetCount(), (int)mS.Rows(), (int)mS.Cols(), 1, true)) ReturnFalse; if(cSInv.getGradient().GetData(mSGrad) <= 0) ReturnFalse; mKL = mS.Transpose(); mKL = (mKL * (-1)).MatMul(mSGrad.MatMul(mKL)); if(cH_P_HT.getGradient().Fill(mKL) <= 0) ReturnFalse; if(!MatMulGrad(cH.getOutput(), cH.getPrevOutput(), cP_HT.getOutput(), cP_HT.getPrevOutput(), cH_P_HT.getGradient(), cHT.GetCount(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cH_P_HT.getGradient(), cR_RT.getGradient(), cR_RT.getGradient(), int(mS.Cols()), false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cH.getGradient(), cH.getPrevOutput(), cH.getPrevOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cP_HT.getGradient(), cP_HT.getPrevOutput(), cP_HT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cPPred.getOutput(), cPPred.getPrevOutput(), cHT.getOutput(), cHT.getPrevOutput(), cP_HT.getGradient(), cPPred.Neurons() / cHT.GetWindow(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cPPred.getGradient(), cPPred.getPrevOutput(), cQ_QT.getGradient(), int(cQT.GetWindow()), false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cHT.getGradient(), cHT.getPrevOutput(), cHT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Prediction step if(!MatMulGrad(cF_P.getOutput(), cF_P.getGradient(), cFT.getOutput(), cFT.getGradient(), cQ_QT.getGradient(), cFT.GetCount(), cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cF.getOutput(), cF.getPrevOutput(), cP.getPrevOutput(), cP.getGradient(), cF_P.getGradient(), cFT.GetCount(), cFT.GetWindow(), cP.Neurons() / cFT.GetWindow(), 1, true)) ReturnFalse; if(!MatMulGrad(cAuxiliaryNet.getOutput(), cAuxiliaryNet.getGradient(), cB.getOutput(), cB.getGradient(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cFT.getOutput(), cFT.getPrevOutput(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormalize(cFT.getGradient(), cFT.getPrevOutput(), cFT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!MatMulGrad(cR.getOutput(), cR.getPrevOutput(), cRT.getOutput(), cRT.getGradient(), cR_RT.getGradient(), cRT.GetCount(), cRT.GetWindow(), cRT.GetCount(), 1, false)) ReturnFalse; if(!MatMulGrad(cQ.getOutput(), cQ.getPrevOutput(), cQT.getOutput(), cQT.getGradient(), cQ_QT.getGradient(), cQT.GetCount(), cQT.GetWindow(), cQT.GetCount(), 1, false)) ReturnFalse; if(!cR.CalcHiddenGradients((CObject*)cRT.AsObject())) ReturnFalse; if(!SumAndNormalize(cR.getGradient(), cR.getPrevOutput(), cR.getGradient(), cRT.GetWindow(), false, 0, 0, 0, 1e-4f)) ReturnFalse; cR.getGradient().BufferRead(); if(!cQ.CalcHiddenGradients(cQT.AsObject())) ReturnFalse; if(!SumAndNormalize(cQ.getGradient(), cQ.getPrevOutput(), cQ.getGradient(), cQT.GetWindow(), false, 0, 0, 0, 1e-4f)) ReturnFalse; cQ.getGradient().BufferRead(); if(!cH.CalcHiddenGradients(cHT.AsObject())) ReturnFalse; if(!SumAndNormalize(cH.getGradient(), cH.getPrevOutput(), cH.getGradient(), cHT.GetWindow(), false, 0, 0, 0, 0.01f)) ReturnFalse; if(!cF.CalcHiddenGradients(cFT.AsObject())) ReturnFalse; if(!SumAndNormalize(cF.getGradient(), cF.getPrevOutput(), cF.getGradient(), cFT.GetWindow(), false, 0, 0, 0, 0.01f)) ReturnFalse; //--- Rest Attention if(!cKoopmanRest.CalcHiddenGradients(cAuxiliaryNet.AsObject())) ReturnFalse; //--- Pred / Rest if(!NeuronOCL.getPrevOutput().Fill(0)) ReturnFalse; if(!DifferentGrad(NeuronOCL.getPrevOutput(), cKoopmanRest.getPrevOutput(), cKoopmanRest.getGradient(), cKoopman.GetWindow())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; } if(!Concat(cKoopmanPred.getGradient(), cKoopmanRest.getPrevOutput(), cKoopman.getGradient(), cKoopman.GetWindow(), cKoopman.Neurons() - cKoopman.GetWindow(), 1)) ReturnFalse; //--- Koopman if(!NeuronOCL.CalcHiddenGradients(cKoopman.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronK2VAEEncoder trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Koopman if(!cKoopman.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cAuxiliaryNet.UpdateInputWeights(cKoopmanRest.AsObject())) ReturnFalse; //--- Kalman Filter if(!cB.UpdateInputWeights()) ReturnFalse; if(!cF.UpdateInputWeights()) ReturnFalse; if(!cH.UpdateInputWeights()) ReturnFalse; if(!cQ.UpdateInputWeights()) ReturnFalse; if(!cR.UpdateInputWeights()) ReturnFalse; //--- if(!SumAndNormalize(cQ.getWeightsParams(), cQ.getWeightsParams(), cQ.getWeightsParams(), 2 * cQT.GetWindow(), true)) ReturnFalse; if(!SumAndNormalize(cR.getWeightsParams(), cR.getWeightsParams(), cR.getWeightsParams(), 2 * cRT.GetWindow(), true)) ReturnFalse; //--- if(!cQ.FeedForward()) ReturnFalse; mKL.Random(-1, 1); mKL = mKL.TriL(); mKL = mKL.MatMul(mKL.Transpose()); float error = 0; if(!cQ.getGradient().Fill(mKL) || !cQ.calcOutputGradients(cQ.getGradient(), error) || !cQ.UpdateInputWeights()) ReturnFalse; //--- if(!cR.FeedForward()) ReturnFalse; mKL.Random(-1, 1); mKL = mKL.TriL(); mKL = mKL.MatMul(mKL.Transpose()); error = 0; if(!cR.getGradient().Fill(mKL) || !cR.calcOutputGradients(cR.getGradient(), error) || !cR.UpdateInputWeights()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronK2VAEEncoder weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronK2VAEEncoder *Source = source; //--- Koopman if(!cKoopman.WeightsUpdate(Source.cKoopman.AsObject(), tau)) ReturnFalse; if(!cAuxiliaryNet.WeightsUpdate(Source.cAuxiliaryNet.AsObject(), tau)) ReturnFalse; //--- Kalman Filter if(!cB.WeightsUpdate(Source.cB.AsObject(), tau)) ReturnFalse; if(!cF.WeightsUpdate(Source.cF.AsObject(), tau)) ReturnFalse; if(!cH.WeightsUpdate(Source.cH.AsObject(), tau)) ReturnFalse; if(!cQ.WeightsUpdate(Source.cQ.AsObject(), tau)) ReturnFalse; if(!cR.WeightsUpdate(Source.cR.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronK2VAEEncoder state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Koopman if(!cKoopman.Save(file_handle)) ReturnFalse; if(!cAuxiliaryNet.Save(file_handle)) ReturnFalse; //--- Kalman Filter if(!cB.Save(file_handle)) ReturnFalse; if(!cF.Save(file_handle)) ReturnFalse; if(!cH.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cR.Save(file_handle)) ReturnFalse; if(!cP.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronK2VAEEncoder state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Koopman if(!LoadInsideLayer(file_handle, cKoopman.AsObject())) ReturnFalse; //--- uint window = cKoopman.GetWindow(); uint units_cross = cKoopman.GetUnits(); int index = 1; if(!cKoopmanPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanRest.Init(0, index, OpenCL, window * (units_cross - 1), optimization, iBatch)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAuxiliaryNet.AsObject())) ReturnFalse; //--- Kalman Filter if(!LoadInsideLayer(file_handle, cB.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cF.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cH.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cP.AsObject())) ReturnFalse; //--- index += 7; if(!cFT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cHT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cQT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cRT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cPT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQ_QT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cR_RT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cXPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cF_P.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cPPred.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cP_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cH_P_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- mS = mSGrad = matrix::Zeros(window, window); //--- index++; if(!cSInv.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cK.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cKT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cYPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cDeltY.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cX.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_HT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- mKL = mP = mPGrad = mS; mNoise = mGrad = matrix::Zeros(Neurons() / window, window); //--- return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronK2VAEEncoder layer. | //+------------------------------------------------------------------+ void CNeuronK2VAEEncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cKoopman.TrainMode(bTrain); cAuxiliaryNet.TrainMode(bTrain); cB.TrainMode(bTrain); cF.TrainMode(bTrain); cH.TrainMode(bTrain); cQ.TrainMode(bTrain); cR.TrainMode(bTrain); cP.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronK2VAEEncoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronK2VAEEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- Koopman cKoopman.SetOpenCL(OpenCL); cKoopmanPred.SetOpenCL(OpenCL); cKoopmanRest.SetOpenCL(OpenCL); //--- cAuxiliaryNet.SetOpenCL(OpenCL); //--- Kalman Filter cB.SetOpenCL(OpenCL); cF.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cR.SetOpenCL(OpenCL); cP.SetOpenCL(OpenCL); cFT.SetOpenCL(OpenCL); cHT.SetOpenCL(OpenCL); cQT.SetOpenCL(OpenCL); cRT.SetOpenCL(OpenCL); cPT.SetOpenCL(OpenCL); cQ_QT.SetOpenCL(OpenCL); cR_RT.SetOpenCL(OpenCL); cXPred.SetOpenCL(OpenCL); cF_P.SetOpenCL(OpenCL); cPPred.SetOpenCL(OpenCL); cP_HT.SetOpenCL(OpenCL); cH_P_HT.SetOpenCL(OpenCL); cSInv.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cKT.SetOpenCL(OpenCL); cYPred.SetOpenCL(OpenCL); cDeltY.SetOpenCL(OpenCL); cX.SetOpenCL(OpenCL); cK_H.SetOpenCL(OpenCL); cIdifK_H.SetOpenCL(OpenCL); cIdifK_HT.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronInterpolationAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * dimension, optimization_type, batch)) ReturnFalse; //--- iCount = units_count; iDimension = dimension; //--- int index = 0; if(!cW.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cA.Init(0, index, OpenCL, 2 * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cGL.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- index++; if(!cH.Init(0, index, OpenCL, iCount * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAdj.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the interpolation attention step of the | //| CNeuronInterpolationAttention layer. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::InterpolationAttention(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iCount * iDimension)) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iCount}; uint local_work_size[2] = {1, iCount}; const int kernel = def_k_InterpolationAttention; setBuffer(kernel, def_k_ia_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ia_A, cA.getOutputIndex()) setBuffer(kernel, def_k_ia_Adj, cAdj.getOutputIndex()) setBuffer(kernel, def_k_ia_Atten, cAttention.getOutputIndex()) setBuffer(kernel, def_k_ia_GL, cGL.getOutputIndex()) setBuffer(kernel, def_k_ia_H, cH.getOutputIndex()) setBuffer(kernel, def_k_ia_W, cW.getOutputIndex()) setBuffer(kernel, def_k_ia_matrix_out, getOutputIndex()) setArgument(kernel, def_k_ia_dimension, (int)iDimension) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the interpolation attention grad step of the | //| CNeuronInterpolationAttention layer. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::InterpolationAttentionGrad(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iCount * iDimension)) ReturnFalse; //--- CBufferFloat* temp = cA.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; temp = cGL.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iCount}; uint local_work_size[2] = {1, iCount}; const int kernel = def_k_InterpolationAttentionGrad; setBuffer(kernel, def_k_iag_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_iag_matrix_in_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_iag_A, cA.getOutputIndex()) setBuffer(kernel, def_k_iag_A_gr, cA.getGradientIndex()) setBuffer(kernel, def_k_iag_Adj, cAdj.getOutputIndex()) setBuffer(kernel, def_k_iag_Atten, cAttention.getOutputIndex()) setBuffer(kernel, def_k_iag_GL, cGL.getOutputIndex()) setBuffer(kernel, def_k_iag_GL_gr, cGL.getGradientIndex()) setBuffer(kernel, def_k_iag_H, cH.getOutputIndex()) setBuffer(kernel, def_k_iag_H_gr, cH.getGradientIndex()) setBuffer(kernel, def_k_iag_W, cW.getOutputIndex()) setBuffer(kernel, def_k_iag_W_gr, cW.getGradientIndex()) setBuffer(kernel, def_k_iag_matrix_out_gr, getGradientIndex()) setArgument(kernel, def_k_iag_dimension, (int)iDimension) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronInterpolationAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cW.FeedForward()) ReturnFalse; if(!cA.FeedForward()) ReturnFalse; if(!cGL.FeedForward()) ReturnFalse; } if(!InterpolationAttention(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronInterpolationAttention layer. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!InterpolationAttentionGrad(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronInterpolationAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cW.UpdateInputWeights()) ReturnFalse; if(!cA.UpdateInputWeights()) ReturnFalse; if(!cGL.UpdateInputWeights() || !Normalize(cGL.getWeightsParams(), 2 * iDimension)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronInterpolationAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronInterpolationAttention *Source = source; if(!cW.WeightsUpdate(Source.cW.AsObject(), tau)) ReturnFalse; if(!cA.WeightsUpdate(Source.cA.AsObject(), tau)) ReturnFalse; if(!cGL.WeightsUpdate(Source.cGL.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronInterpolationAttention state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; //--- if(!cW.Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cGL.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronInterpolationAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cW.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGL.AsObject())) ReturnFalse; //--- int index = 3; if(!cH.Init(0, index, OpenCL, iCount * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAdj.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronInterpolationAttention GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronInterpolationAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cW.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cGL.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cAdj.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronInterpolationAttention | //| layer. | //+------------------------------------------------------------------+ void CNeuronInterpolationAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cW.TrainMode(bTrain); cA.TrainMode(bTrain); cGL.TrainMode(bTrain); cH.TrainMode(bTrain); cAdj.TrainMode(bTrain); cAttention.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPeriodNorm layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * period * variables, optimization_type, batch)) ReturnFalse; //--- iPeriod = period; iVariables = variables; iCount = units_count; //--- if(!cMeanSTDevs.Init(0, 0, OpenCL, iCount * iVariables * 2, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPeriodNorm feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, iPeriod, iVariables }; uint local_work_size[3] = { 1, iPeriod, 1 }; //--- uint kernel = def_k_PeriodNorm; setBuffer(kernel, def_k_pn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_pn_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_pn_outputs, getOutputIndex()) setArgument(kernel, def_k_pn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / (iVariables * iPeriod), iCount)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPeriodNorm layer. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, iPeriod, iVariables }; uint local_work_size[3] = { 1, iPeriod, 1 }; //--- uint kernel = def_k_PeriodNormGrad; setBuffer(kernel, def_k_png_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_png_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_png_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_png_mean_stdevs_gr, cMeanSTDevs.getGradientIndex()) setBuffer(kernel, def_k_png_outputs, getOutputIndex()) setBuffer(kernel, def_k_png_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_png_total_inputs, (int)MathMin(NeuronOCL.Neurons() / (iVariables * iPeriod), iCount)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPeriodNorm state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cMeanSTDevs.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPeriod)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPeriodNorm state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMeanSTDevs.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iPeriod = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPeriodNorm GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPeriodNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cMeanSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Checks checked-transfer support of PeriodNorm nested statistics. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::SupportsOpenCLChecked(void) const { return(Type() == defNeuronPeriodNorm && cMeanSTDevs.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers owned buffers before inherited context publication. | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::SetOpenCLChecked(COpenCLMy *obj) { //--- Reject invalid targets and incomplete trees before any mutation. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or period tree"); //--- Transfer the complete statistics child before outer base publication. if(!cMeanSTDevs.SetOpenCLChecked(obj)) ReturnFalseEx("mean standard deviations context"); //--- Publish inherited outer buffers only after nested statistics succeeded. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("inherited base context"); //--- Finalize the checked PeriodNorm transfer. return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMoEConv layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint experts, float dropout, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch)) ReturnFalse; //--- int index = 0; //--- CNeuronConvOCL *conv = NULL; CNeuronDropoutOCL *dout = NULL; CNeuronSparseSoftMax *softmax = NULL; //--- Probability acProbability.Clear(); acProbability.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(experts, index, OpenCL, window, window, 2 * experts, units, 1, optimization, iBatch) || !acProbability.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(experts, index, OpenCL, 2 * experts, 2 * experts, experts, units, 1, optimization, iBatch) || !acProbability.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; dout = new CNeuronDropoutOCL(); if(!dout || !dout.Init(0, index, OpenCL, conv.Neurons(), dropout, optimization, iBatch) || !acProbability.Add(dout)) { DeleteObj(dout); ReturnFalse; } index++; softmax = new CNeuronSparseSoftMax; if(!softmax || !softmax.Init(0, index, OpenCL, units, experts, topK, optimization, iBatch) || !acProbability.Add(softmax)) { DeleteObj(softmax); ReturnFalse; } softmax.SetHeads(units); //--- index++; if(!cExperts.Init(0, index, OpenCL, window * window_out * experts, optimization, iBatch)) ReturnFalse; index++; if(!cWeightsConv.Init(0, index, OpenCL, window * window_out * units, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMoEConv feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; CNeuronConvOCL *conv = NULL; CNeuronSparseSoftMax *softmax = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } conv = acProbability[0]; softmax = prev; //--- Experts if(bTrain) if(!cExperts.FeedForward()) ReturnFalse; //--- uint units = conv.GetUnits(); uint window_in = conv.GetWindow(); uint window_out = Neurons() / units; uint topK = softmax.DimensionOut(); uint experts = cExperts.Neurons() / (window_in * window_out); //--- Generate Weights if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cExperts.getOutput(), cWeightsConv.getOutput(), units, topK, window_in * window_out, experts)) ReturnFalse; //--- Result if(!MatMul(NeuronOCL.getOutput(), cWeightsConv.getOutput(), Output, 1, window_in, window_out, units, true)) ReturnFalse; if(activation != None) if(!Activation(Output, Output, activation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMoEConv | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!acProbability[0] || !acProbability[-1]) ReturnFalse; //--- CNeuronConvOCL* conv = acProbability[0]; CNeuronSparseSoftMax* softmax = acProbability[-1]; uint units = conv.GetUnits(); uint window_in = conv.GetWindow(); uint window_out = Neurons() / units; uint topK = softmax.DimensionOut(); uint experts = cExperts.Neurons() / (window_in * window_out); //--- if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), cWeightsConv.getOutput(), cWeightsConv.getGradient(), Gradient, 1, window_in, window_out, units, true)) ReturnFalse; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cExperts.getOutput(), cExperts.getGradient(), cWeightsConv.getGradient(), units, topK, window_in * window_out, experts)) ReturnFalse; //--- CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = acProbability.Total() - 1; i >= 0; i--) { next = acProbability[i]; current = (i == 0 ? NeuronOCL : acProbability[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), window_in, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMoEConv trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Experts if(!cExperts.UpdateInputWeights()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMoEConv weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMoEConv *Source = source; if(!acProbability.WeightsUpdate(Source.acProbability.AsObject(), tau)) ReturnFalse; if(!cExperts.WeightsUpdate(Source.cExperts.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMoEConv state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!acProbability.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMoEConv state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!acProbability.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cExperts.AsObject())) ReturnFalse; //--- CNeuronConvOCL* conv = acProbability[0]; uint window_in = conv.GetWindow(); int index = acProbability.Total() + 1; if(!cWeightsConv.Init(0, index, OpenCL, window_in * Neurons(), optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMoEConv GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMoEConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); acProbability.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); cWeightsConv.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMoEConv layer. | //+------------------------------------------------------------------+ void CNeuronMoEConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); acProbability.TrainMode(bTrain); cExperts.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiScaleDifference layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint bottleneck, uint &shifts[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck, bottleneck, dimension, units, 1, optimization_type, batch)) ReturnFalse; //--- int scales = int(shifts.Size()); if(scales == 0 || ArrayResize(iaShifts, scales) < scales || ArrayCopy(iaShifts, shifts, 0, 0, scales) < scales) ReturnFalse; ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, scales, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- cBottleneck.Clear(); cExcitation.Clear(); cProjection.Clear(); cBottleneck.SetOpenCL(OpenCL); cExcitation.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); //--- CNeuronMultiWindowsConvWPadOCL* padconv = NULL; CNeuronSpikeDepthWiseConv* dwconv = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronMultiScaleExcitation* excitation = NULL; CNeuronTransposeOCL* transpose = NULL; uint index = 0; //--- Bottleneck uint units_out = (units + 1) / 2; dwconv = new CNeuronSpikeDepthWiseConv(); if(!dwconv || !dwconv.Init(0, index, OpenCL, dimension, bottleneck, 5, 2, units_out, 1, optimization, iBatch) || !cBottleneck.Add(dwconv)) { DeleteObj(dwconv); ReturnFalse; } index++; uint windows[] = { 3 * bottleneck}; padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, bottleneck, bottleneck, units_out, 1, optimization, iBatch) || !cBottleneck.Add(padconv)) { DeleteObj(padconv) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- Excitation neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, conv.Neurons()*scales, optimization, iBatch) || !cExcitation.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; excitation = new CNeuronMultiScaleExcitation(); if(!excitation || !excitation.Init(0, index, OpenCL, bottleneck, units_out, scales, (bottleneck + 1) / 2, optimization, iBatch) || !cExcitation.Add(excitation)) { DeleteObj(excitation) ReturnFalse; } index++; transpose = new CNeuronTransposeOCL(); if(!transpose || !transpose.Init(0, index, OpenCL, bottleneck * units_out, scales, optimization, iBatch) || !cExcitation.Add(transpose)) { DeleteObj(transpose) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, scales, scales, 1, transpose.GetCount(), 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- Projection neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) || !cProjection.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); if(!neuron.SetGradient(conv.getGradient(), true)) ReturnFalse; index++; dwconv = new CNeuronSpikeDepthWiseConv(); if(!dwconv || !dwconv.Init(0, index, OpenCL, bottleneck, 2 * bottleneck, 5, 1, units_out, 1, optimization, iBatch) || !cProjection.Add(dwconv)) { DeleteObj(dwconv); ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiScaleDifference feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Bottleneck for(int i = 0; i < cBottleneck.Total(); i++) { curr = cBottleneck[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Excitation curr = cExcitation[0]; if(!curr) ReturnFalse; uint dimension = cConv.GetWindow(); uint units = prev.Neurons() / dimension; if(!DilatedDifference(prev, curr, ibShifts, dimension, units)) ReturnFalse; prev = curr; for(int i = 1; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Projection curr = cProjection[0]; if(!curr || !SumAndNormalize(cBottleneck[-1].getOutput(), prev.getOutput(), curr.getOutput(), dimension, true, 0, 0, 0, 1)) ReturnFalse; prev = curr; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), Output, Output, cConv.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiScaleDifference layer. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Projection CNeuronBaseOCL* next = cProjection[-1]; CNeuronBaseOCL* curr = NULL; if(!CNeuronSpikeConvBlock::calcInputGradients(next)) ReturnFalse; for(int i = cProjection.Total() - 2; i >= 0; i--) { curr = cProjection[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Excitation next = cExcitation[-1]; if(!next) ReturnFalse; Deactivation(next) for(int i = cExcitation.Total() - 2; i >= 0; i--) { curr = cExcitation[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Bottleneck curr = cBottleneck[-1]; if(!curr) ReturnFalse; uint dimension = cConv.GetWindow(); uint units = curr.Neurons() / dimension; if(!DilatedDifferenceGrad(curr, next, ibShifts, dimension, units)) ReturnFalse; Deactivation(curr) next = curr; for(int i = cBottleneck.Total() - 2; i >= 0; i--) { curr = cBottleneck[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiScaleDifference trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Bottleneck for(int i = 0; i < cBottleneck.Total(); i++) { curr = cBottleneck[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Excitation prev = cExcitation[0]; for(int i = 1; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Projection prev = cProjection[0]; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiScaleDifference state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- uint size = iaShifts.Size(); if(FileWriteInteger(file_handle, size, INT_VALUE) < INT_VALUE) ReturnFalse; for(uint i = 0; i < size; i++) { if(FileWriteInteger(file_handle, iaShifts[i], INT_VALUE) < INT_VALUE) ReturnFalse; } //--- if(!cBottleneck.Save(file_handle)) ReturnFalse; if(!cExcitation.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiScaleDifference state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; uint size = FileReadInteger(file_handle); if(ArrayResize(iaShifts, size) < (int)size) ReturnFalse; for(uint i = 0; i < size; i++) { if(FileIsEnding(file_handle)) ReturnFalse; iaShifts[i] = FileReadInteger(file_handle); } OpenCL.BufferFree(ibShifts); ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, size, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- if(!cBottleneck.Load(file_handle)) ReturnFalse; if(!cExcitation.Load(file_handle)) ReturnFalse; if(!cProjection.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMultiScaleDifference weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMultiScaleDifference* Source = source; if(!cBottleneck.WeightsUpdate(Source.cBottleneck.AsObject(), tau)) ReturnFalse; if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronMultiScaleDifference execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cBottleneck.ClearStates()) ReturnFalse; if(!cExcitation.ClearStates()) ReturnFalse; if(!cProjection.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultiScaleDifference GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultiScaleDifference::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); CNeuronSpikeConvBlock::SetOpenCL(obj); ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, iaShifts.Size(), CL_MEM_READ_ONLY); cBottleneck.SetOpenCL(OpenCL); cExcitation.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMultiScaleDifference layer. | //+------------------------------------------------------------------+ void CNeuronMultiScaleDifference::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cBottleneck.TrainMode(bTrain); cExcitation.TrainMode(bTrain); cProjection.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronConfGateTailAwareMoE layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint forecast, uint experts, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * forecast, optimization_type, batch)) ReturnFalse; activation = None; //--- cConfidenceGating.Clear(); cScore.Clear(); cCGScore.Clear(); cConfidenceGating.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cCGScore.SetOpenCL(OpenCL); //--- uint index = 0; if(!cTailAwareMoE.Init(0, index, OpenCL, dimension * forecast * 4 * experts, optimization, iBatch)) ReturnFalse; cTailAwareMoE.SetActivationFunction(None); //--- uint windows[] = {7, 5, 3}; uint steps[] = {2, 2, 2}; CNeuronSpikeSuperKernelBlock* conv = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSoftMaxOCL* softmax = NULL; CNeuronSpikeActivation* spike = NULL; //--- Score index++; conv = new CNeuronSpikeSuperKernelBlock(); if(!conv || !conv.Init(experts, index, OpenCL, dimension, (dimension + 1) / 2, windows, steps, units, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cScore.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(SoftPlus); index++; softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cScore.Add(softmax)) DeleteObjAndFalse(softmax); softmax.SetHeads(1); index++; if(!cScoreStack.Init(0, index, OpenCL, 50, neuron.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- Confidence-Gating index++; conv = new CNeuronSpikeSuperKernelBlock(); if(!conv || !conv.Init(experts, index, OpenCL, dimension, (dimension + 1) / 2, windows, steps, units, 1, optimization, iBatch) || !cConfidenceGating.Add(conv)) DeleteObjAndFalse(conv); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cConfidenceGating.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, neuron.Neurons(), iBatch, optimization) || !cConfidenceGating.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(SoftPlus); index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cConfidenceGating.Add(spike)) DeleteObjAndFalse(spike); index++; if(!cCGStack.Init(0, index, OpenCL, 50, neuron.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- Confidence-Gating Score index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cCGScore.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); //--- index++; if(!cDensity.Init(0, index, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cDensity.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the mix experts predict step of the | //| CNeuronConfGateTailAwareMoE layer. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::MixExpertsPredict(void) { if(!OpenCL || !Output || !cDensity.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[] = { Neurons() }; uint kernel = def_k_MixExpertsPredict; setBuffer(kernel, def_k_moepred_experts, cDensity.getOutputIndex()) setBuffer(kernel, def_k_moepred_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the mix experts predict grad step of the | //| CNeuronConfGateTailAwareMoE layer. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::MixExpertsPredictGrad(void) { if(!OpenCL || !Gradient || !cDensity.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[] = { Neurons() }; uint kernel = def_k_MixExpertsPredictGrad; setBuffer(kernel, def_k_moepredgr_experts, cDensity.getOutputIndex()) setBuffer(kernel, def_k_moepredgr_experts_gr, cDensity.getGradientIndex()) setBuffer(kernel, def_k_moepredgr_outputs_gr, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cDensity.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronConfGateTailAwareMoE feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Score for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating prev = NeuronOCL; for(int i = 0; i < cConfidenceGating.Total(); i++) { curr = cConfidenceGating[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating Score prev = cCGScore[0]; if(!prev || !ElementMult(cConfidenceGating[-1].getOutput(), cScore[-1].getOutput(), prev.getOutput())) ReturnFalse; for(int i = 1; i < cCGScore.Total(); i++) { curr = cCGScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Density if(bTrain) if(!cTailAwareMoE.FeedForward()) ReturnFalse; if(!MatMul(prev.getOutput(), cTailAwareMoE.getOutput(), cDensity.getOutput(), 1, prev.Neurons(), cDensity.Neurons(), 1, false)) ReturnFalse; if(!MixExpertsPredict()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronConfGateTailAwareMoE layer. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MixExpertsPredictGrad()) ReturnFalse; //--- Confidence-Gating Score CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* curr = cCGScore[-1]; if(!curr || !MatMulGrad(curr.getOutput(), curr.getGradient(), cTailAwareMoE.getOutput(), cTailAwareMoE.getGradient(), cDensity.getGradient(), 1, curr.Neurons(), cDensity.Neurons(), 1, false)) ReturnFalse; next = curr; for(int i = cCGScore.Total() - 2; i >= 0; i--) { curr = cCGScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- next = cConfidenceGating[-1]; curr = cScore[-1]; if(!next || !curr || !ElementMultGrad(next.getOutput(), next.getGradient(), curr.getOutput(), curr.getGradient(), cCGScore[0].getGradient(), next.Activation(), curr.Activation())) ReturnFalse; //--- Confidence-Gating if(!cCGStack.FeedForward(next) || !DiversityLoss(cCGStack.AsObject(), cCGStack.GetStackSize(), next.Neurons(), false)) ReturnFalse; if(next.Activation() != None) if(!DeActivation(next.getOutput(), cCGStack.getGradient(), cCGStack.getGradient(), next.Activation())) ReturnFalse; if(!SumAndNormalize(next.getGradient(), cCGStack.getGradient(), next.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; for(int i = cConfidenceGating.Total() - 2; i >= 0; i--) { curr = cConfidenceGating[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Score next = cScore[-1]; if(!cScoreStack.FeedForward(next) || !DiversityLoss(cScoreStack.AsObject(), cScoreStack.GetStackSize(), next.Neurons(), false)) ReturnFalse; if(next.Activation() != None) if(!DeActivation(next.getOutput(), cScoreStack.getGradient(), cScoreStack.getGradient(), next.Activation())) ReturnFalse; if(!SumAndNormalize(next.getGradient(), cScoreStack.getGradient(), next.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; for(int i = cScore.Total() - 2; i >= 0; i--) { curr = cScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cConfidenceGating[0]) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronConfGateTailAwareMoE trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Score for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating prev = NeuronOCL; for(int i = 0; i < cConfidenceGating.Total(); i++) { curr = cConfidenceGating[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating Score prev = cCGScore[0]; for(int i = 1; i < cCGScore.Total(); i++) { curr = cCGScore[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Density if(!cTailAwareMoE.UpdateInputWeights()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronConfGateTailAwareMoE weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronConfGateTailAwareMoE* Source = source; dWeightsUpdate(cScore, Source, tau); dWeightsUpdate(cConfidenceGating, Source, tau); dWeightsUpdate(cCGScore, Source, tau); dWeightsUpdate(cTailAwareMoE, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronConfGateTailAwareMoE state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cConfidenceGating.Save(file_handle)) ReturnFalse; if(!cScore.Save(file_handle)) ReturnFalse; if(!cCGScore.Save(file_handle)) ReturnFalse; if(!cTailAwareMoE.Save(file_handle)) ReturnFalse; if(!cScoreStack.Save(file_handle)) ReturnFalse; if(!cCGStack.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronConfGateTailAwareMoE state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cConfidenceGating.Load(file_handle)) ReturnFalse; if(!cScore.Load(file_handle)) ReturnFalse; if(!cCGScore.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTailAwareMoE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScoreStack.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCGStack.AsObject())) ReturnFalse; //--- uint index = 3 + cScore.Total() + cConfidenceGating.Total() + cCGScore.Total(); if(!cDensity.Init(0, index, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cDensity.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronConfGateTailAwareMoE execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cConfidenceGating.ClearStates()) ReturnFalse; if(!cScore.ClearStates()) ReturnFalse; if(!cCGScore.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronConfGateTailAwareMoE GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronConfGateTailAwareMoE::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cConfidenceGating.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cCGScore.SetOpenCL(OpenCL); cTailAwareMoE.SetOpenCL(OpenCL); cScoreStack.SetOpenCL(OpenCL); cCGStack.SetOpenCL(OpenCL); cDensity.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronConfGateTailAwareMoE layer. | //+------------------------------------------------------------------+ void CNeuronConfGateTailAwareMoE::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cConfidenceGating.TrainMode(bTrain); cScore.TrainMode(bTrain); cCGScore.TrainMode(bTrain); cTailAwareMoE.TrainMode(bTrain); cScoreStack.TrainMode(bTrain); cCGStack.TrainMode(bTrain); cDensity.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSparseMoEConv layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint filters, uint units, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, filters * units, optimization_type, batch)) ReturnFalse; if(!cParams.Init(0, 0, OpenCL, window, units, window * filters, candidates, topK, optimization, iBatch)) ReturnFalse; cParams.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSparseMoEConv feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cParams.FeedForward(NeuronOCL)) ReturnFalse; //--- uint units = cParams.GetFields(); uint filters = Neurons() / units; uint window = cParams.Neurons() / (filters * units); if(!MatMul(NeuronOCL.getOutput(), cParams.getOutput(), Output, 1, window, filters, units, true)) ReturnFalse; dActivation(this); //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSparseMoEConv layer. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cParams.GetFields(); uint filters = Neurons() / units; uint window = cParams.Neurons() / (filters * units); if(NeuronOCL.Neurons() < int(window * units)) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), cParams.getOutput(), cParams.getGradient(), Gradient, 1, window, filters, units, true)) ReturnFalse; Deactivation(cParams); if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cParams.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSparseMoEConv trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSparseMoEConv state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSparseMoEConv state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSparseMoEConv weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSparseMoEConv* Source = source; dWeightsUpdate(cParams, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSparseMoEConv GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSparseMoEConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronSparseMoEConv layer. | //+------------------------------------------------------------------+ void CNeuronSparseMoEConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMDL layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMDL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() <= units_s || layers < 1) ReturnFalse; uint units_ns = dimensions.Size() - units_s; //--- if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, embed_size, units_ns, (embed_size + 1) / 2, candidates, topK, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cPrepare.Init(0, index, OpenCL, dimensions, units_s + units_ns, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; if(!cLastNonSequence.Init(0, index, OpenCL, embed_size * units_ns, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cScenarios.Init(0, index, OpenCL, embed_size, units_s, embed_size, scenarios, (embed_size + 1) / 2, candidates, topK, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(None); //--- cFeatures.Clear(); cFlow.Clear(); cFeatures.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * units_s, optimization, iBatch) || !cFeatures.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (units_ns + scenarios), optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); //--- CNeuronFeatureSelfIteration* feature = NULL; CNeuronDomainAwareAttention* domain = NULL; CNeuronDomainFused* fused = NULL; for(uint i = 0; i < layers; i++) { index++; feature = new CNeuronFeatureSelfIteration(); if(!feature || !feature.Init(0, index, OpenCL, embed_size, units_s, heads, stack_size, (embed_size + 1) / 2, candidates, topK, optimization, iBatch) || !cFeatures.Add(feature)) DeleteObjAndFalse(feature); index++; domain = new CNeuronDomainAwareAttention(); if(!domain || !domain.Init(0, index, OpenCL, embed_size, units_ns + scenarios, heads, embed_size, stack_size * units_s, (embed_size + 1) / 2, candidates, topK, optimization, iBatch) || !cFlow.Add(domain)) DeleteObjAndFalse(domain); index++; fused = new CNeuronDomainFused(); if(!fused || !fused.Init(0, index, OpenCL, embed_size, units_ns, scenarios, (embed_size + 1) / 2, candidates, topK, optimization, iBatch) || !cFlow.Add(fused)) DeleteObjAndFalse(fused); } index++; CNeuronAddToStack* stack = new CNeuronAddToStack(); if(!stack || !stack.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch) || !cFeatures.Add(stack)) DeleteObjAndFalse(stack); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMDL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMDL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.FeedForward(NeuronOCL)) ReturnFalse; //--- uint layers = cFeatures.Total() - 2; //--- Features CNeuronBaseOCL* feature = cFeatures[0]; if(!feature || !DeConcat(feature.getOutput(), cLastNonSequence.getOutput(), cPrepare.getOutput(), feature.Neurons(), cLastNonSequence.Neurons(), 1)) ReturnFalse; for(int i = 1; i < cFeatures.Total(); i++) { feature = cFeatures[i]; if(!feature || !feature.FeedForward(cFeatures[i - 1])) ReturnFalse; } //--- Scenarios if(!cScenarios.FeedForward(cFeatures[0])) ReturnFalse; CNeuronBaseOCL* flow = cFlow[0]; if(!flow || !Concat(cLastNonSequence.getOutput(), cScenarios.getOutput(), flow.getOutput(), cLastNonSequence.Neurons(), cScenarios.Neurons(), 1)) ReturnFalse; //--- Flow for(uint i = 0; i < layers; i++) { uint shift_domain = i * 2 + 1; uint shift_feature = i + 2; if(!cFeatures[shift_feature]) ReturnFalse; if(cFeatures[shift_feature].Type() == defNeuronFeatureSelfIteration) feature = ((CNeuronFeatureSelfIteration*)cFeatures[shift_feature]).GetStack(); else feature = cFeatures[shift_feature]; flow = cFlow[shift_domain]; if(!flow || !flow.FeedForward(cFlow[shift_domain - 1], feature.getOutput())) ReturnFalse; flow = cFlow[shift_domain + 1]; if(!flow || !flow.FeedForward(cFlow[shift_domain])) ReturnFalse; } //--- if(!CNeuronPerTokenFFN::feedForward(flow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMDL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMDL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPerTokenFFN::calcInputGradients(cFlow[-1])) ReturnFalse; //---Flow int layers = int(cFeatures.Total()) - 2; CNeuronBaseOCL* flow = cFlow[-2]; if(!flow || !flow.CalcHiddenGradients(cFlow[-1])) ReturnFalse; flow = cFlow[-3]; if(!flow || !cFeatures[-1] || !flow.CalcHiddenGradients(cFlow[-2], cFeatures[-1].getOutput(), cFeatures[-1].getGradient(), (ENUM_ACTIVATION)cFeatures[-1].Activation())) ReturnFalse; CNeuronFeatureSelfIteration* feature = NULL; CNeuronBaseOCL* stack = NULL; for(int i = cFlow.Total() - 4; i >= 0; i--) { flow = cFlow[i]; if(!flow) ReturnFalse; feature = cFeatures[(i + 2) / 2]; if(!feature || !feature.GetStack()) ReturnFalse; stack = feature.GetStack(); if(!flow.CalcHiddenGradients(cFlow[i + 1], stack.getOutput(), stack.getGradient(), (ENUM_ACTIVATION)stack.Activation())) ReturnFalse; } //--- Features for(int i = cFeatures.Total() - 2; i >= 0; i--) { stack = cFeatures[i]; if(!stack || !stack.CalcHiddenGradients(cFeatures[i + 1])) ReturnFalse; } //--- Scenarios if(!DeConcat(cLastNonSequence.getGradient(), cScenarios.getGradient(), flow.getGradient(), cLastNonSequence.Neurons(), cScenarios.Neurons(), 1)) ReturnFalse; CBufferFloat *temp = stack.getGradient(); if(!stack.SetGradient(stack.getPrevOutput(), false) || !stack.CalcHiddenGradients(cScenarios.AsObject()) || !SumAndNormalize(temp, stack.getGradient(), temp, 1, false, 0, 0, 0, 1) || !stack.SetGradient(temp, false)) ReturnFalse; //--- if(!Concat(stack.getGradient(), cLastNonSequence.getGradient(), cPrepare.getGradient(), stack.Neurons(), cLastNonSequence.Neurons(), 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cPrepare.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMDL trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMDL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- uint layers = cFeatures.Total() - 2; //--- Features CNeuronBaseOCL *feature = NULL; for(int i = 1; i < cFeatures.Total(); i++) { feature = cFeatures[i]; if(!feature || !feature.UpdateInputWeights(cFeatures[i - 1])) ReturnFalse; } //--- Scenarios if(!cScenarios.UpdateInputWeights(cFeatures[0])) ReturnFalse; CNeuronBaseOCL* flow = cFlow[0]; //--- Flow for(uint i = 0; i < layers; i++) { uint shift_domain = i * 2 + 1; uint shift_feature = i + 2; if(!cFeatures[shift_feature]) ReturnFalse; if(cFeatures[shift_feature].Type() == defNeuronFeatureSelfIteration) feature = ((CNeuronFeatureSelfIteration*)cFeatures[shift_feature]).GetStack(); else feature = cFeatures[shift_feature]; flow = cFlow[shift_domain]; if(!flow || !flow.UpdateInputWeights(cFlow[shift_domain - 1], feature.getOutput())) ReturnFalse; flow = cFlow[shift_domain + 1]; if(!flow || !flow.UpdateInputWeights(cFlow[shift_domain])) ReturnFalse; } //--- if(!CNeuronPerTokenFFN::updateInputWeights(flow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMDL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMDL::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cFeatures.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMDL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMDL::Load(const int file_handle) { if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cPrepare.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cFeatures.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronMDL execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMDL::Clear(void) { if(!CNeuronPerTokenFFN::Clear()) ReturnFalse; //--- if(!cPrepare.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cScenarios.Clear()) ReturnFalse; if(!cFeatures.ClearStates()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMDL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronMDL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMDL* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cFeatures, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMDL GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMDL::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenFFN::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cFeatures.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMDL layer. | //+------------------------------------------------------------------+ void CNeuronMDL::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cFeatures.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUniMixerTokenizer layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units, uint embed_size, uint blocks, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(blocks <= 0 || embed_size <= 0 || units <= 0 || (embed_size * units) % blocks != 0) ReturnFalse; uint block_size = (embed_size * units) / blocks; if(!CNeuronFieldAwareConv::Init(numOutputs, myIndex, open_cl, block_size, block_size, blocks, (block_size + 3) / 4, candidates, topK, optimization_type, batch)) ReturnFalse; if(!cTokenizer.Init(0, 0, OpenCL, dimensions, units, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUniMixerTokenizer feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::feedForward(cTokenizer.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronUniMixerTokenizer layer. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldAwareConv::calcInputGradients(cTokenizer.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTokenizer.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUniMixerTokenizer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::updateInputWeights(cTokenizer.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUniMixerTokenizer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::Save(const int file_handle) { if(!CNeuronFieldAwareConv::Save(file_handle)) ReturnFalse; if(!cTokenizer.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUniMixerTokenizer state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::Load(const int file_handle) { if(!CNeuronFieldAwareConv::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTokenizer.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronUniMixerTokenizer weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUniMixerTokenizer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldAwareConv::WeightsUpdate(source, tau)) ReturnFalse; CNeuronUniMixerTokenizer* Source = source; dWeightsUpdate(cTokenizer, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUniMixerTokenizer GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronUniMixerTokenizer::SetOpenCL(COpenCLMy *obj) { CNeuronFieldAwareConv::SetOpenCL(obj); cTokenizer.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUniMixerTokenizer layer. | //+------------------------------------------------------------------+ void CNeuronUniMixerTokenizer::TrainMode(bool flag) { CNeuronFieldAwareConv::TrainMode(flag); cTokenizer.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUniMixer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint blocks, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units <= blocks || units % blocks != 0) ReturnFalse; if(!CNeuronPerTokenSwiGLU::Init(numOutputs, myIndex, open_cl, window, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iBlocks = blocks; uint block_size = units / iBlocks; //--- uint index = 0; if(!cWb.Init(0, index, OpenCL, block_size * block_size, iBlocks, block_size, candidates, topK, optimization, iBatch)) ReturnFalse; cWb.SetActivationFunction(None); index++; if(!cLocalMixing.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cLocalMixing.SetActivationFunction(None); for(uint i = 0; i < cG.Size(); i++) { index++; if(!cG[i].Init(0, index, OpenCL, blocks * (blocks + 3) / 4, optimization, iBatch)) ReturnFalse; cG[i].SetActivationFunction(None); } index++; if(!cGlobalMixing[0].Init(0, index, OpenCL, block_size * window * (blocks + 3) / 4, optimization, iBatch)) ReturnFalse; cGlobalMixing[0].SetActivationFunction(None); index++; if(!cGlobalMixing[1].Init(0, index, OpenCL, window * units, optimization, iBatch)) ReturnFalse; cGlobalMixing[1].SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUniMixer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(bTrain) { if(!cWb.FeedForward()) ReturnFalse; for(uint i = 0; i < cG.Size(); i++) if(!cG[i].FeedForward()) ReturnFalse; } //--- uint dimension = GetWindow(); uint units = GetFields(); uint block_size = units / iBlocks; //--- if(!MatMul(cWb.getOutput(), NeuronOCL.getOutput(), cLocalMixing.getOutput(), block_size, block_size, dimension, iBlocks, true)) ReturnFalse; if(!MatMul(cG[0].getOutput(), cLocalMixing.getOutput(), cGlobalMixing[0].getOutput(), cG[0].Neurons() / iBlocks, iBlocks, block_size * dimension, 1, true)) ReturnFalse; if(!MatMul(cG[1].getOutput(), cGlobalMixing[0].getOutput(), cGlobalMixing[1].getOutput(), iBlocks, cG[1].Neurons() / iBlocks, block_size * dimension, 1, true)) ReturnFalse; if(!Normalize(cGlobalMixing[1].getOutput(), dimension)) ReturnFalse; if(!SumAndNormalize(cGlobalMixing[1].getOutput(), NeuronOCL.getOutput(), cGlobalMixing[1].getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronPerTokenSwiGLU::feedForward(cGlobalMixing[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronUniMixer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPerTokenSwiGLU::calcInputGradients(cGlobalMixing[1].AsObject())) ReturnFalse; //--- uint dimension = GetWindow(); uint units = GetFields(); uint block_size = units / iBlocks; //--- if(!MatMulGrad(cG[1].getOutput(), cG[1].getGradient(), cGlobalMixing[0].getOutput(), cGlobalMixing[0].getGradient(), cGlobalMixing[1].getGradient(), iBlocks, cG[1].Neurons() / iBlocks, block_size * dimension, 1, true)) ReturnFalse; if(!MatMulGrad(cG[0].getOutput(), cG[0].getGradient(), cLocalMixing.getOutput(), cLocalMixing.getGradient(), cGlobalMixing[0].getGradient(), cG[0].Neurons() / iBlocks, iBlocks, block_size * dimension, 1, true)) ReturnFalse; if(!MatMulGrad(cWb.getOutput(), cWb.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), cLocalMixing.getGradient(), block_size, block_size, dimension, iBlocks, true)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cGlobalMixing[1].getGradient(), NeuronOCL.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUniMixer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cWb.UpdateInputWeights()) ReturnFalse; for(uint i = 0; i < cG.Size(); i++) if(!cG[i].UpdateInputWeights()) ReturnFalse; if(!CNeuronPerTokenSwiGLU::updateInputWeights(cGlobalMixing[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronUniMixer weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenSwiGLU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUniMixer* Source = source; dWeightsUpdate(cWb, Source, tau); for(uint i = 0; i < cG.Size(); i++) dWeightsUpdate(cG[i], Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUniMixer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::Save(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Save(file_handle)) ReturnFalse; if(!cWb.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cG.Size(); i++) if(!cG[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUniMixer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixer::Load(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cWb.AsObject())) ReturnFalse; for(uint i = 0; i < cG.Size(); i++) if(!LoadInsideLayer(file_handle, cG[i].AsObject())) ReturnFalse; //--- iBlocks = cWb.GetFields(); uint dimension = GetWindow(); uint units = GetFields(); uint block_size = units / iBlocks; //--- uint index = 1; if(!cLocalMixing.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cLocalMixing.SetActivationFunction(None); index += cG.Size() + 1; if(!cGlobalMixing[0].Init(0, index, OpenCL, block_size * dimension * (iBlocks + 3) / 4, optimization, iBatch)) ReturnFalse; cGlobalMixing[0].SetActivationFunction(None); index++; if(!cGlobalMixing[1].Init(0, index, OpenCL, dimension * units, optimization, iBatch)) ReturnFalse; cGlobalMixing[1].SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUniMixer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUniMixer::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenSwiGLU::SetOpenCL(obj); //--- cWb.SetOpenCL(OpenCL); for(uint i = 0; i < cG.Size(); i++) { cG[i].SetOpenCL(OpenCL); cGlobalMixing[i].SetOpenCL(OpenCL); } cLocalMixing.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUniMixer layer. | //+------------------------------------------------------------------+ void CNeuronUniMixer::TrainMode(bool flag) { CNeuronPerTokenSwiGLU::TrainMode(flag); //--- cWb.TrainMode(bTrain); for(uint i = 0; i < cG.Size(); i++) cG[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUniMixerBlock layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint blocks, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() <= units_s || layers < 1) ReturnFalse; uint units_ns = dimensions.Size() - units_s; uint inside_embed_size = (embed_size + 1) / 2; //--- if(!CNeuronPerTokenSwiGLU::Init(numOutputs, myIndex, open_cl, embed_size, blocks, inside_embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- Prepare uint index = 0; if(!cPrepare.Init(0, index, OpenCL, dimensions, units_s + units_ns, embed_size, blocks, candidates, topK, optimization, iBatch)) ReturnFalse; index++; if(!cLastNonSequence.Init(0, index, OpenCL, embed_size * units_ns, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cScenarios.Init(0, index, OpenCL, embed_size, units_s, embed_size, scenarios, inside_embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(None); //--- cFeatures.Clear(); cFlow.Clear(); cOutputProj.Clear(); cFeatures.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cOutputProj.SetOpenCL(OpenCL); //--- Features index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * units_s, optimization, iBatch) || !cFeatures.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronAddToStack* stack = new CNeuronAddToStack(); if(!stack || !stack.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch) || !cFeatures.Add(stack)) DeleteObjAndFalse(stack); index++; CNeuronSiameseNorm* mixing = new CNeuronSiameseNorm(); if(!mixing || !mixing.Init(0, index, OpenCL, embed_size, units_s * stack_size, blocks, inside_embed_size, candidates, topK, optimization, iBatch) || !cFeatures.Add(mixing)) DeleteObjAndFalse(mixing); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, mixing.Neurons(), optimization, iBatch) || !cFeatures.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); //--- Flow index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, mixing.Neurons() + cScenarios.Neurons() + cLastNonSequence.Neurons(), optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); for(uint i = 0; i < layers; i++) { index++; mixing = new CNeuronSiameseNorm(); if(!mixing || !mixing.Init(0, index, OpenCL, embed_size, units_s * stack_size + units_ns + scenarios, blocks, inside_embed_size, candidates, topK, optimization, iBatch) || !cFlow.Add(mixing)) DeleteObjAndFalse(mixing); } //--- Ouptput Projection index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, mixing.Neurons(), optimization, iBatch) || !cOutputProj.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; uint window_in = neuron.Neurons() / blocks; CNeuronFieldAwareConv* proj = new CNeuronFieldAwareConv(); if(!proj || !proj.Init(0, index, OpenCL, window_in, embed_size, blocks, inside_embed_size, candidates, topK, optimization, iBatch) || !cOutputProj.Add(proj)) DeleteObjAndFalse(proj); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUniMixerBlock feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.FeedForward(NeuronOCL)) ReturnFalse; //--- Features CNeuronBaseOCL* feature = cFeatures[0]; if(!feature || !DeConcat(feature.getOutput(), cLastNonSequence.getOutput(), cPrepare.getOutput(), feature.Neurons(), cLastNonSequence.Neurons(), 1)) ReturnFalse; for(int i = 1; i < cFeatures.Total() - 1; i++) { feature = cFeatures[i]; if(!feature || !feature.FeedForward(cFeatures[i - 1])) ReturnFalse; } if(feature.Type() == defNeuronSiameseNorm) { CNeuronSiameseNorm* mixing = feature; CNeuronBaseOCL* Y = mixing.GetYOut(); uint dimension = mixing.GetWindow(); uint units = mixing.GetFields(); feature = cFeatures[-1]; if(!feature) ReturnFalse; if(!Concat(Y.getOutput(), Y.getOutput(), feature.getOutput(), dimension, 0, units)) ReturnFalse; if(!Normalize(feature.getOutput(), dimension)) ReturnFalse; if(!SumAndNormalize(mixing.getOutput(), feature.getOutput(), feature.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; } else { feature = cFeatures[-1]; if(!feature || !feature.FeedForward(cFeatures[-2])) ReturnFalse; } //--- Scenarios if(!cScenarios.FeedForward(cFeatures[0])) ReturnFalse; //--- Flow uint blocks = CNeuronPerTokenSwiGLU::GetFields(); CNeuronBaseOCL* flow = cFlow[0]; if(!flow || !Concat(cLastNonSequence.getOutput(), cScenarios.getOutput(), feature.getOutput(), flow.getOutput(), cLastNonSequence.Neurons() / blocks, cScenarios.Neurons() / blocks, feature.Neurons() / blocks, blocks)) ReturnFalse; flow = cFlow[1]; if(!flow || !flow.FeedForward(cFlow[0], cFlow[0].getOutput())) ReturnFalse; for(int i = 2; i < cFlow.Total(); i++) { flow = cFlow[i]; if(!flow || !flow.FeedForward(cFlow[i - 1])) ReturnFalse; } //--- Output Projection if(flow.Type() != defNeuronSiameseNorm) ReturnFalse; CNeuronSiameseNorm* mixing = flow; CNeuronBaseOCL* Y = mixing.GetYOut(); uint dimension = mixing.GetWindow(); uint units = mixing.GetFields(); CNeuronBaseOCL* proj = cOutputProj[0]; if(!proj) ReturnFalse; if(!Concat(Y.getOutput(), Y.getOutput(), proj.getOutput(), dimension, 0, units)) ReturnFalse; if(!Normalize(proj.getOutput(), dimension)) ReturnFalse; if(!SumAndNormalize(mixing.getOutput(), proj.getOutput(), proj.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; for(int i = 1; i < cOutputProj.Total(); i++) { proj = cOutputProj[i]; if(!proj || !proj.FeedForward(cOutputProj[i - 1])) ReturnFalse; } if(!CNeuronPerTokenSwiGLU::feedForward(proj)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronUniMixerBlock layer. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPerTokenSwiGLU::calcInputGradients(cOutputProj[-1])) ReturnFalse; //--- Output Projection CNeuronBaseOCL* proj = NULL; for(int i = cOutputProj.Total() - 2; i >= 0; i--) { proj = cOutputProj[i]; if(!proj || !proj.CalcHiddenGradients(cOutputProj[i + 1])) ReturnFalse; } //--- Flow CNeuronBaseOCL* flow = cFlow[-1]; if(flow.Type() != defNeuronSiameseNorm) ReturnFalse; CNeuronSiameseNorm* mixing = flow; CNeuronBaseOCL* Y = mixing.GetYOut(); if(!DeActivation(mixing.getOutput(), mixing.getGradient(), proj.getGradient(), mixing.Activation())) ReturnFalse; if(!DeActivation(Y.getOutput(), Y.getGradient(), proj.getGradient(), Y.Activation())) ReturnFalse; for(int i = cFlow.Total() - 2; i >= 0; i--) { flow = cFlow[i]; if(!flow || !flow.CalcHiddenGradients(cFlow[i + 1])) ReturnFalse; } //--- Features CNeuronBaseOCL* feature = cFeatures[-1]; uint blocks = CNeuronPerTokenSwiGLU::GetFields(); if(!feature || !DeConcat(cLastNonSequence.getGradient(), cScenarios.getGradient(), feature.getGradient(), flow.getGradient(), cLastNonSequence.Neurons() / blocks, cScenarios.Neurons() / blocks, feature.Neurons() / blocks, blocks)) ReturnFalse; Deactivation(feature); feature = cFeatures[-2]; if(!feature) ReturnFalse; if(feature.Type() == defNeuronSiameseNorm) { mixing = feature; Y = mixing.GetYOut(); if(!DeActivation(feature.getOutput(), feature.getGradient(), cFeatures[-1].getGradient(), feature.Activation())) ReturnFalse; if(!DeActivation(Y.getOutput(), Y.getGradient(), cFeatures[-1].getGradient(), Y.Activation())) ReturnFalse; } else if(!feature.CalcHiddenGradients(cFeatures[-1])) ReturnFalse; for(int i = cFeatures.Total() - 3; i >= 0; i--) { feature = cFeatures[i]; if(!feature || !feature.CalcHiddenGradients(cFeatures[i + 1])) ReturnFalse; } //--- Scenarios Deactivation(cScenarios); CBufferFloat *temp = feature.getGradient(); if(!feature.SetGradient(feature.getPrevOutput(), false) || !feature.CalcHiddenGradients(cScenarios.AsObject()) || !SumAndNormalize(temp, feature.getGradient(), temp, 1, false, 0, 0, 0, 1) || !feature.SetGradient(temp, false)) ReturnFalse; //--- if(!Concat(feature.getGradient(), cLastNonSequence.getGradient(), cPrepare.getGradient(), feature.Neurons(), cLastNonSequence.Neurons(), 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cPrepare.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUniMixerBlock trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- Features CNeuronBaseOCL* feature = NULL; for(int i = 1; i < cFeatures.Total() - 1; i++) { feature = cFeatures[i]; if(!feature || !feature.UpdateInputWeights(cFeatures[i - 1])) ReturnFalse; } if(feature.Type() != defNeuronSiameseNorm) { feature = cFeatures[-1]; if(!feature || !feature.UpdateInputWeights(cFeatures[-2])) ReturnFalse; } //--- Scenarios if(!cScenarios.UpdateInputWeights(cFeatures[0])) ReturnFalse; //--- Flow CNeuronBaseOCL* flow = NULL; for(int i = 1; i < cFlow.Total(); i++) { flow = cFlow[i]; if(!flow || !flow.UpdateInputWeights(cFlow[i - 1])) ReturnFalse; } //--- Output Projection CNeuronBaseOCL* proj = NULL; for(int i = 1; i < cOutputProj.Total(); i++) { proj = cOutputProj[i]; if(!proj || !proj.UpdateInputWeights(cOutputProj[i - 1])) ReturnFalse; } if(!CNeuronPerTokenSwiGLU::updateInputWeights(proj)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUniMixerBlock state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::Save(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cFeatures.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cOutputProj.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUniMixerBlock state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::Load(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cPrepare.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cFeatures.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; if(!cOutputProj.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronUniMixerBlock execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::Clear(void) { if(!CNeuronPerTokenSwiGLU::Clear()) ReturnFalse; //--- if(!cPrepare.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cScenarios.Clear()) ReturnFalse; if(!cFeatures.ClearStates()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; if(!cOutputProj.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronUniMixerBlock weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUniMixerBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenSwiGLU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUniMixerBlock* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cFeatures, Source, tau); dWeightsUpdate(cFlow, Source, tau); dWeightsUpdate(cOutputProj, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUniMixerBlock GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUniMixerBlock::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenSwiGLU::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cFeatures.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cOutputProj.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUniMixerBlock layer. | //+------------------------------------------------------------------+ void CNeuronUniMixerBlock::TrainMode(bool flag) { CNeuronPerTokenSwiGLU::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cFeatures.TrainMode(bTrain); cFlow.TrainMode(bTrain); cOutputProj.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronINFNetScenarios layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint &fields[], uint window_out, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Fields require [Sequence, Context, Global] if(fields.Size() < 3) ReturnFalse; uint fields_total = 0; for(int i = 0; i < 3; i++) { if(fields[i] <= 0) ReturnFalse; fields_total += fields[i]; } if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, fields_total * window_out, optimization_type, batch)) ReturnFalse; //--- cSequence.Clear(); cContext.Clear(); cGlobal.Clear(); cSequence.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cGlobal.SetOpenCL(OpenCL); uint index = 0; //--- Sequence { CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, window_in * fields[0], optimization, iBatch) || !cSequence.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronScenariosToken* scen = new CNeuronScenariosToken(); if(!scen || !scen.Init(0, index, OpenCL, window_in, fields[0], window_out, fields[0], embed_size, candidates, topK, optimization, iBatch) || !cSequence.Add(scen)) DeleteObjAndFalse(scen); scen.SetActivationFunction(None); } //--- Context { index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, window_in * fields[1], optimization, iBatch) || !cContext.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronFieldPatternEmbedding* scen = new CNeuronFieldPatternEmbedding(); if(!scen || !scen.Init(fields[1]*window_out, index, OpenCL, window_in, fields[1], (window_out + 1) / 2, candidates, topK, optimization, iBatch) || !cContext.Add(scen)) DeleteObjAndFalse(scen); scen.SetActivationFunction(None); CNeuronBaseOCL* mix = new CNeuronBaseOCL(); if(!mix || !mix.Init(0, index, OpenCL, window_out * fields[1], optimization, iBatch) || !cContext.Add(mix)) DeleteObjAndFalse(mix); mix.SetActivationFunction(None); } //--- Global { index++; CFieldAwareParams* scen = new CFieldAwareParams(); if(!scen || !scen.Init(0, index, OpenCL, window_out, fields[2], embed_size, candidates, topK, optimization, iBatch) || !cGlobal.Add(scen)) DeleteObjAndFalse(scen); scen.SetActivationFunction(None); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronINFNetScenarios feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSequence[0]; CNeuronBaseOCL* context = cContext[0]; CNeuronBaseOCL* global = NULL; //--- if(!sequence || !context || !DeConcat(sequence.getOutput(), context.getOutput(), NeuronOCL.getOutput(), sequence.Neurons(), context.Neurons(), 1)) ReturnFalse; //--- Sequence for(int i = 1; i < cSequence.Total(); i++) { sequence = cSequence[i]; if(!sequence || !sequence.FeedForward(cSequence[i - 1])) ReturnFalse; } //--- Context for(int i = 1; i < cContext.Total(); i++) { context = cContext[i]; if(!context || !context.FeedForward(cContext[i - 1])) ReturnFalse; } //--- Global if(bTrain) { for(int i = 0; i < cGlobal.Total(); i++) { global = cGlobal[i]; if(!global || !global.FeedForward((i == 0 ? NeuronOCL : cGlobal[i - 1]))) ReturnFalse; } } else { global = cGlobal[-1]; if(!global) ReturnFalse; } //--- Concatenate if(!Concat(sequence.getOutput(), context.getOutput(), global.getOutput(), Output, sequence.Neurons(), context.Neurons(), global.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronINFNetScenarios layer. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSequence[-1]; CNeuronBaseOCL* context = cContext[-1]; CNeuronBaseOCL* global = cGlobal[-1]; //--- Concatenating if(!sequence || !context || !global || !DeConcat(sequence.getGradient(), context.getGradient(), global.getGradient(), Gradient, sequence.Neurons(), context.Neurons(), global.Neurons(), 1)) ReturnFalse; Deactivation(sequence); Deactivation(context); Deactivation(global); //--- Global for(int i = cGlobal.Total() - 2; i >= 0; i--) { global = cGlobal[i]; if(!global || !global.CalcHiddenGradients(cGlobal[i + 1])) ReturnFalse; } //--- Context for(int i = cContext.Total() - 2; i >= 0; i--) { context = cContext[i]; if(!context || !context.CalcHiddenGradients(cContext[i + 1])) ReturnFalse; } //--- Sequence for(int i = cSequence.Total() - 2; i >= 0; i--) { sequence = cSequence[i]; if(!sequence || !sequence.CalcHiddenGradients(cSequence[i + 1])) ReturnFalse; } //--- if(!Concat(sequence.getGradient(), context.getGradient(), NeuronOCL.getGradient(), sequence.Neurons(), context.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronINFNetScenarios trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* neuron = NULL; //--- Sequence for(int i = 1; i < cSequence.Total(); i++) { neuron = cSequence[i]; if(!neuron || !neuron.UpdateInputWeights(cSequence[i - 1])) ReturnFalse; } //--- Context for(int i = 1; i < cContext.Total(); i++) { neuron = cContext[i]; if(!neuron || !neuron.UpdateInputWeights(cContext[i - 1])) ReturnFalse; } //--- Global for(int i = 0; i < cGlobal.Total(); i++) { neuron = cGlobal[i]; if(!neuron || !neuron.UpdateInputWeights((i == 0 ? NeuronOCL : cGlobal[i - 1]))) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronINFNetScenarios state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cSequence.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cGlobal.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronINFNetScenarios state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cSequence.Load(file_handle)) ReturnFalse; if(!cContext.Load(file_handle)) ReturnFalse; if(!cGlobal.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronINFNetScenarios weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronINFNetScenarios* Source = source; dWeightsUpdate(cSequence, Source, tau); dWeightsUpdate(cContext, Source, tau); dWeightsUpdate(cGlobal, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronINFNetScenarios execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetScenarios::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cSequence.ClearStates()) ReturnFalse; if(!cContext.ClearStates()) ReturnFalse; if(!cGlobal.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronINFNetScenarios GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronINFNetScenarios::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cSequence.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cGlobal.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronINFNetScenarios layer. | //+------------------------------------------------------------------+ void CNeuronINFNetScenarios::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cSequence.TrainMode(bTrain); cContext.TrainMode(bTrain); cGlobal.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronINFNetHUBSequence layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint hub_size, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint hidden_dim = (window + 3) / 4; if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, hidden_dim * units_count, hidden_dim * units_count, hub_size, 1, variables, optimization_type, batch)) ReturnFalse; if(!cProjection.Init(0, 0, OpenCL, window, window, hidden_dim, units_count, variables, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronINFNetHUBSequence feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronINFNetHUBSequence layer. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeConvBlock::calcInputGradients(cProjection.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronINFNetHUBSequence trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronINFNetHUBSequence state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronINFNetHUBSequence state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronINFNetHUBSequence weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronINFNetHUBSequence* Source = source; dWeightsUpdate(cProjection, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronINFNetHUBSequence execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetHUBSequence::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cProjection.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronINFNetHUBSequence GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronINFNetHUBSequence::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronINFNetHUBSequence layer. | //+------------------------------------------------------------------+ void CNeuronINFNetHUBSequence::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cProjection.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronINFNetBlock layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() <= units_s || layers < 1 || units_s < 1) ReturnFalse; iSequenceVar = units_s; iSequenceUnits = stack_size; iContextUnits = dimensions.Size() - units_s; iDimension = embed_size; iScenariosUnits = iSequenceVar + iContextUnits + 1; uint inside_embed_size = (embed_size + 1) / 2; //--- if(!CNeuronPerTokenSwiGLU::Init(numOutputs, myIndex, open_cl, iDimension, iScenariosUnits + iSequenceVar + 1, inside_embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- Prepare uint index = 0; if(!cPrepare.Init(0, index, OpenCL, dimensions, iSequenceVar + iContextUnits, iDimension, iSequenceVar + iContextUnits, candidates, topK, optimization, iBatch)) ReturnFalse; //--- cSequence.Clear(); cContext.Clear(); cScenarios.Clear(); cHUBs.Clear(); cSequence.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cHUBs.SetOpenCL(OpenCL); //--- Sequence index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iDimension * iSequenceVar, optimization, iBatch) || !cSequence.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronAddToStack* stack = new CNeuronAddToStack(); if(!stack || !stack.Init(0, index, OpenCL, iSequenceUnits, iDimension, iSequenceVar, optimization, iBatch) || !cSequence.Add(stack)) DeleteObjAndFalse(stack); for(uint i = 0; i < layers; i++) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, stack.Neurons(), optimization, iBatch) || !cSequence.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); } //--- Context for(uint i = 0; i <= layers; i++) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iDimension * iContextUnits, optimization, iBatch) || !cContext.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); } //--- Scenarios index++; CNeuronINFNetScenarios* scen = new CNeuronINFNetScenarios(); uint fields[] = {iSequenceVar, iContextUnits, 1}; if(!scen || !scen.Init(0, index, OpenCL, iDimension, fields, iDimension, inside_embed_size, candidates, topK, optimization, iBatch) || !cScenarios.Add(scen)) DeleteObjAndFalse(scen); for(uint i = 0; i < layers; i++) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, scen.Neurons(), optimization, iBatch) || !cScenarios.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); } //--- HUBs index++; CNeuronINFNetHUBSequence* hub = new CNeuronINFNetHUBSequence(); if(!hub || !hub.Init(0, index, OpenCL, iDimension, iSequenceUnits, iDimension, iSequenceVar, optimization, iBatch) || !cHUBs.Add(hub)) DeleteObjAndFalse(hub); hub = new CNeuronINFNetHUBSequence(); if(!hub || !hub.Init(0, index, OpenCL, iDimension, iContextUnits, iDimension, 1, optimization, iBatch) || !cHUBs.Add(hub)) DeleteObjAndFalse(hub); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, scen.Neurons() + iDimension * (iSequenceVar + 1), optimization, iBatch) || !cHUBs.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); CNeuronDomainAwareAttention* att = NULL; CNeuronFieldAwareConv* conv = NULL; for(uint i = 0; i < layers; i++) { index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iSequenceUnits * iSequenceVar, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iContextUnits, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iScenariosUnits, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 3 * att.Neurons(), optimization, iBatch) || !cHUBs.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 3 * iDimension, 2 * iDimension, iSequenceVar + iScenariosUnits + 1, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 3 * iDimension, iDimension, iSequenceVar + iScenariosUnits + 1, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(conv)) DeleteObjAndFalse(conv); } index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iSequenceUnits * iSequenceVar, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iContextUnits, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; att = new CNeuronDomainAwareAttention(); if(!att || !att.Init(0, index, OpenCL, iDimension, iScenariosUnits + iSequenceVar + 1, heads, iDimension, iScenariosUnits, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(att)) DeleteObjAndFalse(att); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 3 * att.Neurons(), optimization, iBatch) || !cHUBs.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 3 * iDimension, iDimension, iSequenceVar + iScenariosUnits + 1, inside_embed_size, candidates, topK, optimization, iBatch) || !cHUBs.Add(conv)) DeleteObjAndFalse(conv); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronINFNetBlock feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSequence[0]; CNeuronBaseOCL* context = cContext[0]; if(!sequence || !context || !DeConcat(sequence.getOutput(), context.getOutput(), cPrepare.getOutput(), sequence.Neurons(), context.Neurons(), 1)) ReturnFalse; sequence = cSequence[1]; if(!sequence || !sequence.FeedForward(cSequence[0])) ReturnFalse; //--- CNeuronBaseOCL* scen = cScenarios[0]; if(!scen || !scen.FeedForward(cPrepare.AsObject())) ReturnFalse; //--- HUBs CNeuronBaseOCL* hub = cHUBs[0]; if(!hub || !hub.FeedForward(sequence)) ReturnFalse; hub = cHUBs[1]; if(!hub || !hub.FeedForward(context)) ReturnFalse; hub = cHUBs[2]; if(!hub || !Concat(cHUBs[0].getOutput(), cHUBs[1].getOutput(), scen.getOutput(), hub.getOutput() , cHUBs[0].Neurons(), cHUBs[1].Neurons(), scen.Neurons(), 1)) ReturnFalse; //--- int layers = cContext.Total() - 1; for(int i = 0; i < layers; i++) { CNeuronBaseOCL* att_seq = cHUBs[i * 6 + 3]; if(!att_seq || !att_seq.FeedForward(hub, sequence.getOutput())) ReturnFalse; CNeuronBaseOCL* att_cont = cHUBs[i * 6 + 4]; if(!att_cont || !att_cont.FeedForward(hub, context.getOutput())) ReturnFalse; CNeuronBaseOCL* att_scen = cHUBs[i * 6 + 5]; if(!att_scen || !att_scen.FeedForward(hub, scen.getOutput())) ReturnFalse; CNeuronBaseOCL* concat = cHUBs[i * 6 + 6]; if(!concat || !Concat(att_seq.getOutput(), att_cont.getOutput(), att_scen.getOutput(), concat.getOutput(), iDimension, iDimension, iDimension, iSequenceVar + iScenariosUnits + 1)) ReturnFalse; CNeuronBaseOCL* scal_and_shift = cHUBs[i * 6 + 7]; if(!scal_and_shift || !scal_and_shift.FeedForward(concat)) ReturnFalse; sequence = cSequence[i + 2]; context = cContext[i + 1]; scen = cScenarios[i + 1]; if(!sequence || !context || !scen || !INFNetBGU(scal_and_shift, cSequence[i + 1], sequence, cContext[i], context, cScenarios[i], scen)) ReturnFalse; hub = cHUBs[i * 6 + 8]; if(!hub.FeedForward(concat)) ReturnFalse; } CNeuronBaseOCL* att_seq = cHUBs[layers * 6 + 3]; if(!att_seq || !att_seq.FeedForward(hub, sequence.getOutput())) ReturnFalse; CNeuronBaseOCL* att_cont = cHUBs[layers * 6 + 4]; if(!att_cont || !att_cont.FeedForward(hub, context.getOutput())) ReturnFalse; CNeuronBaseOCL* att_scen = cHUBs[layers * 6 + 5]; if(!att_scen || !att_scen.FeedForward(hub, scen.getOutput())) ReturnFalse; CNeuronBaseOCL* concat = cHUBs[layers * 6 + 6]; if(!concat || !Concat(att_seq.getOutput(), att_cont.getOutput(), att_scen.getOutput(), concat.getOutput(), iDimension, iDimension, iDimension, iSequenceVar + iScenariosUnits + 1)) ReturnFalse; hub = cHUBs[layers * 6 + 7]; if(!hub.FeedForward(concat)) ReturnFalse; //--- if(!CNeuronPerTokenSwiGLU::feedForward(hub)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the inf net bgu step of the CNeuronINFNetBlock layer. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::INFNetBGU(CNeuronBaseOCL *scale_and_shift, CNeuronBaseOCL *sequence_in, CNeuronBaseOCL *sequence_out, CNeuronBaseOCL *context_in, CNeuronBaseOCL *context_out, CNeuronBaseOCL *scenarios_in, CNeuronBaseOCL *scenarios_out) { if(!OpenCL || !scale_and_shift || !sequence_in || !sequence_out || !context_in || !context_out || !scenarios_in || !scenarios_out) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { (uint)MathMin( MathMax( MathMax(iScenariosUnits, iContextUnits), iSequenceUnits), OpenCL.GetMaxLocalSize(1)), iDimension, iSequenceVar + 2 }; uint local_work_size[] = { global_work_size[0], 1, 1}; uint kernel = def_k_INFNetBGU; setBuffer(kernel, def_k_infnbgu_scal_and_shift, scale_and_shift.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_context, context_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_context_out, context_out.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_sequence, sequence_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_sequence_out, sequence_out.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_scenarios, scenarios_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgu_scenarios_out, scenarios_out.getOutputIndex()) setArgument(kernel, def_k_infnbgu_context_units, iContextUnits) setArgument(kernel, def_k_infnbgu_scenarios_units, iScenariosUnits) setArgument(kernel, def_k_infnbgu_sequence_units, iSequenceUnits) setArgument(kernel, def_k_infnbgu_sequence_vars, iSequenceVar) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!scenarios_out.getOutput().BufferRead()) ReturnFalse; #endif //--- if(!Normalize(sequence_out.getOutput(), iDimension) || !Normalize(context_out.getOutput(), iDimension) || !Normalize(scenarios_out.getOutput(), iDimension)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the inf net bgu grad step of the CNeuronINFNetBlock | //| layer. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::INFNetBGUGrad(CNeuronBaseOCL *scale_and_shift, CNeuronBaseOCL *sequence_in, CNeuronBaseOCL *sequence_out, CNeuronBaseOCL *context_in, CNeuronBaseOCL *context_out, CNeuronBaseOCL *scenarios_in, CNeuronBaseOCL *scenarios_out) { if(!OpenCL || !scale_and_shift || !sequence_in || !sequence_out || !context_in || !context_out || !scenarios_in || !scenarios_out) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { (uint)MathMin( MathMax( MathMax(iScenariosUnits, iContextUnits), iSequenceUnits), OpenCL.GetMaxLocalSize(1)), iDimension, iSequenceVar + 2 }; uint local_work_size[] = { global_work_size[0], 1, 1}; uint kernel = def_k_INFNetBGUGrad; setBuffer(kernel, def_k_infnbgugr_scal_and_shift, scale_and_shift.getOutputIndex()) setBuffer(kernel, def_k_infnbgugr_scal_and_shift_grad, scale_and_shift.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_context, context_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgugr_context_grad, context_in.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_context_out_grad, context_out.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_sequence, sequence_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgugr_sequence_grad, sequence_in.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_sequence_out_grad, sequence_out.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_scenarios, scenarios_in.getOutputIndex()) setBuffer(kernel, def_k_infnbgugr_scenarios_grad, scenarios_in.getGradientIndex()) setBuffer(kernel, def_k_infnbgugr_scenarios_out_grad, scenarios_out.getGradientIndex()) setArgument(kernel, def_k_infnbgugr_context_units, iContextUnits) setArgument(kernel, def_k_infnbgugr_scenarios_units, iScenariosUnits) setArgument(kernel, def_k_infnbgugr_sequence_units, iSequenceUnits) setArgument(kernel, def_k_infnbgugr_sequence_vars, iSequenceVar) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!scale_and_shift.getGradient().BufferRead()) ReturnFalse; #endif Deactivation(scenarios_in); Deactivation(sequence_in); Deactivation(context_in); //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronINFNetBlock layer. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPerTokenSwiGLU::calcInputGradients(cHUBs[-1])) ReturnFalse; //--- int layers = cContext.Total() - 1; CNeuronBaseOCL* concat = cHUBs[-2]; if(!concat || !concat.CalcHiddenGradients(cHUBs[-1])) ReturnFalse; CNeuronBaseOCL* att_seq = cHUBs[-5]; CNeuronBaseOCL* att_cont = cHUBs[-4]; CNeuronBaseOCL* att_scen = cHUBs[-3]; if(!att_seq || !att_cont || !att_scen || !DeConcat(att_seq.getGradient(), att_cont.getGradient(), att_scen.getGradient(), concat.getGradient(), iDimension, iDimension, iDimension, iSequenceVar + iScenariosUnits + 1)) ReturnFalse; CNeuronBaseOCL* hub = cHUBs[-6]; CNeuronBaseOCL* sequence = cSequence[-1]; CNeuronBaseOCL* context = cContext[-1]; CNeuronBaseOCL* scen = cScenarios[-1]; CNeuronBaseOCL* scal_and_shift = NULL; //--- if(!hub || !sequence || !hub.CalcHiddenGradients(att_seq, sequence.getOutput(), sequence.getGradient(), (ENUM_ACTIVATION)sequence.Activation())) ReturnFalse; CBufferFloat* temp = hub.getGradient(); if(!hub.SetGradient(hub.getPrevOutput(), false) || !context || !hub.CalcHiddenGradients(att_cont, context.getOutput(), context.getGradient(), (ENUM_ACTIVATION)context.Activation()) || !SumAndNormalize(temp, hub.getGradient(), temp, iDimension, false, 0, 0, 0, 1)) ReturnFalse; if(!scen || !hub.CalcHiddenGradients(att_scen, scen.getOutput(), scen.getGradient(), (ENUM_ACTIVATION)scen.Activation()) || !SumAndNormalize(temp, hub.getGradient(), temp, iDimension, false, 0, 0, 0, 1) || !hub.SetGradient(temp, false)) ReturnFalse; //--- for(int i = layers - 1; i >= 0; i--) { att_seq = cHUBs[i * 6 + 3]; att_cont = cHUBs[i * 6 + 4]; att_scen = cHUBs[i * 6 + 5]; concat = cHUBs[i * 6 + 6]; scal_and_shift = cHUBs[i * 6 + 7]; sequence = cSequence[i + 1]; context = cContext[i]; scen = cScenarios[i]; if(!sequence || !context || !scen || !INFNetBGUGrad(scal_and_shift, sequence, cSequence[i + 2], context, cContext[i + 1], scen, cScenarios[i + 1])) ReturnFalse; if(!concat || !concat.CalcHiddenGradients(scal_and_shift)) ReturnFalse; temp = concat.getGradient(); if(!concat.SetGradient(concat.getPrevOutput(), false) || !concat.CalcHiddenGradients(hub) || !SumAndNormalize(temp, concat.getGradient(), temp, iDimension, false, 0, 0, 0, 1) || !concat.SetGradient(temp, false)) ReturnFalse; hub = cHUBs[i * 6 + 2]; if(!att_seq || !att_cont || !att_scen || !DeConcat(att_seq.getGradient(), att_cont.getGradient(), att_scen.getGradient(), concat.getGradient(), iDimension, iDimension, iDimension, iSequenceVar + iScenariosUnits + 1)) ReturnFalse; //--- if(!hub || !sequence || !hub.CalcHiddenGradients(att_seq, sequence.getOutput(), sequence.getGradient(), (ENUM_ACTIVATION)sequence.Activation())) ReturnFalse; temp = hub.getGradient(); if(!hub.SetGradient(hub.getPrevOutput(), false) || !context || !hub.CalcHiddenGradients(att_cont, context.getOutput(), context.getGradient(), (ENUM_ACTIVATION)context.Activation()) || !SumAndNormalize(temp, hub.getGradient(), temp, iDimension, false, 0, 0, 0, 1)) ReturnFalse; if(!scen || !hub.CalcHiddenGradients(att_scen, scen.getOutput(), scen.getGradient(), (ENUM_ACTIVATION)scen.Activation()) || !SumAndNormalize(temp, hub.getGradient(), temp, iDimension, false, 0, 0, 0, 1) || !hub.SetGradient(temp, false)) ReturnFalse; } if(!cHUBs[0] || !cHUBs[1] || !DeConcat(cHUBs[0].getGradient(), cHUBs[1].getGradient(), scen.getPrevOutput(), hub.getGradient(), cHUBs[0].Neurons(), cHUBs[1].Neurons(), scen.Neurons(), 1)) ReturnFalse; if(!SumAndNormalize(scen.getGradient(), scen.getPrevOutput(), scen.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; temp = context.getGradient(); if(!context.SetGradient(context.getPrevOutput(), false) || !context.CalcHiddenGradients(cHUBs[1]) || !SumAndNormalize(temp, context.getGradient(), temp, iDimension, false, 0, 0, 0, 1) || !context.SetGradient(temp, false)) ReturnFalse; temp = sequence.getGradient(); if(!sequence.SetGradient(sequence.getPrevOutput(), false) || !sequence.CalcHiddenGradients(cHUBs[0]) || !SumAndNormalize(temp, sequence.getGradient(), temp, iDimension, false, 0, 0, 0, 1) || !sequence.SetGradient(temp, false)) ReturnFalse; sequence = cSequence[0]; if(!sequence || !sequence.CalcHiddenGradients(cSequence[1])) ReturnFalse; //--- if(!cPrepare.CalcHiddenGradients(scen)) ReturnFalse; if(!Concat(sequence.getGradient(), context.getGradient(), cPrepare.getPrevOutput(), sequence.Neurons(), context.Neurons(), 1)) ReturnFalse; if(cPrepare.Activation() != None) if(!DeActivation(cPrepare.getOutput(), cPrepare.getPrevOutput(), cPrepare.getPrevOutput(), cPrepare.Activation())) ReturnFalse; if(!SumAndNormalize(cPrepare.getGradient(), cPrepare.getPrevOutput(), cPrepare.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cPrepare.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronINFNetBlock trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSequence[1]; CNeuronBaseOCL* context = cContext[0]; if(!sequence || !sequence.UpdateInputWeights(cSequence[0])) ReturnFalse; //--- CNeuronBaseOCL* scen = cScenarios[0]; if(!scen || !scen.UpdateInputWeights(cPrepare.AsObject())) ReturnFalse; //--- HUBs CNeuronBaseOCL* hub = cHUBs[0]; if(!hub || !hub.UpdateInputWeights(sequence)) ReturnFalse; hub = cHUBs[1]; if(!hub || !hub.UpdateInputWeights(context)) ReturnFalse; hub = cHUBs[2]; //--- int layers = cContext.Total() - 1; for(int i = 0; i < layers; i++) { CNeuronBaseOCL* att_seq = cHUBs[i * 6 + 3]; if(!att_seq || !att_seq.UpdateInputWeights(hub, sequence.getOutput())) ReturnFalse; CNeuronBaseOCL* att_cont = cHUBs[i * 6 + 4]; if(!att_cont || !att_cont.UpdateInputWeights(hub, context.getOutput())) ReturnFalse; CNeuronBaseOCL* att_scen = cHUBs[i * 6 + 5]; if(!att_scen || !att_scen.UpdateInputWeights(hub, scen.getOutput())) ReturnFalse; CNeuronBaseOCL* concat = cHUBs[i * 6 + 6]; CNeuronBaseOCL* scal_and_shift = cHUBs[i * 6 + 7]; if(!scal_and_shift || !scal_and_shift.UpdateInputWeights(concat)) ReturnFalse; hub = cHUBs[i * 6 + 8]; if(!hub.UpdateInputWeights(concat)) ReturnFalse; } CNeuronBaseOCL* att_seq = cHUBs[layers * 6 + 3]; if(!att_seq || !att_seq.UpdateInputWeights(hub, sequence.getOutput())) ReturnFalse; CNeuronBaseOCL* att_cont = cHUBs[layers * 6 + 4]; if(!att_cont || !att_cont.UpdateInputWeights(hub, context.getOutput())) ReturnFalse; CNeuronBaseOCL* att_scen = cHUBs[layers * 6 + 5]; if(!att_scen || !att_scen.UpdateInputWeights(hub, scen.getOutput())) ReturnFalse; CNeuronBaseOCL* concat = cHUBs[layers * 6 + 6]; hub = cHUBs[layers * 6 + 7]; if(!hub.UpdateInputWeights(concat)) ReturnFalse; //--- if(!CNeuronPerTokenSwiGLU::updateInputWeights(hub)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronINFNetBlock state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::Save(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iSequenceUnits); dFileWriteUInt(file_handle, iSequenceVar); dFileWriteUInt(file_handle, iContextUnits); dFileWriteUInt(file_handle, iScenariosUnits); if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cSequence.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cHUBs.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronINFNetBlock state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::Load(const int file_handle) { if(!CNeuronPerTokenSwiGLU::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, iSequenceUnits); dFileReadUInt(file_handle, iSequenceVar); dFileReadUInt(file_handle, iContextUnits); dFileReadUInt(file_handle, iScenariosUnits); if(!LoadInsideLayer(file_handle, cPrepare.AsObject())) ReturnFalse; if(!cSequence.Load(file_handle)) ReturnFalse; if(!cContext.Load(file_handle)) ReturnFalse; if(!cScenarios.Load(file_handle)) ReturnFalse; if(!cHUBs.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronINFNetBlock execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::Clear(void) { if(!CNeuronPerTokenSwiGLU::Clear()) ReturnFalse; //--- if(!cPrepare.Clear()) ReturnFalse; if(!cSequence.ClearStates()) ReturnFalse; if(!cContext.ClearStates()) ReturnFalse; if(!cScenarios.ClearStates()) ReturnFalse; if(!cHUBs.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronINFNetBlock weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronINFNetBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenSwiGLU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronINFNetBlock* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cSequence, Source, tau); dWeightsUpdate(cContext, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cHUBs, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronINFNetBlock GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronINFNetBlock::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenSwiGLU::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cSequence.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cHUBs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronINFNetBlock layer. | //+------------------------------------------------------------------+ void CNeuronINFNetBlock::TrainMode(bool flag) { CNeuronPerTokenSwiGLU::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cSequence.TrainMode(bTrain); cContext.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cHUBs.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSequenceTokenizer layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint embed_size, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, embed_size, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cNorm.Init(0, index, OpenCL, dimension * units, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; uint units_out = MathMax(dimension - 1, 1); uint win_out = (embed_size / 2 + units_out - 1) / units_out; if(!cProj[0].Init(0, index, OpenCL, MathMin(3, dimension), 1, win_out, units_out, units, optimization, iBatch)) ReturnFalse; cProj[0].SetActivationFunction(SIGMOID); index++; if(!cProj[1].Init(embed_size, index, OpenCL, 2 * win_out * units_out, win_out * units_out, embed_size / 2, units - 1, 1, optimization, iBatch)) ReturnFalse; cProj[1].SetActivationFunction(SIGMOID); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSequenceTokenizer feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; CNeuronBaseOCL* prev = cNorm.AsObject(); for(uint i = 0; i < cProj.Size(); i++) { if(!cProj[i].FeedForward(prev)) ReturnFalse; prev = cProj[i].AsObject(); } if(!CNeuronBaseOCL::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSequenceTokenizer layer. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; int last_proj = (int)cProj.Size() - 1; if(!CNeuronBaseOCL::calcInputGradients(cProj[last_proj].AsObject())) ReturnFalse; for(int i = last_proj - 1; i >= 0; i--) if(!cProj[i].CalcHiddenGradients(cProj[i + 1].AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cProj[0].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSequenceTokenizer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; CNeuronBaseOCL* prev = cNorm.AsObject(); for(uint i = 0; i < cProj.Size(); i++) { if(!cProj[i].UpdateInputWeights(prev)) ReturnFalse; prev = cProj[i].AsObject(); } if(!CNeuronBaseOCL::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSequenceTokenizer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cProj.Size(); i++) if(!cProj[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSequenceTokenizer state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; for(uint i = 0; i < cProj.Size(); i++) if(!LoadInsideLayer(file_handle, cProj[i].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSequenceTokenizer weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSequenceTokenizer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSequenceTokenizer* Source = source; dWeightsUpdate(cNorm, Source, tau); for(uint i = 0; i < cProj.Size(); i++) dWeightsUpdate(cProj[i], Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSequenceTokenizer GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronSequenceTokenizer::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cNorm.SetOpenCL(OpenCL); for(uint i = 0; i < cProj.Size(); i++) cProj[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronSequenceTokenizer layer. | //+------------------------------------------------------------------+ void CNeuronSequenceTokenizer::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cNorm.TrainMode(bTrain); for(uint i = 0; i < cProj.Size(); i++) cProj[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDifferenceTokenizer layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint inpSize, uint tokenSize, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, tokenSize, optimization_type, batch)) ReturnFalse; //--- cFlow.Clear(); cFlow.SetOpenCL(OpenCL); //--- uint index = 0; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, inpSize, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, inpSize, optimization, iBatch) || !cFlow.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; uint window = 3; uint units = (inpSize + window - 1) / window; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(tokenSize, index, OpenCL, 2 * window, window, window, MathMax(units - 1, 1), 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDifferenceTokenizer feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; CNeuronBaseOCL* neuron = cFlow[0]; if(!neuron || NeuronOCL.Neurons() < neuron.Neurons() || SecondInput.Total() < neuron.Neurons()) ReturnFalse; //--- if(!Different(NeuronOCL.getOutput(), SecondInput, neuron.getOutput(), neuron.Neurons(), 0, 0, 0, 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { neuron = cFlow[i]; if(!neuron || !neuron.FeedForward(cFlow[i - 1])) ReturnFalse; } //--- if(!CNeuronBaseOCL::feedForward(neuron)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronDifferenceTokenizer layer. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; CNeuronBaseOCL* neuron = cFlow[0]; if(!neuron || NeuronOCL.Neurons() < neuron.Neurons() || SecondGradient.Total() < neuron.Neurons()) ReturnFalse; //--- if(!CNeuronBaseOCL::calcInputGradients(cFlow[-1])) ReturnFalse; //--- for(int i = cFlow.Total() - 2; i >= 0; i--) { neuron = cFlow[i]; if(!neuron || !neuron.CalcHiddenGradients(cFlow[i + 1])) ReturnFalse; } //--- if(!DifferentGrad(NeuronOCL.getGradient(), SecondGradient, neuron.getGradient(), neuron.Neurons(), 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDifferenceTokenizer trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { CNeuronBaseOCL* neuron = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { neuron = cFlow[i]; if(!neuron || !neuron.UpdateInputWeights(cFlow[i - 1])) ReturnFalse; } //--- if(!CNeuronBaseOCL::updateInputWeights(neuron)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronDifferenceTokenizer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDifferenceTokenizer state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronDifferenceTokenizer weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronDifferenceTokenizer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronDifferenceTokenizer* Source = source; dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDifferenceTokenizer GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronDifferenceTokenizer::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronDifferenceTokenizer layer. | //+------------------------------------------------------------------+ void CNeuronDifferenceTokenizer::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronReGENTADBackbone layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() < 2 || layers < 1 || units_s < 1) ReturnFalse; uint iSequenceDim = units_s * dimensions[0]; uint iContextDim = 0; for(uint i = 1; i < dimensions.Size(); i++) iContextDim += dimensions[i]; uint iScenariosUnits = dimensions.Size() + 1; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (iScenariosUnits + 4)*embed_size, optimization_type, batch)) ReturnFalse; activation = None; //--- cSeqToken.Clear(); cTransformer.Clear(); cRecurrent.Clear(); cSeqToken.SetOpenCL(OpenCL); cTransformer.SetOpenCL(OpenCL); cRecurrent.SetOpenCL(OpenCL); //--- Sequence tokenizer uint index = 0; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iSequenceDim, optimization, iBatch) || !cSeqToken.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronSequenceTokenizer* seq_tok = new CNeuronSequenceTokenizer(); if(!seq_tok || !seq_tok.Init(0, index, OpenCL, dimensions[0], units_s, embed_size, optimization, iBatch) || !cSeqToken.Add(seq_tok)) DeleteObjAndFalse(seq_tok); seq_tok.SetActivationFunction(SoftPlus); //--- Transformer index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iContextDim, optimization, iBatch) || !cTransformer.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iContextDim + seq_tok.Neurons(), optimization, iBatch) || !cTransformer.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); uint dims[]; if(ArrayCopy(dims, dimensions, 0, 0, dimensions.Size()) < (int)dimensions.Size()) ReturnFalse; dims[0] = seq_tok.Neurons(); index++; CNeuronINFNetBlock* transf = new CNeuronINFNetBlock(); if(!transf || !transf.Init(0, index, OpenCL, dims, 1, heads, stack_size, layers, embed_size, candidates, topK, optimization, iBatch) || !cTransformer.Add(transf)) DeleteObjAndFalse(transf); //--- Recurrent CNeuronLSTMOCL* lstm = NULL; uint size = seq_tok.Neurons(); for(uint i = 0; i < layers; i++) { lstm = new CNeuronLSTMOCL(); index++; if(!lstm || !lstm.Init(0, index, OpenCL, size, optimization, iBatch) || !lstm.SetInputs(size) || !cRecurrent.Add(lstm)) DeleteObjAndFalse(lstm); lstm.SetActivationFunction(TANH); } index++; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, lstm.Neurons(), optimization, iBatch) || !cRecurrent.Add(norm)) DeleteObjAndFalse(norm); //--- index++; if(!cDifference.Init(0, index, OpenCL, norm.Neurons(), embed_size, optimization, iBatch)) ReturnFalse; cDifference.SetActivationFunction(TANH); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronReGENTADBackbone feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSeqToken[0]; CNeuronBaseOCL* context = cTransformer[0]; if(!sequence || !context) ReturnFalse; uint iSequenceDim = sequence.Neurons(); uint iContextDim = context.Neurons(); if(!DeConcat(sequence.getOutput(), context.getOutput(), NeuronOCL.getOutput(), iSequenceDim, iContextDim, 1)) ReturnFalse; //--- Sequence tokenizer for(int i = 1; i < cSeqToken.Total(); i++) { sequence = cSeqToken[i]; if(!sequence || !sequence.FeedForward(cSeqToken[i - 1])) ReturnFalse; } iSequenceDim = sequence.Neurons(); //--- Transformer context = cTransformer[1]; if(!context || !Concat(sequence.getOutput(), cTransformer[0].getOutput(), context.getOutput(), iSequenceDim, iContextDim, 1)) ReturnFalse; for(int i = 2; i < cTransformer.Total(); i++) { context = cTransformer[i]; if(!context || !context.FeedForward(cTransformer[i - 1])) ReturnFalse; } //--- Recurrent sequence = cRecurrent[0]; if(!sequence || !sequence.FeedForward(cSeqToken[-1])) ReturnFalse; for(int i = 1; i < cRecurrent.Total(); i++) { sequence = cRecurrent[i]; if(!sequence || !sequence.FeedForward(cRecurrent[i - 1])) ReturnFalse; } //--- if(!cDifference.FeedForward(context, sequence.getOutput())) ReturnFalse; if(!Concat(context.getOutput(), sequence.getOutput(), cDifference.getOutput(), Output, context.Neurons(), sequence.Neurons(), cDifference.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronReGENTADBackbone layer. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cRecurrent[-1]; CNeuronBaseOCL* context = cTransformer[-1]; if(!sequence || !context) ReturnFalse; uint iSequenceDim = sequence.Neurons(); uint iContextDim = context.Neurons(); if(!DeConcat(context.getGradient(), sequence.getGradient(), cDifference.getGradient(), Gradient, iContextDim, iSequenceDim, cDifference.Neurons(), 1)) ReturnFalse; Deactivation(cDifference); Deactivation(context); Deactivation(sequence); CBufferFloat* temp = context.getGradient(); if(!context.SetGradient(context.getPrevOutput(), false) || !context.getGradient().Fill(0) || !sequence.getPrevOutput().Fill(0)) ReturnFalse; if(!context.CalcHiddenGradients(cDifference.AsObject(), sequence.getOutput(), sequence.getPrevOutput(), (ENUM_ACTIVATION)sequence.Activation())) ReturnFalse; if(!SumAndNormalize(temp, context.getGradient(), temp, 1, false, 0, 0, 0, 1) || !context.SetGradient(temp, false) || !SumAndNormalize(sequence.getGradient(), sequence.getPrevOutput(), sequence.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Recurrent for(int i = cRecurrent.Total() - 2; i >= 0; i--) { sequence = cRecurrent[i]; if(!sequence || !sequence.CalcHiddenGradients(cRecurrent[i + 1])) ReturnFalse; } //--- Transformer for(int i = cTransformer.Total() - 2; i >= 1; i--) { context = cTransformer[i]; if(!context || !context.CalcHiddenGradients(cTransformer[i + 1])) ReturnFalse; } context = cTransformer[0]; sequence = cSeqToken[-1]; if(!context || !sequence) ReturnFalse; iSequenceDim = sequence.Neurons(); iContextDim = context.Neurons(); if(!DeConcat(sequence.getPrevOutput(), context.getGradient(), cTransformer[1].getGradient(), iSequenceDim, iContextDim, 1)) ReturnFalse; //--- Sequence tokenizer if(!sequence.CalcHiddenGradients(cRecurrent[0])) ReturnFalse; if(sequence.Activation() != None) if(!DeActivation(sequence.getOutput(), sequence.getPrevOutput(), sequence.getPrevOutput(), sequence.Activation())) ReturnFalse; if(!SumAndNormalize(sequence.getGradient(), sequence.getPrevOutput(), sequence.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; for(int i = cSeqToken.Total() - 2; i >= 0; i--) { sequence = cSeqToken[i]; if(!sequence || !sequence.CalcHiddenGradients(cSeqToken[i + 1])) ReturnFalse; } iSequenceDim = sequence.Neurons(); //--- if(!Concat(sequence.getGradient(), context.getGradient(), NeuronOCL.getGradient(), iSequenceDim, iContextDim, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronReGENTADBackbone trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* sequence = cSeqToken[0]; CNeuronBaseOCL* context = cTransformer[0]; //--- Sequence tokenizer for(int i = 1; i < cSeqToken.Total(); i++) { sequence = cSeqToken[i]; if(!sequence || !sequence.UpdateInputWeights(cSeqToken[i - 1])) ReturnFalse; } //--- Transformer for(int i = 2; i < cTransformer.Total(); i++) { context = cTransformer[i]; if(!context || !context.UpdateInputWeights(cTransformer[i - 1])) ReturnFalse; } //--- Recurrent sequence = cRecurrent[0]; if(!sequence || !sequence.UpdateInputWeights(cSeqToken[-1])) ReturnFalse; for(int i = 1; i < cRecurrent.Total(); i++) { sequence = cRecurrent[i]; if(!sequence || !sequence.UpdateInputWeights(cRecurrent[i - 1])) ReturnFalse; } //--- if(!cDifference.UpdateInputWeights(context, sequence.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronReGENTADBackbone state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cSeqToken.Save(file_handle)) ReturnFalse; if(!cTransformer.Save(file_handle)) ReturnFalse; if(!cRecurrent.Save(file_handle)) ReturnFalse; if(!cDifference.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronReGENTADBackbone state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cSeqToken.Load(file_handle)) ReturnFalse; if(!cTransformer.Load(file_handle)) ReturnFalse; if(!cRecurrent.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cDifference.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronReGENTADBackbone weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronReGENTADBackbone* Source = source; dWeightsUpdate(cSeqToken, Source, tau); dWeightsUpdate(cTransformer, Source, tau); dWeightsUpdate(cRecurrent, Source, tau); dWeightsUpdate(cDifference, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronReGENTADBackbone execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTADBackbone::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cSeqToken.ClearStates()) ReturnFalse; if(!cTransformer.ClearStates()) ReturnFalse; if(!cRecurrent.ClearStates()) ReturnFalse; if(!cDifference.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronReGENTADBackbone GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronReGENTADBackbone::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cSeqToken.SetOpenCL(OpenCL); cTransformer.SetOpenCL(OpenCL); cRecurrent.SetOpenCL(OpenCL); cDifference.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronReGENTADBackbone layer. | //+------------------------------------------------------------------+ void CNeuronReGENTADBackbone::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cSeqToken.TrainMode(bTrain); cTransformer.TrainMode(bTrain); cRecurrent.TrainMode(bTrain); cDifference.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Returns the norm of the CNeuronReGENTADBackbone. | //+------------------------------------------------------------------+ CNeuronBatchNormOCL* CNeuronReGENTADBackbone::GetNorm(void) { CNeuronBatchNormOCL* result = NULL; for(int i = 0; (i < cSeqToken.Total() && !result); i++) { if(!cSeqToken[i] || cSeqToken[i].Type() != defNeuronSequenceTokenizer) continue; result = ((CNeuronSequenceTokenizer*)cSeqToken[i]).GetNorm(); break; } //--- return(result); } //+------------------------------------------------------------------+ //| Initializes the CNeuronReGENTADHead layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint units_in, uint dimension_out, uint units_out, CNeuronBatchNormOCL *norm, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRevINDenormOCL::Init(numOutputs, myIndex, open_cl, dimension_out * units_out, -1, NULL)) ReturnFalse; activation = None; if(!!norm) { if(!SetNorm(norm)) ReturnFalse; } iBatch = batch; //--- cFlow.Clear(); cFlow.SetOpenCL(OpenCL); //--- uint index = 0; CNeuronTransposeOCL* transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, dimension_in, optimization, iBatch) || !cFlow.Add(transp)) DeleteObjAndFalse(transp); transp.SetActivationFunction(None); index++; uint dim_hide = (MathMax(units_in, units_out) + 1) / 2; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, dim_hide, dimension_in, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dim_hide, dim_hide, units_out, dimension_in, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension_in, units_out, optimization, iBatch) || !cFlow.Add(transp)) DeleteObjAndFalse(transp); transp.SetActivationFunction(None); dim_hide = (MathMax(dimension_in, dimension_out) + 1) / 2; index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_in, dimension_in, dim_hide, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dim_hide, dim_hide, dimension_out, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_out, dimension_out, optimization, iBatch) || !cFlow.Add(transp)) DeleteObjAndFalse(transp); transp.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronReGENTADHead feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } return(CNeuronRevINDenormOCL::feedForward(prev)); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronReGENTADHead layer. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* next = (cFlow.Total() == 0 ? NeuronOCL : cFlow[-1]); if(!CNeuronRevINDenormOCL::calcInputGradients(next)) ReturnFalse; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(NeuronOCL != next) if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronReGENTADHead trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } return(CNeuronRevINDenormOCL::updateInputWeights(prev)); } //+------------------------------------------------------------------+ //| Saves the CNeuronReGENTADHead state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::Save(const int file_handle) { if(!CNeuronRevINDenormOCL::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronReGENTADHead state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::Load(const int file_handle) { if(!CNeuronRevINDenormOCL::Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronReGENTADHead weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronRevINDenormOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronReGENTADHead* Source = source; dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronReGENTADHead execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::Clear(void) { if(!CNeuronRevINDenormOCL::Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronReGENTADHead GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronReGENTADHead::SetOpenCL(COpenCLMy *obj) { CNeuronRevINDenormOCL::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronReGENTADHead layer. | //+------------------------------------------------------------------+ void CNeuronReGENTADHead::TrainMode(bool flag) { CNeuronRevINDenormOCL::TrainMode(flag); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Sets the norm of the CNeuronReGENTADHead. | //+------------------------------------------------------------------+ bool CNeuronReGENTADHead::SetNorm(CNeuronBatchNormOCL *norm) { if(!norm) ReturnFalse; switch(norm.Type()) { case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: break; default: ReturnFalse; } if(BatchOptions != norm.BatchOptions) { DeleteObj(BatchOptions); BatchOptions = norm.BatchOptions; } iBatchSize = norm.iBatchSize; iBatchCount = iBatchSize; iBatch = iBatchSize; for(int i = 0; i < cFlow.Total(); i++) cFlow[i].iBatch = iBatch; //--- return(!!BatchOptions); } //+------------------------------------------------------------------+ //| Initializes the CNeuronReGENTAD layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!cBackbone) cBackbone = CreateBackbone(); if(!cBackbone || !cBackbone.Init(0, 0, open_cl, dimensions, units_s, heads, stack_size, layers, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, cBackbone.Neurons() + 3 * embed_size, optimization_type, batch)) ReturnFalse; activation = None; //--- for(uint i = 0; i < cHeads.Size(); i++) { cHeads[i].Clear(); cHeads[i].SetOpenCL(OpenCL); } //--- CNeuronReGENTADHead* head = NULL; CNeuronDifferenceTokenizer* diff = NULL; uint backbone_units = cBackbone.Neurons() / embed_size; uint index = 1; for(int i = 0; i < 2; i++) { head = new CNeuronReGENTADHead(); if(!head || !head.Init(0, index, OpenCL, embed_size, backbone_units, dimensions[0], units_s, cBackbone.GetNorm(), optimization, iBatch) || !cHeads[i].Add(head)) DeleteObjAndFalse(head); index++; diff = new CNeuronDifferenceTokenizer(); if(!diff || !diff.Init(0, index, OpenCL, head.Neurons(), embed_size, optimization, iBatch) || !cHeads[i].Add(diff)) DeleteObjAndFalse(diff); diff.SetActivationFunction(TANH); index++; } //--- CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, cBackbone.Neurons() + diff.Neurons(), optimization, iBatch) || !cHeads[2].Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; head = new CNeuronReGENTADHead(); if(!head || !head.Init(0, index, OpenCL, embed_size, backbone_units + 1, dimensions[0], units_s, cBackbone.GetNorm(), optimization, iBatch) || !cHeads[2].Add(head)) DeleteObjAndFalse(head); index++; diff = new CNeuronDifferenceTokenizer(); if(!diff || !diff.Init(0, index, OpenCL, head.Neurons(), embed_size, optimization, iBatch) || !cHeads[2].Add(diff)) DeleteObjAndFalse(diff); diff.SetActivationFunction(TANH); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronReGENTAD feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cBackbone) ReturnFalse; //--- Forecats 1 CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* prev = cBackbone.AsObject(); for(int i = 0; i < cHeads[1].Total(); i++) { curr = cHeads[1][i]; if(!curr || !curr.FeedForward(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- Forecats 2 curr = cHeads[2][0]; if(!curr || !Concat(cBackbone.getOutput(), prev.getOutput(), curr.getOutput(), cBackbone.Neurons(), prev.Neurons(), 1)) ReturnFalse; prev = curr; for(int i = 1; i < cHeads[2].Total(); i++) { curr = cHeads[2][i]; if(!curr || !curr.FeedForward(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- Backprop if(bTrain) { //--- Forecast 2 curr = cHeads[2][-2]; float error = 0; if(!curr || !curr.calcOutputGradients(NeuronOCL.getOutput(), error)) ReturnFalse; for(int i = cHeads[2].Total() - 3; i >= 0; i--) { curr = cHeads[2][i]; if(!curr || !curr.CalcHiddenGradients(cHeads[2][i + 1])) ReturnFalse; } for(int i = 1; i < cHeads[2].Total() - 1; i++) { curr = cHeads[2][i]; if(!curr || !curr.UpdateInputWeights(cHeads[2][i - 1])) ReturnFalse; } //--- Forecast 1 curr = cHeads[2][0]; prev = cHeads[1][-1]; if(!DeConcat(cBackbone.getPrevOutput(), prev.getGradient(), curr.getGradient(), cBackbone.Neurons(), prev.Neurons(), 1)) ReturnFalse; Deactivation(prev); curr = cHeads[1][-2]; if(!curr || !curr.CalcHiddenGradients(prev, NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; CBufferFloat* temp = curr.getGradient(); if(!curr.SetGradient(curr.getPrevOutput(), false) || !curr.calcOutputGradients(NeuronOCL.getOutput(), error) || !SumAndNormalize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; for(int i = cHeads[1].Total() - 3; i >= 0; i--) { curr = cHeads[1][i]; if(!curr || !curr.CalcHiddenGradients(cHeads[1][i + 1], NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; } if(!cBackbone.CalcHiddenGradients(curr) || !SumAndNormalize(cBackbone.getGradient(), cBackbone.getPrevOutput(), cBackbone.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cBackbone.AsObject())) ReturnFalse; if(!cBackbone.UpdateInputWeights(NeuronOCL)) ReturnFalse; prev = cBackbone.AsObject(); for(int i = 0; i < cHeads[1].Total(); i++) { curr = cHeads[1][i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } } //--- Backbone if(!cBackbone.FeedForward(NeuronOCL)) ReturnFalse; //--- Recover prev = cBackbone.AsObject(); for(int i = 0; i < cHeads[0].Total(); i++) { curr = cHeads[0][i]; if(!curr || !curr.FeedForward(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- if(!Concat(cBackbone.getOutput(), cHeads[0][-1].getOutput(), cHeads[1][-1].getOutput(), cHeads[2][-1].getOutput(), Output, cBackbone.Neurons(), cHeads[0][-1].Neurons(), cHeads[1][-1].Neurons(), cHeads[2][-1].Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronReGENTAD | //| layer. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cBackbone) ReturnFalse; //--- if(!cHeads[0][-1] || !cHeads[1][-1] || !cHeads[2][-1] || !DeConcat(cBackbone.getPrevOutput(), cHeads[0][-1].getGradient(), cHeads[1][-1].getGradient(), cHeads[2][-1].getGradient(), Gradient, cBackbone.Neurons(), cHeads[0][-1].Neurons(), cHeads[1][-1].Neurons(), cHeads[2][-1].Neurons(), 1)) ReturnFalse; //--- Forecast 2 CNeuronBaseOCL* curr = cHeads[2][-1]; Deactivation(curr) for(int i = cHeads[2].Total() - 2; i >= 0; i--) { curr = cHeads[2][i]; if(!curr || !curr.CalcHiddenGradients(cHeads[2][i + 1], NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; } //--- Forecast 1 curr = cHeads[2][0]; CNeuronBaseOCL* prev = cHeads[1][-1]; if(!DeConcat(cBackbone.getPrevOutput(), prev.getPrevOutput(), curr.getGradient(), cBackbone.Neurons(), prev.Neurons(), 1) || !SumAndNormalize(prev.getGradient(), prev.getPrevOutput(), prev.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(prev); for(int i = cHeads[1].Total() - 2; i >= 0; i--) { curr = cHeads[1][i]; if(!curr || !curr.CalcHiddenGradients(cHeads[1][i + 1], NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; } //--- Recovery prev = cHeads[0][-1]; Deactivation(prev); curr = cHeads[0][-2]; if(!curr || !curr.CalcHiddenGradients(prev, NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; CBufferFloat* temp = curr.getGradient(); float error = 0; if(!curr.SetGradient(curr.getPrevOutput(), false) || !curr.calcOutputGradients(NeuronOCL.getOutput(), error) || !SumAndNormalize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; for(int i = cHeads[0].Total() - 3; i >= 0; i--) { curr = cHeads[0][i]; if(!curr || !curr.CalcHiddenGradients(cHeads[0][i + 1], NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; } if(!cBackbone.CalcHiddenGradients(curr) || !SumAndNormalize(cBackbone.getGradient(), cBackbone.getPrevOutput(), cBackbone.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cBackbone.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronReGENTAD trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cBackbone) ReturnFalse; //--- Forecast 1 CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* prev = cBackbone.AsObject(); for(int i = 0; i < cHeads[1].Total(); i++) { curr = cHeads[1][i]; if(!curr || !curr.UpdateInputWeights(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- Forecast 2 prev = cHeads[2][0]; for(int i = 1; i < cHeads[2].Total(); i++) { curr = cHeads[2][i]; if(!curr || !curr.UpdateInputWeights(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- Backbone if(!cBackbone.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- Recover prev = cBackbone.AsObject(); for(int i = 0; i < cHeads[0].Total(); i++) { curr = cHeads[0][i]; if(!curr || !curr.UpdateInputWeights(prev, NeuronOCL.getOutput())) ReturnFalse; prev = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronReGENTAD state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cBackbone || !cBackbone.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cHeads.Size(); i++) if(!cHeads[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronReGENTAD state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cBackbone) cBackbone = CreateBackbone(); if(!cBackbone) ReturnFalse; if(!LoadInsideLayer(file_handle, cBackbone.AsObject())) ReturnFalse; for(uint i = 0; i < cHeads.Size(); i++) { if(!cHeads[i].Load(file_handle)) ReturnFalse; for(int n = 0; n < cHeads[i].Total(); n++) if(cHeads[i][n].Type() == defNeuronReGENTADHead) { CNeuronReGENTADHead* head = cHeads[i][n]; if(!head.SetNorm(cBackbone.GetNorm())) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronReGENTAD weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronReGENTAD* Source = source; if(!Source || !cBackbone || !Source.cBackbone || !cBackbone.WeightsUpdate(Source.cBackbone.AsObject(), tau)) ReturnFalse; for(uint i = 0; i < cHeads.Size(); i++) dWeightsUpdate(cHeads[i], Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronReGENTAD execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronReGENTAD::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cBackbone || !cBackbone.Clear()) ReturnFalse; for(uint i = 0; i < cHeads.Size(); i++) if(!cHeads[i].ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronReGENTAD GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronReGENTAD::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!cBackbone) cBackbone.SetOpenCL(OpenCL); for(uint i = 0; i < cHeads.Size(); i++) cHeads[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronReGENTAD layer. | //+------------------------------------------------------------------+ void CNeuronReGENTAD::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); if(!!cBackbone) cBackbone.TrainMode(bTrain); for(uint i = 0; i < cHeads.Size(); i++) cHeads[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGDformerBackbone layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGDformerBackbone::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() < 2 || layers < 1 || units_s < 1 || heads < 1 || embed_size < 1 || embed_size % heads != 0) ReturnFalse; uint iSequenceDim = units_s * dimensions[0]; uint iDCAlayers = 3; uint iContextUnits = dimensions.Size() - 1; uint iGDSequenceUnits = 1 + iDCAlayers; uint iOutputUnits = 3 * iGDSequenceUnits + iContextUnits + 3; uint iContextDim = 0; for(uint i = 1; i < dimensions.Size(); i++) iContextDim += dimensions[i]; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, iOutputUnits * embed_size, optimization_type, batch)) ReturnFalse; activation = None; //--- cSeqToken.Clear(); cTransformer.Clear(); cRecurrent.Clear(); cSeqToken.SetOpenCL(OpenCL); cTransformer.SetOpenCL(OpenCL); cRecurrent.SetOpenCL(OpenCL); //--- Sequence tokenizer uint index = 0; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iSequenceDim, optimization, iBatch) || !cSeqToken.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronSequenceTokenizer* seq_tok = new CNeuronSequenceTokenizer(); if(!seq_tok || !seq_tok.Init(0, index, OpenCL, dimensions[0], units_s, embed_size, optimization, iBatch) || !cSeqToken.Add(seq_tok)) DeleteObjAndFalse(seq_tok); seq_tok.SetActivationFunction(SoftPlus); CNeuronDictionaryCrossAtt* dca = NULL; for(uint i = 0; i < iDCAlayers; i++) { index++; dca = new CNeuronDictionaryCrossAtt(); if(!dca || !dca.Init(0, index, OpenCL, embed_size, 1, heads, stack_size, optimization, iBatch) || !cSeqToken.Add(dca)) DeleteObjAndFalse(dca); } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 4 * embed_size, optimization, iBatch) || !cSeqToken.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); //--- Transformer index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iContextDim, optimization, iBatch) || !cTransformer.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, iContextDim + cSeqToken[-1].Neurons(), optimization, iBatch) || !cTransformer.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); uint dims[]; uint sequence_units = cSeqToken[-1].Neurons() / embed_size; if(sequence_units != iGDSequenceUnits || ArrayResize(dims, sequence_units + iContextUnits) < int(sequence_units + iContextUnits)) ReturnFalse; ArrayFill(dims, 0, sequence_units, embed_size); ArrayCopy(dims, dimensions, sequence_units, 1); index++; CNeuronINFNetBlock* transf = new CNeuronINFNetBlock(); if(!transf || !transf.Init(0, index, OpenCL, dims, sequence_units, heads, stack_size, layers, embed_size, candidates, topK, optimization, iBatch) || !cTransformer.Add(transf)) DeleteObjAndFalse(transf); //--- Recurrent CNeuronLSTMOCL* lstm = NULL; uint size = cSeqToken[-1].Neurons(); for(uint i = 0; i < layers; i++) { lstm = new CNeuronLSTMOCL(); index++; if(!lstm || !lstm.Init(0, index, OpenCL, size, optimization, iBatch) || !lstm.SetInputs(size) || !cRecurrent.Add(lstm)) DeleteObjAndFalse(lstm); lstm.SetActivationFunction(TANH); } index++; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, lstm.Neurons(), optimization, iBatch) || !cRecurrent.Add(norm)) DeleteObjAndFalse(norm); //--- index++; if(!cDifference.Init(0, index, OpenCL, norm.Neurons(), embed_size, optimization, iBatch)) ReturnFalse; cDifference.SetActivationFunction(TANH); if(Neurons() != cTransformer[-1].Neurons() + cSeqToken[-1].Neurons() + cDifference.Neurons()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGDformerBackbone feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronGDformerBackbone::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cSeqToken[0]; CNeuronBaseOCL* context = cTransformer[0]; if(!sequence || !context) ReturnFalse; uint iSequenceDim = sequence.Neurons(); uint iContextDim = context.Neurons(); if(!DeConcat(sequence.getOutput(), context.getOutput(), NeuronOCL.getOutput(), iSequenceDim, iContextDim, 1)) ReturnFalse; //--- Sequence tokenizer CNeuronBaseOCL *sims[3] = {NULL}; uint sims_pos = 0; for(int i = 1; i < cSeqToken.Total() - 1; i++) { sequence = cSeqToken[i]; if(!sequence || !sequence.FeedForward(cSeqToken[i - 1])) ReturnFalse; if(sequence.Type() == defNeuronDictionaryCrossAtt) { if(sims_pos >= 3) ReturnFalse; CNeuronDictionaryCrossAtt* temp = sequence; sims[sims_pos] = temp.GetSimilarity(); sims_pos++; } } iSequenceDim = sequence.Neurons(); sequence = cSeqToken[-1]; if(sims_pos != 3 || !sims[0] || !sims[1] || !sims[2]) ReturnFalse; if(!sequence || !Concat(cSeqToken[-2].getOutput(), sims[0].getOutput(), sims[1].getOutput(), sims[2].getOutput(), sequence.getOutput(), iSequenceDim, iSequenceDim, iSequenceDim, iSequenceDim, 1)) ReturnFalse; iSequenceDim = sequence.Neurons(); //--- Transformer context = cTransformer[1]; if(!context || !Concat(sequence.getOutput(), cTransformer[0].getOutput(), context.getOutput(), iSequenceDim, iContextDim, 1)) ReturnFalse; for(int i = 2; i < cTransformer.Total(); i++) { context = cTransformer[i]; if(!context || !context.FeedForward(cTransformer[i - 1])) ReturnFalse; } //--- Recurrent sequence = cRecurrent[0]; if(!sequence || !sequence.FeedForward(cSeqToken[-1])) ReturnFalse; for(int i = 1; i < cRecurrent.Total(); i++) { sequence = cRecurrent[i]; if(!sequence || !sequence.FeedForward(cRecurrent[i - 1])) ReturnFalse; } //--- if(!cDifference.FeedForward(context, sequence.getOutput())) ReturnFalse; if(!Concat(context.getOutput(), sequence.getOutput(), cDifference.getOutput(), Output, context.Neurons(), sequence.Neurons(), cDifference.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronGDformerBackbone layer. | //+------------------------------------------------------------------+ bool CNeuronGDformerBackbone::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* sequence = cRecurrent[-1]; CNeuronBaseOCL* context = cTransformer[-1]; if(!sequence || !context) ReturnFalse; uint iSequenceDim = sequence.Neurons(); uint iContextDim = context.Neurons(); if(!DeConcat(context.getGradient(), sequence.getGradient(), cDifference.getGradient(), Gradient, iContextDim, iSequenceDim, cDifference.Neurons(), 1)) ReturnFalse; Deactivation(cDifference); Deactivation(context); Deactivation(sequence); CBufferFloat* temp = context.getGradient(); if(!context.SetGradient(context.getPrevOutput(), false)) ReturnFalse; if(!context.CalcHiddenGradients(cDifference.AsObject(), sequence.getOutput(), sequence.getPrevOutput(), (ENUM_ACTIVATION)sequence.Activation())) ReturnFalse; if(!SumAndNormalize(temp, context.getGradient(), temp, 1, false, 0, 0, 0, 1) || !context.SetGradient(temp, false) || !SumAndNormalize(sequence.getGradient(), sequence.getPrevOutput(), sequence.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Recurrent for(int i = cRecurrent.Total() - 2; i >= 0; i--) { sequence = cRecurrent[i]; if(!sequence || !sequence.CalcHiddenGradients(cRecurrent[i + 1])) ReturnFalse; } //--- Transformer for(int i = cTransformer.Total() - 2; i >= 1; i--) { context = cTransformer[i]; if(!context || !context.CalcHiddenGradients(cTransformer[i + 1])) ReturnFalse; } context = cTransformer[0]; sequence = cSeqToken[-1]; if(!context || !sequence) ReturnFalse; iSequenceDim = sequence.Neurons(); iContextDim = context.Neurons(); if(!DeConcat(sequence.getPrevOutput(), context.getGradient(), cTransformer[1].getGradient(), iSequenceDim, iContextDim, 1)) ReturnFalse; //--- Sequence tokenizer if(!sequence.CalcHiddenGradients(cRecurrent[0])) ReturnFalse; if(sequence.Activation() != None) if(!DeActivation(sequence.getOutput(), sequence.getPrevOutput(), sequence.getPrevOutput(), sequence.Activation())) ReturnFalse; if(!SumAndNormalize(sequence.getGradient(), sequence.getPrevOutput(), sequence.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; CNeuronBaseOCL* sims[3] = {NULL}; uint sims_pos = 0; for(int i = 1; i < cSeqToken.Total() - 1; i++) { if(!cSeqToken[i] || cSeqToken[i].Type() != defNeuronDictionaryCrossAtt) continue; if(sims_pos >= 3) ReturnFalse; CNeuronDictionaryCrossAtt* temp = cSeqToken[i]; sims[sims_pos] = temp.GetSimilarity(); sims_pos++; } if(sims_pos != 3 || !sims[0] || !sims[1] || !sims[2]) ReturnFalse; iSequenceDim = sims[0].Neurons(); if(!cSeqToken[-2] || !DeConcat(cSeqToken[-2].getGradient(), sims[0].getGradient(), sims[1].getGradient(), sims[2].getGradient(), sequence.getGradient(), iSequenceDim, iSequenceDim, iSequenceDim, iSequenceDim, 1)) ReturnFalse; for(uint i = 0; i < sims_pos; i++) Deactivation(sims[i]); for(int i = cSeqToken.Total() - 3; i >= 0; i--) { sequence = cSeqToken[i]; if(!sequence || !sequence.CalcHiddenGradients(cSeqToken[i + 1])) ReturnFalse; } iSequenceDim = sequence.Neurons(); //--- if(!Concat(sequence.getGradient(), context.getGradient(), NeuronOCL.getGradient(), iSequenceDim, iContextDim, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGDformerBackbone trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronGDformerBackbone::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* sequence = cSeqToken[0]; CNeuronBaseOCL* context = cTransformer[0]; //--- Sequence tokenizer for(int i = 1; i < cSeqToken.Total() - 1; i++) { sequence = cSeqToken[i]; if(!sequence || !sequence.UpdateInputWeights(cSeqToken[i - 1])) ReturnFalse; } //--- Transformer for(int i = 2; i < cTransformer.Total(); i++) { context = cTransformer[i]; if(!context || !context.UpdateInputWeights(cTransformer[i - 1])) ReturnFalse; } //--- Recurrent sequence = cRecurrent[0]; if(!sequence || !sequence.UpdateInputWeights(cSeqToken[-1])) ReturnFalse; for(int i = 1; i < cRecurrent.Total(); i++) { sequence = cRecurrent[i]; if(!sequence || !sequence.UpdateInputWeights(cRecurrent[i - 1])) ReturnFalse; } //--- if(!cDifference.UpdateInputWeights(context, sequence.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMomADTTM layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint latent_dim, uint candidates, uint decisions, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(latent_dim == 0 || candidates == 0 || decisions == 0 || topK == 0 || topK > candidates) ReturnFalse; //--- iCandidates = candidates; iDecisions = decisions; iDimension = latent_dim; iTopK = topK; //--- if(!CNeuronBaseOCL::Init(0, myIndex, open_cl, latent_dim, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- if(!cMeanDistance.Init(0, 0, OpenCL, iCandidates, optimization, iBatch)) ReturnFalse; cMeanDistance.SetActivationFunction(None); if(!cScores.Init(0, 1, OpenCL, 1, iCandidates, iTopK, optimization, iBatch)) ReturnFalse; cScores.SetActivationFunction(None); //--- int distance_total = int(iCandidates * iDecisions); int top_total = int(distance_total * iTopK); int candidates_total = int(iCandidates * iDimension); if(!cDistance.Init(0, 2, OpenCL, distance_total, optimization, iBatch) || !cCandidatesGradient.BufferInit(candidates_total, 0)) ReturnFalse; cDistance.SetActivationFunction(None); if(!cCandidatesGradient.BufferCreate(OpenCL)) ReturnFalse; //--- CBufferFloat* dist_index = cDistance.getPrevOutput(); dist_index.BufferFree(); if(!dist_index.BufferInit(top_total, 0) || !dist_index.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the mean scores step of the CNeuronMomADTTM layer. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::MeanScores(void) { if(!OpenCL || cMeanDistance.getOutputIndex() < 0) ReturnFalse; if(cDistance.Neurons() != int(iCandidates * iDecisions) || cMeanDistance.Neurons() != int(iCandidates)) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {iCandidates}; const int kernel = def_k_MomADTTMMeanScores; ResetLastError(); setBuffer(kernel, def_k_madttm_ms_distance, cDistance.getOutputIndex()) setBuffer(kernel, def_k_madttm_ms_output, cMeanDistance.getOutputIndex()) setArgument(kernel, def_k_madttm_ms_decisions, int(iDecisions)) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cMeanDistance.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the mean scores gradient step of the CNeuronMomADTTM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::MeanScoresGradient(void) { if(!OpenCL || cMeanDistance.getGradientIndex() < 0) ReturnFalse; if(cMeanDistance.Neurons() != int(iCandidates) || cDistance.Neurons() != int(iCandidates * iDecisions)) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iCandidates, iDecisions}; const int kernel = def_k_MomADTTMMeanScoresGrad; ResetLastError(); setBuffer(kernel, def_k_madttm_msg_scores_gr, cMeanDistance.getGradientIndex()) setBuffer(kernel, def_k_madttm_msg_distance_gr, cDistance.getGradientIndex()) setArgument(kernel, def_k_madttm_msg_decisions, int(iDecisions)) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cDistance.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMomADTTM feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() != int(iCandidates * iDimension) || Neurons() != int(iDimension) || SecondInput.Total() != int(iDecisions * iDimension)) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SparseSoftmaxHausdorff(NeuronOCL.getOutput(), SecondInput, cDistance.getOutput(), cDistance.getPrevOutput(), int(iDimension), int(iTopK))) ReturnFalse; if(!MeanScores()) ReturnFalse; if(!cScores.FeedForward(cMeanDistance.AsObject())) ReturnFalse; if(!SparseMatMul(cScores.GetIndexes(), cScores.getOutput(), NeuronOCL.getOutput(), Output, 1, int(iTopK), int(iCandidates), int(iDimension))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMomADTTM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() != int(iCandidates * iDimension) || Gradient.Total() != int(iDimension) || SecondInput.Total() != int(iDecisions * iDimension)) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; if(!!SecondGradient) { if(SecondGradient.Total() != int(iDecisions * iDimension)) ReturnFalse; if(SecondGradient.GetIndex() < 0 && !SecondGradient.BufferCreate(OpenCL)) ReturnFalse; } //--- weighted latent output branch if(!cCandidatesGradient.Fill(0) || !cScores.getGradient().Fill(0)) ReturnFalse; if(!SparseMatMulGrad(cScores.GetIndexes(), cScores.getOutput(), cScores.getGradient(), NeuronOCL.getOutput(), GetPointer(cCandidatesGradient), Gradient, 1, int(iTopK), int(iCandidates), int(iDimension))) ReturnFalse; //--- sparse candidate-score branch if(!cMeanDistance.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!MeanScoresGradient()) ReturnFalse; //--- Hausdorff branch CBufferFloat *second_grad = SecondGradient; CBufferFloat tempSecondGradient; if(!second_grad) { if(!tempSecondGradient.BufferInit(int(iDecisions * iDimension), 0) || !tempSecondGradient.BufferCreate(OpenCL)) ReturnFalse; second_grad = GetPointer(tempSecondGradient); } bool result = SparseSoftmaxHausdorffGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), SecondInput, second_grad, cDistance.getGradient(), cDistance.getPrevOutput(), int(iDimension), int(iTopK)); if(!SecondGradient) tempSecondGradient.BufferFree(); if(!result) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), GetPointer(cCandidatesGradient), NeuronOCL.getGradient(), int(iDimension), false, 0, 0, 0, 1.0f)) ReturnFalse; Deactivation(NeuronOCL); if(!!SecondGradient && SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMomADTTM state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iCandidates) dFileWriteUInt(file_handle, iDecisions) dFileWriteUInt(file_handle, iDimension) dFileWriteUInt(file_handle, iTopK) if(!cMeanDistance.Save(file_handle) || !cScores.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Append trainable TTM layers without candidate runtime state. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronBaseOCL::AppendParameterFingerprint(fingerprint) || !cMeanDistance.AppendParameterFingerprint(fingerprint) || !cScores.AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMomADTTM state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iCandidates) dFileReadUInt(file_handle, iDecisions) dFileReadUInt(file_handle, iDimension) dFileReadUInt(file_handle, iTopK) if(iDimension == 0 || iDimension > 64 || iCandidates == 0 || iDecisions == 0 || iTopK == 0 || iTopK > iCandidates || iTopK > iDimension) ReturnFalse; if(!LoadInsideLayer(file_handle, cMeanDistance.AsObject()) || !LoadInsideLayer(file_handle, cScores.AsObject())) ReturnFalse; if(Neurons() != int(iDimension) || cMeanDistance.Neurons() != int(iCandidates) || cScores.Neurons() != int(iTopK)) ReturnFalse; //--- int distance_total = int(iCandidates * iDecisions); int top_total = int(distance_total * iTopK); int candidates_total = int(iCandidates * iDimension); if(!cDistance.Init(0, 2, OpenCL, distance_total, optimization, iBatch) || !cCandidatesGradient.BufferInit(candidates_total, 0)) ReturnFalse; cDistance.SetActivationFunction(None); if(!cCandidatesGradient.BufferCreate(OpenCL)) ReturnFalse; //--- CBufferFloat* dist_index = cDistance.getPrevOutput(); dist_index.BufferFree(); if(!dist_index.BufferInit(top_total, 0) || !dist_index.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Reports checked-transfer support for the MomADTTM ownership | //| tree. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::SupportsOpenCLChecked(void) const { return(Type() == defNeuronMomADTTM && cMeanDistance.SupportsOpenCLChecked() && cScores.SupportsOpenCLChecked() && cDistance.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers MomADTTM caches through their checked ownership tree. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and complete ownership tree before mutation. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or ownership tree"); //--- Preserve the direct cache before moving nested owners. if(cCandidatesGradient.Total() > 0 && cCandidatesGradient.GetIndex() >= 0 && cCandidatesGradient.GetOpenCL() != obj && !cCandidatesGradient.BufferRead()) ReturnFalseEx("candidate gradient read"); //--- Transfer complete nested caches through their checked contracts. if(!cMeanDistance.SetOpenCLChecked(obj)) ReturnFalseEx("mean distance transfer"); if(!cScores.SetOpenCLChecked(obj)) ReturnFalseEx("scores transfer"); if(!cDistance.SetOpenCLChecked(obj)) ReturnFalseEx("distance transfer"); //--- Preserve already transferred device data on retry. if(cCandidatesGradient.Total() > 0 && (cCandidatesGradient.GetOpenCL() != obj || cCandidatesGradient.GetIndex() < 0) && !cCandidatesGradient.BufferCreate(obj)) ReturnFalseEx("candidate gradient create"); //--- Publish the inherited context after all nested owners are ready. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("base context transfer"); //--- Finalize without resetting configuration or cached state. return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMomADTTM GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMomADTTM::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cMeanDistance.SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); if(cDistance.Neurons() > 0) cDistance.SetOpenCL(OpenCL); if(cCandidatesGradient.Total() > 0) cCandidatesGradient.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMomADTTM layer. | //+------------------------------------------------------------------+ void CNeuronMomADTTM::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cMeanDistance.TrainMode(bTrain); cScores.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Clears the CNeuronMomADTTM execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMomADTTM::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cMeanDistance.Clear() || !cScores.Clear()) ReturnFalse; if(!cDistance.Clear()) ReturnFalse; if(cCandidatesGradient.Total() > 0 && !cCandidatesGradient.Fill(0)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMomADMPI layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint latent_dim, uint candidates, uint market_dim, uint market_units, uint stack_size, uint topK, uint heads, uint key_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(latent_dim == 0 || candidates == 0 || market_dim == 0 || market_units == 0 || stack_size == 0 || topK == 0 || topK > candidates || heads == 0 || key_dimension == 0) ReturnFalse; //--- final MPI refine: Q_hist attends to market state if(!CNeuronCrossAttention::Init(numOutputs, myIndex, open_cl, latent_dim, key_dimension, heads, 1, market_dim, market_units, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- history stack and internal MPI components if(!cStack.Init(0, 0, OpenCL, stack_size, latent_dim, 1, optimization, iBatch)) ReturnFalse; if(!cTTM.Init(0, 1, OpenCL, latent_dim, candidates, stack_size, topK, optimization, iBatch)) ReturnFalse; if(!cPolicyMarket.Init(0, 2, OpenCL, latent_dim, key_dimension, heads, stack_size, market_dim, market_units, optimization, iBatch)) ReturnFalse; if(!cTTMHistory.Init(0, 3, OpenCL, latent_dim, key_dimension, heads, 1, latent_dim, stack_size, optimization, iBatch)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Runs the CNeuronMomADMPI feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() != int(iWindow_K * iUnits_K)) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- push the previous MPI scenario before overwriting Output if(!cStack.FeedForward(GetPointer(this))) ReturnFalse; //--- TTM builds the weighted scenario from the strategy pool and recent scenarios if(!cTTM.FeedForward(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- D_cur = actions stack attends to market if(!cPolicyMarket.FeedForward(cStack.AsObject(), SecondInput)) ReturnFalse; //--- Q_hist = TTM scenario attends to the current action-aware history if(!cTTMHistory.FeedForward(cTTM.AsObject(), cPolicyMarket.getOutput())) ReturnFalse; //--- parent cross-attention is the final MPI market refine block if(!CNeuronCrossAttention::feedForward(cTTMHistory.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMomADMPI | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() != int(iWindow_K * iUnits_K)) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- market gradient target for the final parent refine branch CBufferFloat tempMarketGradient; CBufferFloat *market_gradient = SecondGradient; if(!!market_gradient) { if(market_gradient.Total() != SecondInput.Total()) ReturnFalse; if(market_gradient.GetIndex() < 0 && !market_gradient.BufferCreate(OpenCL)) ReturnFalse; if(!market_gradient.Fill(0)) ReturnFalse; } else { if(!tempMarketGradient.BufferInit(SecondInput.Total(), 0) || !tempMarketGradient.BufferCreate(OpenCL)) ReturnFalse; market_gradient = GetPointer(tempMarketGradient); } //--- final MPI refine gradients: Q_hist and market context if(!CNeuronCrossAttention::calcInputGradients(cTTMHistory.AsObject(), SecondInput, market_gradient, None)) ReturnFalse; //--- TTM-history gradients: Q_ttm and D_cur if(!cTTM.CalcHiddenGradients(cTTMHistory.AsObject(), cPolicyMarket.getOutput(), cPolicyMarket.getGradient(), None)) ReturnFalse; //--- TTM gradients to the strategy pool. The stack branch is intentionally stop-gradient. if(!NeuronOCL.CalcHiddenGradients(cTTM.AsObject(), cStack.getOutput(), NULL, None)) ReturnFalse; //--- policy-market gradients are needed for its weights and optional market input gradient CBufferFloat policyMarketGradient; if(!policyMarketGradient.BufferInit(SecondInput.Total(), 0) || !policyMarketGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cStack.CalcHiddenGradients(cPolicyMarket.AsObject(), SecondInput, GetPointer(policyMarketGradient), None)) ReturnFalse; if(cStack.getGradient().Total() > 0 && !cStack.getGradient().Fill(0)) ReturnFalse; //--- expose summed market gradient only when the caller provided a target buffer if(!!SecondGradient) { if(!SumAndNormalize(market_gradient, GetPointer(policyMarketGradient), market_gradient, int(iWindow_K), false, 0, 0, 0, 1.0f)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMomADMPI trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() != int(iWindow_K * iUnits_K)) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- if(!CNeuronCrossAttention::updateInputWeights(cTTMHistory.AsObject(), SecondInput)) ReturnFalse; if(!cTTMHistory.UpdateInputWeights(cTTM.AsObject(), cPolicyMarket.getOutput())) ReturnFalse; if(!cPolicyMarket.UpdateInputWeights(cStack.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMomADMPI state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::Save(const int file_handle) { if(!CNeuronCrossAttention::Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle) || !cTTM.Save(file_handle) || !cPolicyMarket.Save(file_handle) || !cTTMHistory.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Append the trainable MPI graph; stacks are runtime state only. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronCrossAttention::AppendParameterFingerprint(fingerprint) || !cTTM.AppendParameterFingerprint(fingerprint) || !cPolicyMarket.AppendParameterFingerprint(fingerprint) || !cTTMHistory.AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMomADMPI state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::Load(const int file_handle) { if(!CNeuronCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTTM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPolicyMarket.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTTMHistory.AsObject())) ReturnFalse; if(cStack.GetDimension() != iWindow || cTTM.Neurons() != int(iWindow) || cPolicyMarket.Neurons() != int(iWindow * cStack.GetStackSize()) || cTTMHistory.Neurons() != int(iWindow)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMomADMPI GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMomADMPI::SetOpenCL(COpenCLMy *obj) { CNeuronCrossAttention::SetOpenCL(obj); cStack.SetOpenCL(OpenCL); cTTM.SetOpenCL(OpenCL); cPolicyMarket.SetOpenCL(OpenCL); cTTMHistory.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMomADMPI layer. | //+------------------------------------------------------------------+ void CNeuronMomADMPI::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); Q_Embedding.TrainMode(bTrain); KV_Embedding.TrainMode(bTrain); Transpose.TrainMode(bTrain); MHAttentionOut.TrainMode(bTrain); W0.TrainMode(bTrain); AttentionOut.TrainMode(bTrain); FF[0].TrainMode(bTrain); FF[1].TrainMode(bTrain); if(!!cContext) cContext.TrainMode(bTrain); cStack.TrainMode(bTrain); cTTM.TrainMode(bTrain); cPolicyMarket.TrainMode(bTrain); cTTMHistory.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronMomADMPI weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMomADMPI *src = source; if(!CNeuronMH2AttentionOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cTTM.WeightsUpdate(GetPointer(src.cTTM), tau)) ReturnFalse; if(!cPolicyMarket.WeightsUpdate(GetPointer(src.cPolicyMarket), tau)) ReturnFalse; if(!cTTMHistory.WeightsUpdate(GetPointer(src.cTTMHistory), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronMomADMPI execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMomADMPI::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!Q_Embedding.Clear() || !KV_Embedding.Clear() || !Transpose.Clear() || !MHAttentionOut.Clear() || !W0.Clear() || !AttentionOut.Clear() || !FF[0].Clear() || !FF[1].Clear()) ReturnFalse; if(!!cContext && !cContext.Clear()) ReturnFalse; if(!cStack.Clear() || !cTTM.Clear() || !cPolicyMarket.Clear() || !cTTMHistory.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| VLADriver RAG MPI extension | //+------------------------------------------------------------------+ #define RAG_MPI_SERIALIZATION_VERSION 5 #define RAG_NULL_GATE_SCALE 12.0f //+------------------------------------------------------------------+ //| Initializes the CNeuronVLADriverRAGMPI layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint latent_dim, uint candidates, uint market_dim, uint market_units, uint stack_size, uint topK, uint heads, uint key_dimension, uint rag_token_count, uint rag_token_width, ENUM_OPTIMIZATION optimization_type, uint batch) { pRAGTokens = NULL; pRAGAbsoluteRelevance = NULL; iRAGTokenCount = rag_token_count; iRAGTokenWidth = rag_token_width; iMarketEmbeddingSize = market_dim; if(iRAGTokenCount == 0 || iRAGTokenWidth < 2 || iMarketEmbeddingSize == 0) ReturnFalse; //--- The external market input is a single frozen ScenarioEmbedding. The //--- inherited MPI still consumes market_units K/V tokens, each latent-wide. if(!CNeuronMomADMPI::Init(numOutputs, myIndex, open_cl, latent_dim, candidates, latent_dim, market_units, stack_size, topK, heads, key_dimension, optimization_type, batch)) ReturnFalse; if(!cMarketEmbedding.Init(market_units * latent_dim, 4, OpenCL, iMarketEmbeddingSize, optimization, iBatch)) ReturnFalse; if(!cMarketProjection.Init(0, 5, OpenCL, market_units * latent_dim, optimization, iBatch)) ReturnFalse; if(!cMarketState.Init(0, 6, OpenCL, latent_dim, optimization, iBatch)) ReturnFalse; if(!cNullThresholdInput.Init(0, 7, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!cNullScenarioThreshold.Init(0, 8, OpenCL, 1, 1, 1, 1, optimization, iBatch)) ReturnFalse; if(!cPolicyRAG.Init(0, 9, OpenCL, latent_dim, key_dimension, heads, 1, iRAGTokenCount, iRAGTokenWidth, optimization, iBatch)) ReturnFalse; if(!cGatedRAGTokens.BufferInit(iRAGTokenCount * iRAGTokenWidth, 0)) ReturnFalse; if(!cGatedRAGTokens.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchActivity.BufferInit(1, 0)) ReturnFalse; if(!cRAGBranchActivity.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchActivityGradient.BufferInit(1, 0)) ReturnFalse; if(!cRAGBranchActivityGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchDelta.BufferInit(iWindow, 0)) ReturnFalse; if(!cRAGBranchDelta.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchOutput.BufferInit(iWindow, 0)) ReturnFalse; if(!cRAGBranchOutput.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGResidualGradient.BufferInit(iWindow, 0)) ReturnFalse; if(!cRAGResidualGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGGradient.BufferInit(iRAGTokenCount * iRAGTokenWidth, 0)) ReturnFalse; if(!cRAGGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cNullThresholdInput.getOutput().Fill(0.0f)) ReturnFalse; if(!cNullScenarioThreshold.GetWeightsConv()) ReturnFalse; if(cNullScenarioThreshold.GetWeightsConv().Total() < 2) ReturnFalse; if(!cNullScenarioThreshold.GetWeightsConv().Fill(0.0f)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Sets the rag tokens of the CNeuronVLADriverRAGMPI. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::SetRAGTokens(CBufferFloat *tokens, CBufferFloat *absolute_relevance, const uint token_width) { if(token_width < 2 || token_width != iRAGTokenWidth) ReturnFalse; if(!tokens) { pRAGTokens = NULL; pRAGAbsoluteRelevance = NULL; return(true); } if(tokens.Total() != int(iRAGTokenCount * iRAGTokenWidth) || absolute_relevance == NULL || absolute_relevance.Total() != 1 || absolute_relevance.GetOpenCL() != OpenCL || tokens.GetOpenCL() != OpenCL || absolute_relevance.GetIndex() < 0 || tokens.GetIndex() < 0) ReturnFalse; pRAGTokens = tokens; pRAGAbsoluteRelevance = absolute_relevance; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the rag token gate step of the CNeuronVLADriverRAGMPI | //| layer. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::RAGTokenGate(void) { const int token_total = int(iRAGTokenCount * iRAGTokenWidth); CBufferFloat *threshold_weights = cNullScenarioThreshold.GetWeightsConv(); if(!OpenCL || !pRAGTokens || !pRAGAbsoluteRelevance || iRAGTokenCount == 0 || iRAGTokenWidth < 2 || token_total <= 0 || pRAGTokens.Total() != token_total || pRAGAbsoluteRelevance.Total() != 1 || cGatedRAGTokens.Total() != token_total || cRAGBranchActivity.Total() != 1 || !threshold_weights || threshold_weights.Total() < 2) ReturnFalse; if(pRAGTokens.GetOpenCL() != OpenCL || pRAGAbsoluteRelevance.GetOpenCL() != OpenCL || cGatedRAGTokens.GetOpenCL() != OpenCL || cRAGBranchActivity.GetOpenCL() != OpenCL || threshold_weights.GetOpenCL() != OpenCL) ReturnFalse; if(pRAGTokens.GetIndex() < 0 || pRAGAbsoluteRelevance.GetIndex() < 0 || cGatedRAGTokens.GetIndex() < 0 || cRAGBranchActivity.GetIndex() < 0 || threshold_weights.GetIndex() < 0) ReturnFalse; //--- uint gate_offset[] = {0}; uint gate_global[] = {1}; const int kernel = def_k_RAGTokenGate; ResetLastError(); setBuffer(kernel, def_k_ragtg_tokens, pRAGTokens.GetIndex()) setBuffer(kernel, def_k_ragtg_gated_tokens, cGatedRAGTokens.GetIndex()) setBuffer(kernel, def_k_ragtg_absolute_relevance, pRAGAbsoluteRelevance.GetIndex()) setBuffer(kernel, def_k_ragtg_threshold_weights, threshold_weights.GetIndex()) setBuffer(kernel, def_k_ragtg_branch_activity, cRAGBranchActivity.GetIndex()) setArgument(kernel, def_k_ragtg_token_count, int(iRAGTokenCount)) setArgument(kernel, def_k_ragtg_token_width, int(iRAGTokenWidth)) setArgument(kernel, def_k_ragtg_scale, RAG_NULL_GATE_SCALE) kernelExecute(kernel, gate_offset, gate_global) #ifdef _DEBUG if(!cGatedRAGTokens.BufferRead()) ReturnFalse; if(!cRAGBranchActivity.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the rag token gate gradient step of the | //| CNeuronVLADriverRAGMPI layer. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::RAGTokenGateGradient(void) { const int token_total = int(iRAGTokenCount * iRAGTokenWidth); CBufferFloat *threshold_weights = cNullScenarioThreshold.GetWeightsConv(); CBufferFloat *threshold_gradient = cNullScenarioThreshold.getGradient(); if(!OpenCL || !pRAGTokens || !pRAGAbsoluteRelevance || iRAGTokenCount == 0 || iRAGTokenWidth < 2 || token_total <= 0 || pRAGTokens.Total() != token_total || pRAGAbsoluteRelevance.Total() != 1 || cRAGGradient.Total() != token_total || cRAGBranchActivityGradient.Total() != 1 || !threshold_weights || threshold_weights.Total() < 2 || !threshold_gradient || threshold_gradient.Total() != 1) ReturnFalse; if(pRAGTokens.GetOpenCL() != OpenCL || pRAGAbsoluteRelevance.GetOpenCL() != OpenCL || cRAGGradient.GetOpenCL() != OpenCL || cRAGBranchActivityGradient.GetOpenCL() != OpenCL || threshold_weights.GetOpenCL() != OpenCL || threshold_gradient.GetOpenCL() != OpenCL) ReturnFalse; if(pRAGTokens.GetIndex() < 0 || pRAGAbsoluteRelevance.GetIndex() < 0 || cRAGGradient.GetIndex() < 0 || cRAGBranchActivityGradient.GetIndex() < 0 || threshold_weights.GetIndex() < 0 || threshold_gradient.GetIndex() < 0) ReturnFalse; //--- uint gate_offset[] = {0}; uint gate_global[] = {1}; const int kernel = def_k_RAGTokenGateGrad; ResetLastError(); setBuffer(kernel, def_k_ragtgg_tokens, pRAGTokens.GetIndex()) setBuffer(kernel, def_k_ragtgg_gated_gradient, cRAGGradient.GetIndex()) setBuffer(kernel, def_k_ragtgg_absolute_relevance, pRAGAbsoluteRelevance.GetIndex()) setBuffer(kernel, def_k_ragtgg_threshold_weights, threshold_weights.GetIndex()) setBuffer(kernel, def_k_ragtgg_threshold_gradient, threshold_gradient.GetIndex()) setBuffer(kernel, def_k_ragtgg_branch_activity_gradient, cRAGBranchActivityGradient.GetIndex()) setArgument(kernel, def_k_ragtgg_token_count, int(iRAGTokenCount)) setArgument(kernel, def_k_ragtgg_token_width, int(iRAGTokenWidth)) setArgument(kernel, def_k_ragtgg_scale, RAG_NULL_GATE_SCALE) kernelExecute(kernel, gate_offset, gate_global) #ifdef _DEBUG if(!threshold_gradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronVLADriverRAGMPI feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput || SecondInput.Total() != int(iMarketEmbeddingSize)) ReturnFalse; if(SecondInput.GetOpenCL() != OpenCL || SecondInput.GetIndex() < 0) ReturnFalse; //--- Copy on the device rather than bind: the encoder owns SecondInput and stays frozen. CBufferFloat *market_embedding = cMarketEmbedding.getOutput(); if(!market_embedding || market_embedding.Total() != SecondInput.Total() || market_embedding.GetOpenCL() != OpenCL || market_embedding.GetIndex() < 0) ReturnFalse; if(market_embedding != SecondInput) { if(cMarketEmbedding.Neurons() > SecondInput.Total()) ReturnFalse; if(!Concat(SecondInput, SecondInput, market_embedding, 1, 0, cMarketEmbedding.Neurons())) ReturnFalse; } if(!cMarketProjection.FeedForward(cMarketEmbedding.AsObject())) ReturnFalse; CBufferFloat *market = cMarketProjection.getOutput(); if(!market || market.Total() != int(iWindow_K * iUnits_K) || market.GetOpenCL() != OpenCL || market.GetIndex() < 0) ReturnFalse; //--- push the previous MPI scenario before overwriting Output if(!cStack.FeedForward(GetPointer(this))) ReturnFalse; //--- TTM builds the weighted scenario from the strategy pool and recent scenarios if(!cTTM.FeedForward(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- D_cur = actions stack attends to market if(!cPolicyMarket.FeedForward(cStack.AsObject(), market)) ReturnFalse; //--- Q_hist = TTM scenario attends to the current action-aware history if(!cTTMHistory.FeedForward(cTTM.AsObject(), cPolicyMarket.getOutput())) ReturnFalse; //--- CNeuronBaseOCL *final_query = cTTMHistory.AsObject(); if(!!pRAGTokens) { if(!RAGTokenGate()) ReturnFalse; if(!cPolicyRAG.FeedForward(cTTMHistory.AsObject(), GetPointer(cGatedRAGTokens))) ReturnFalse; if(!Different(cPolicyRAG.getOutput(), cTTMHistory.getOutput(), GetPointer(cRAGBranchDelta), iWindow)) ReturnFalse; if(!ScalarToVector(GetPointer(cRAGBranchActivity), GetPointer(cRAGBranchDelta), GetPointer(cRAGBranchOutput), iWindow)) ReturnFalse; if(!SumAndNormalize(cTTMHistory.getOutput(), GetPointer(cRAGBranchOutput), cMarketState.getOutput(), iWindow, false, 0, 0, 0, 1.0f)) ReturnFalse; final_query = cMarketState.AsObject(); } //--- parent cross-attention is the final MPI market refine block if(!CNeuronCrossAttention::feedForward(final_query, market)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronVLADriverRAGMPI layer. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !NeuronOCL || !SecondInput || SecondInput.Total() != int(iMarketEmbeddingSize)) ReturnFalse; if(SecondInput.GetOpenCL() != OpenCL || SecondInput.GetIndex() < 0) ReturnFalse; //--- The StateEncoder is frozen while Actor learns this projection. A caller //--- may still supply its gradient buffer, but it must never receive Actor //--- gradients through this second-input boundary. if(SecondGradient) { if(SecondGradient.Total() != SecondInput.Total()) ReturnFalse; if(SecondGradient.GetOpenCL() != OpenCL || SecondGradient.GetIndex() < 0) ReturnFalse; if(!SecondGradient.Fill(0)) ReturnFalse; } //--- final MPI market-refine gradients CBufferFloat *market = cMarketProjection.getOutput(); CBufferFloat *market_gradient = cMarketProjection.getGradient(); if(!market || !market_gradient || market.Total() != int(iWindow_K * iUnits_K) || market_gradient.Total() != market.Total() || market.GetOpenCL() != OpenCL || market_gradient.GetOpenCL() != OpenCL || market.GetIndex() < 0 || market_gradient.GetIndex() < 0) ReturnFalse; if(!market_gradient.Fill(0)) ReturnFalse; CNeuronBaseOCL *final_query = (!!pRAGTokens ? cMarketState.AsObject() : cTTMHistory.AsObject()); if(!CNeuronCrossAttention::calcInputGradients(final_query, market, market_gradient, None)) ReturnFalse; //--- RAG is immediately before the final market-refine block if(!!pRAGTokens) { if(cMarketState.getGradient().Total() != int(iWindow) || cMarketState.getGradient().GetOpenCL() != OpenCL || cMarketState.getGradient().GetIndex() < 0) ReturnFalse; if(!cTTMHistory.getGradient().Fill(0)) ReturnFalse; if(!cRAGGradient.Fill(0)) ReturnFalse; if(!cRAGBranchActivityGradient.Fill(0)) ReturnFalse; if(!cRAGResidualGradient.Fill(0)) ReturnFalse; if(!cNullScenarioThreshold.getGradient().Fill(0)) ReturnFalse; if(!ScalarToVectorGrad(GetPointer(cRAGBranchActivity), GetPointer(cRAGBranchActivityGradient), GetPointer(cRAGBranchDelta), cPolicyRAG.getGradient(), cMarketState.getGradient(), iWindow)) ReturnFalse; if(!Different(cMarketState.getGradient(), cPolicyRAG.getGradient(), GetPointer(cRAGResidualGradient), iWindow)) ReturnFalse; if(!cTTMHistory.CalcHiddenGradients(cPolicyRAG.AsObject(), GetPointer(cGatedRAGTokens), GetPointer(cRAGGradient), None)) ReturnFalse; if(!SumAndNormalize(cTTMHistory.getGradient(), GetPointer(cRAGResidualGradient), cTTMHistory.getGradient(), iWindow, false, 0, 0, 0, 1.0f)) ReturnFalse; if(!RAGTokenGateGradient()) ReturnFalse; } //--- TTM-history gradients: Q_ttm and D_cur if(!cTTM.CalcHiddenGradients(cTTMHistory.AsObject(), cPolicyMarket.getOutput(), cPolicyMarket.getGradient(), None)) ReturnFalse; //--- TTM gradients to the strategy pool. The stack branch is intentionally stop-gradient. if(!NeuronOCL.CalcHiddenGradients(cTTM.AsObject(), cStack.getOutput(), NULL, None)) ReturnFalse; //--- policy-market gradients are needed for its weights and the projected market gradient CBufferFloat policyMarketGradient; if(!policyMarketGradient.BufferInit(market.Total(), 0)) ReturnFalse; if(!policyMarketGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cStack.CalcHiddenGradients(cPolicyMarket.AsObject(), market, GetPointer(policyMarketGradient), None)) ReturnFalse; if(cStack.getGradient().Total() > 0) if(!cStack.getGradient().Fill(0)) ReturnFalse; if(!SumAndNormalize(market_gradient, GetPointer(policyMarketGradient), market_gradient, int(iWindow_K), false, 0, 0, 0, 1.0f)) ReturnFalse; //--- Actor projection learns; the external StateEncoder boundary remains frozen if(!cMarketEmbedding.CalcHiddenGradients(cMarketProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronVLADriverRAGMPI trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput || SecondInput.Total() != int(iMarketEmbeddingSize) || SecondInput.GetOpenCL() != OpenCL || SecondInput.GetIndex() < 0) ReturnFalse; CBufferFloat *market = cMarketProjection.getOutput(); if(!market || market.Total() != int(iWindow_K * iUnits_K) || market.GetOpenCL() != OpenCL || market.GetIndex() < 0) ReturnFalse; //--- final market-refine weights use the actual post-RAG query CNeuronBaseOCL *final_query = (!!pRAGTokens ? cMarketState.AsObject() : cTTMHistory.AsObject()); if(!CNeuronCrossAttention::updateInputWeights(final_query, market)) ReturnFalse; //--- RAG attention consumes TTM history, not the final market-refined state if(!!pRAGTokens) { if(iRAGTokenWidth < 2 || !pRAGAbsoluteRelevance || pRAGTokens.Total() != int(iRAGTokenCount * iRAGTokenWidth) || cGatedRAGTokens.Total() != pRAGTokens.Total() || pRAGTokens.GetOpenCL() != OpenCL || pRAGAbsoluteRelevance.GetOpenCL() != OpenCL || cGatedRAGTokens.GetOpenCL() != OpenCL || pRAGTokens.GetIndex() < 0 || pRAGAbsoluteRelevance.GetIndex() < 0 || cGatedRAGTokens.GetIndex() < 0) ReturnFalse; if(!cPolicyRAG.UpdateInputWeights(cTTMHistory.AsObject(), GetPointer(cGatedRAGTokens))) ReturnFalse; if(!cNullScenarioThreshold.UpdateInputWeights(cNullThresholdInput.AsObject())) ReturnFalse; } //--- inherited MPI prefix weights if(!cTTMHistory.UpdateInputWeights(cTTM.AsObject(), cPolicyMarket.getOutput())) ReturnFalse; if(!cPolicyMarket.UpdateInputWeights(cStack.AsObject(), market)) ReturnFalse; if(!cMarketProjection.UpdateInputWeights(cMarketEmbedding.AsObject())) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronVLADriverRAGMPI state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::Save(const int file_handle) { if(iMarketEmbeddingSize == 0 || iRAGTokenCount == 0 || iRAGTokenWidth < 2) ReturnFalse; if(!CNeuronMomADMPI::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, RAG_MPI_SERIALIZATION_VERSION, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iMarketEmbeddingSize), INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iRAGTokenCount), INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iRAGTokenWidth), INT_VALUE) < INT_VALUE) ReturnFalse; if(!cMarketEmbedding.Save(file_handle)) ReturnFalse; if(!cMarketProjection.Save(file_handle)) ReturnFalse; if(!cNullScenarioThreshold.Save(file_handle)) ReturnFalse; if(!cPolicyRAG.Save(file_handle)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronVLADriverRAGMPI state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::Load(const int file_handle) { if(!CNeuronMomADMPI::Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle, INT_VALUE) != RAG_MPI_SERIALIZATION_VERSION) ReturnFalse; iMarketEmbeddingSize = uint(FileReadInteger(file_handle, INT_VALUE)); iRAGTokenCount = uint(FileReadInteger(file_handle, INT_VALUE)); iRAGTokenWidth = uint(FileReadInteger(file_handle, INT_VALUE)); if(iMarketEmbeddingSize == 0 || iRAGTokenCount == 0 || iRAGTokenWidth < 2) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cMarketEmbedding))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cMarketProjection))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNullScenarioThreshold))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cPolicyRAG))) ReturnFalse; CBufferFloat *threshold_weights = cNullScenarioThreshold.GetWeightsConv(); if(cMarketEmbedding.Neurons() != int(iMarketEmbeddingSize) || cMarketProjection.Neurons() != int(iWindow_K * iUnits_K) || cNullScenarioThreshold.Neurons() != 1 || !threshold_weights || threshold_weights.Total() < 2 || cPolicyRAG.Neurons() != Neurons()) ReturnFalse; if(!cMarketState.Init(0, 6, OpenCL, uint(Neurons()), optimization, iBatch)) ReturnFalse; if(!cNullThresholdInput.Init(0, 7, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!cNullThresholdInput.getOutput().Fill(0.0f)) ReturnFalse; if(!cGatedRAGTokens.BufferInit(iRAGTokenCount * iRAGTokenWidth, 0)) ReturnFalse; if(!cGatedRAGTokens.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchActivity.BufferInit(1, 0)) ReturnFalse; if(!cRAGBranchActivity.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchActivityGradient.BufferInit(1, 0)) ReturnFalse; if(!cRAGBranchActivityGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchDelta.BufferInit(uint(Neurons()), 0)) ReturnFalse; if(!cRAGBranchDelta.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGBranchOutput.BufferInit(uint(Neurons()), 0)) ReturnFalse; if(!cRAGBranchOutput.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGResidualGradient.BufferInit(uint(Neurons()), 0)) ReturnFalse; if(!cRAGResidualGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cRAGGradient.BufferInit(iRAGTokenCount * iRAGTokenWidth, 0)) ReturnFalse; if(!cRAGGradient.BufferCreate(OpenCL)) ReturnFalse; pRAGTokens = NULL; pRAGAbsoluteRelevance = NULL; return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronVLADriverRAGMPI GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronVLADriverRAGMPI::SetOpenCL(COpenCLMy *obj) { CNeuronMomADMPI::SetOpenCL(obj); cMarketEmbedding.SetOpenCL(OpenCL); cMarketProjection.SetOpenCL(OpenCL); cMarketState.SetOpenCL(OpenCL); cNullScenarioThreshold.SetOpenCL(OpenCL); cNullThresholdInput.SetOpenCL(OpenCL); cPolicyRAG.SetOpenCL(OpenCL); if(cGatedRAGTokens.Total() > 0) cGatedRAGTokens.BufferCreate(OpenCL); if(cRAGBranchActivity.Total() > 0) cRAGBranchActivity.BufferCreate(OpenCL); if(cRAGBranchActivityGradient.Total() > 0) cRAGBranchActivityGradient.BufferCreate(OpenCL); if(cRAGBranchDelta.Total() > 0) cRAGBranchDelta.BufferCreate(OpenCL); if(cRAGBranchOutput.Total() > 0) cRAGBranchOutput.BufferCreate(OpenCL); if(cRAGResidualGradient.Total() > 0) cRAGResidualGradient.BufferCreate(OpenCL); if(cRAGGradient.Total() > 0) cRAGGradient.BufferCreate(OpenCL); pRAGTokens = NULL; pRAGAbsoluteRelevance = NULL; } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronVLADriverRAGMPI layer. | //+------------------------------------------------------------------+ void CNeuronVLADriverRAGMPI::TrainMode(bool flag) { CNeuronMomADMPI::TrainMode(flag); cMarketEmbedding.TrainMode(flag); cMarketProjection.TrainMode(flag); cMarketState.TrainMode(flag); cNullScenarioThreshold.TrainMode(flag); cNullThresholdInput.TrainMode(false); cPolicyRAG.TrainMode(flag); } //+------------------------------------------------------------------+ //| Averages the CNeuronVLADriverRAGMPI weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronVLADriverRAGMPI *src = source; if(!CNeuronMomADMPI::WeightsUpdate(source, tau)) ReturnFalse; dWeightsUpdate(cMarketEmbedding, src, tau); dWeightsUpdate(cMarketProjection, src, tau); dWeightsUpdate(cNullScenarioThreshold, src, tau); dWeightsUpdate(cPolicyRAG, src, tau); return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronVLADriverRAGMPI execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronVLADriverRAGMPI::Clear(void) { if(!CNeuronMomADMPI::Clear()) ReturnFalse; if(!cMarketEmbedding.Clear()) ReturnFalse; if(!cMarketProjection.Clear()) ReturnFalse; if(!cMarketState.Clear()) ReturnFalse; if(!cNullScenarioThreshold.Clear()) ReturnFalse; if(!cNullThresholdInput.Clear()) ReturnFalse; if(!cPolicyRAG.Clear()) ReturnFalse; if(cGatedRAGTokens.Total() > 0) if(!cGatedRAGTokens.Fill(0)) ReturnFalse; if(cRAGBranchActivity.Total() > 0) if(!cRAGBranchActivity.Fill(0)) ReturnFalse; if(cRAGBranchActivityGradient.Total() > 0) if(!cRAGBranchActivityGradient.Fill(0)) ReturnFalse; if(cRAGBranchDelta.Total() > 0) if(!cRAGBranchDelta.Fill(0)) ReturnFalse; if(cRAGBranchOutput.Total() > 0) if(!cRAGBranchOutput.Fill(0)) ReturnFalse; if(cRAGResidualGradient.Total() > 0) if(!cRAGResidualGradient.Fill(0)) ReturnFalse; if(cRAGGradient.Total() > 0) if(!cRAGGradient.Fill(0)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUncAD_MSD layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MSD::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint variables, uint quantiles, float range, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units < (10 * quantiles)) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * (2 * quantiles + 1)*variables, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iUnits = units; iVariables = variables; iQuantiles = quantiles; fQuantSize = range / iQuantiles; if(fQuantSize < FLT_EPSILON) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUncAD_MSD feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MSD::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; if(NeuronOCL.Neurons() < int(iDimension * iUnits * iVariables)) ReturnFalse; //--- uint global_work_offset[] = { 0, 0, 0 }; uint global_work_size[] = { iDimension, (uint)MathMin(iUnits, OpenCL.GetMaxLocalSize(1)), iVariables }; uint local_work_size[] = { 1, global_work_size[1], 1 }; //--- const int kernel = def_k_MarketStateDensity; setBuffer(kernel, def_k_msd_sequence, NeuronOCL.getOutputIndex()); setBuffer(kernel, def_k_msd_distribution, getOutputIndex()); setArgument(kernel, def_k_msd_sequence_size, iUnits); setArgument(kernel, def_k_msd_quantiles, iQuantiles); setArgument(kernel, def_k_msd_quant_step, fQuantSize); //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size); #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUncAD_MSD state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MSD::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iUnits); dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iQuantiles); dFileWriteFloat(file_handle, fQuantSize); //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUncAD_MSD state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MSD::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, iUnits); dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iQuantiles); dFileReadUFloat(file_handle, fQuantSize); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCogDriverData layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint variables, uint state_stack_size, uint quant_stack_size, uint quantiles, float range, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window == 0 || variables == 0 || state_stack_size < (10 * quantiles) || quant_stack_size < 2 || quantiles == 0 || range <= 0) ReturnFalse; uint bins = 2 * quantiles + 1; uint quant_dimension = window * bins; uint quant_features = quant_dimension * variables; if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, quant_stack_size, quant_stack_size, 1, 1, quant_features, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iVariables = variables; iStateStackSize = state_stack_size; iQuantStackSize = quant_stack_size; iQuantiles = quantiles; fRange = range; //--- if(!cNorm.Init(0, 0, OpenCL, window * variables, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); if(!cStateStack.Init(0, 1, OpenCL, state_stack_size, window, variables, optimization, iBatch)) ReturnFalse; if(!cMSD.Init(0, 2, OpenCL, window, state_stack_size, variables, quantiles, range, optimization, iBatch)) ReturnFalse; if(!cAffine.Init(0, 3, OpenCL, window, window, 2, 1, variables, optimization, iBatch)) ReturnFalse; cAffine.SetActivationFunction(None); if(!cQuantStack.Init(0, 4, OpenCL, quant_stack_size, 1, quant_features, optimization, iBatch)) ReturnFalse; if(!cPrevQuantStack.Init(0, 5, OpenCL, quant_stack_size * quant_features, optimization, iBatch)) ReturnFalse; cPrevQuantStack.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the affine stack step of the CNeuronCogDriverData | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::AffineStack(void) { if(!OpenCL) ReturnFalse; //--- uint quant_dimension = iWindow * (2 * iQuantiles + 1); uint quant_features = quant_dimension * iVariables; if(cQuantStack.Neurons() != int(iQuantStackSize * quant_features) || cPrevQuantStack.Neurons() != cQuantStack.Neurons() || cAffine.Neurons() != int(2 * iVariables)) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {quant_features, iQuantStackSize}; uint kernel = def_k_CogDriverAffineStack; setBuffer(kernel, def_k_cdas_stack, cQuantStack.getOutputIndex()) setBuffer(kernel, def_k_cdas_snapshot, cPrevQuantStack.getOutputIndex()) setBuffer(kernel, def_k_cdas_affine, cAffine.getOutputIndex()) setArgument(kernel, def_k_cdas_stack_size, (int)iQuantStackSize) setArgument(kernel, def_k_cdas_quant_dimension, (int)quant_dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQuantStack.getOutput().BufferRead() || !cPrevQuantStack.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the affine stack gradient step of the | //| CNeuronCogDriverData layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::AffineStackGradient(void) { if(!OpenCL) ReturnFalse; //--- uint quant_dimension = iWindow * (2 * iQuantiles + 1); uint kernel = def_k_CogDriverAffineStackGrad; uint global_work_offset[] = {0, 0}; uint global_work_size[] = {iVariables, 2}; setBuffer(kernel, def_k_cdasg_snapshot, cPrevQuantStack.getOutputIndex()) setBuffer(kernel, def_k_cdasg_stack_gr, cQuantStack.getGradientIndex()) setBuffer(kernel, def_k_cdasg_affine_gr, cAffine.getGradientIndex()) setArgument(kernel, def_k_cdasg_stack_size, (int)iQuantStackSize) setArgument(kernel, def_k_cdasg_quant_dimension, (int)quant_dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cAffine.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCogDriverData feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < int(iWindow * iVariables)) ReturnFalse; if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cStateStack.FeedForward(cNorm.AsObject())) ReturnFalse; if(!cMSD.FeedForward(cStateStack.AsObject())) ReturnFalse; if(!cAffine.FeedForward(cNorm.AsObject())) ReturnFalse; if(!AffineStack()) ReturnFalse; if(!cQuantStack.FeedForward(cMSD.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cQuantStack.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCogDriverData layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeConvBlock::calcInputGradients(cQuantStack.AsObject())) ReturnFalse; if(!AffineStackGradient()) ReturnFalse; if(!NeuronOCL.getGradient().Fill(0)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCogDriverData trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cQuantStack.AsObject())) ReturnFalse; if(!cAffine.UpdateInputWeights(cNorm.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCogDriverData state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iWindow); dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iStateStackSize); dFileWriteUInt(file_handle, iQuantStackSize); dFileWriteUInt(file_handle, iQuantiles); dFileWriteFloat(file_handle, fRange); if(!cNorm.Save(file_handle)) ReturnFalse; if(!cStateStack.Save(file_handle)) ReturnFalse; if(!cMSD.Save(file_handle)) ReturnFalse; if(!cAffine.Save(file_handle)) ReturnFalse; if(!cQuantStack.Save(file_handle)) ReturnFalse; if(!cPrevQuantStack.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Append the trainable CogDriver encoder graph. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronSpikeConvBlock::AppendParameterFingerprint(fingerprint) || !cNorm.AppendParameterFingerprint(fingerprint) || !cStateStack.AppendParameterFingerprint(fingerprint) || !cMSD.AppendParameterFingerprint(fingerprint) || !cAffine.AppendParameterFingerprint(fingerprint) || !cQuantStack.AppendParameterFingerprint(fingerprint) || !cPrevQuantStack.AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCogDriverData state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iWindow); dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iStateStackSize); dFileReadUInt(file_handle, iQuantStackSize); dFileReadUInt(file_handle, iQuantiles); dFileReadUFloat(file_handle, fRange); if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateStack.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMSD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAffine.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQuantStack.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrevQuantStack.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCogDriverData GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronCogDriverData::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cNorm.SetOpenCL(OpenCL); cStateStack.SetOpenCL(OpenCL); cMSD.SetOpenCL(OpenCL); cAffine.SetOpenCL(OpenCL); cQuantStack.SetOpenCL(OpenCL); cPrevQuantStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Reports inherited checked-transfer support for the MSD layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MSD::SupportsOpenCLChecked(void) const { //--- Base owns every MSD device buffer; reject unsupported subtypes. return(Type() == defNeuronUncAD_MSD); } //+------------------------------------------------------------------+ //| Reports support for the complete CogDriverData ownership tree. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::SupportsOpenCLChecked(void) const { //--- Validate inherited and own owners without the parent type guard. return(Type() == defNeuronCogDriverData && CheckPointer(BatchOptions) != POINTER_INVALID && cActivation.SupportsOpenCLChecked() && cConv.SupportsOpenCLChecked() && cStack.SupportsOpenCLChecked() && cNorm.SupportsOpenCLChecked() && cStateStack.SupportsOpenCLChecked() && cMSD.SupportsOpenCLChecked() && cAffine.SupportsOpenCLChecked() && cQuantStack.SupportsOpenCLChecked() && cPrevQuantStack.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers CogDriverData children before inherited publication. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::SetOpenCLChecked(COpenCLMy *obj) { //--- Reject unsupported ownership trees before any mutation. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or data ownership tree"); //--- Preserve normalization parameters and accumulated state. if(!cNorm.SetOpenCLChecked(obj)) ReturnFalseEx("normalization context"); //--- Preserve the raw state history. if(!cStateStack.SetOpenCLChecked(obj)) ReturnFalseEx("state stack context"); //--- Transfer the distribution descriptor buffers. if(!cMSD.SetOpenCLChecked(obj)) ReturnFalseEx("distribution descriptor context"); //--- Preserve affine parameters and optimizer owners. if(!cAffine.SetOpenCLChecked(obj)) ReturnFalseEx("affine context"); //--- Preserve current and previous quantile history. if(!cQuantStack.SetOpenCLChecked(obj)) ReturnFalseEx("quantile stack context"); if(!cPrevQuantStack.SetOpenCLChecked(obj)) ReturnFalseEx("previous quantile stack context"); //--- Publish inherited owners only after all own children succeeded. if(!CNeuronSpikeConvBlock::SetOpenCLChecked(obj)) ReturnFalseEx("inherited spike convolution context"); //--- Finalize without resetting configuration or history. return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronCogDriverData layer. | //+------------------------------------------------------------------+ void CNeuronCogDriverData::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cNorm.TrainMode(bTrain); cStateStack.TrainMode(bTrain); cMSD.TrainMode(bTrain); cAffine.TrainMode(bTrain); cQuantStack.TrainMode(bTrain); cPrevQuantStack.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronCogDriverData weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronCogDriverData* Source = source; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; if(!cStateStack.WeightsUpdate(Source.cStateStack.AsObject(), tau)) ReturnFalse; if(!cMSD.WeightsUpdate(Source.cMSD.AsObject(), tau)) ReturnFalse; if(!cAffine.WeightsUpdate(Source.cAffine.AsObject(), tau)) ReturnFalse; if(!cQuantStack.WeightsUpdate(Source.cQuantStack.AsObject(), tau)) ReturnFalse; if(!cPrevQuantStack.WeightsUpdate(Source.cPrevQuantStack.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronCogDriverData execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverData::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cNorm.Clear()) ReturnFalse; if(!cStateStack.Clear()) ReturnFalse; if(!cMSD.Clear()) ReturnFalse; if(!cAffine.Clear()) ReturnFalse; if(!cQuantStack.Clear()) ReturnFalse; if(!cPrevQuantStack.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCogDriverRankTCM layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::InitStateBuffers(void) { uint memory_total = iMemoryLen * iVariables * iQueryDim; uint kv_total = 2 * iMemoryLen * iTotalQueryDim; uint self_kv_total = 2 * iVariables * iTotalQueryDim; if(memory_total == 0 || kv_total == 0 || self_kv_total == 0 || iTotalHeads == 0) ReturnFalse; //--- if(!cMemoryQuery.BufferInit(memory_total, 0) || !cMemoryQuery.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryAge.BufferInit(iMemoryLen * iVariables, 0) || !cMemoryAge.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryScore.BufferInit(iMemoryLen * iVariables, 0) || !cMemoryScore.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryValid.BufferInit(iMemoryLen * iVariables, 0) || !cMemoryValid.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryKV.BufferInit(kv_total, 0) || !cMemoryKV.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryKVGradient.BufferInit(kv_total, 0) || !cMemoryKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfKV.BufferInit(self_kv_total, 0) || !cSelfKV.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfKVGradient.BufferInit(self_kv_total, 0) || !cSelfKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cLogSumExp.BufferInit(iTotalHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfLogSumExp.BufferInit(iTotalHeads, 0) || !cSelfLogSumExp.BufferCreate(OpenCL)) ReturnFalse; if(!cQueryGateGradient.BufferInit(iTotalQueryDim, 0) || !cQueryGateGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cGateQueryGradient.BufferInit(iTotalQueryDim, 0) || !cGateQueryGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cGateContextGradient.BufferInit(iTotalQueryDim, 0) || !cGateContextGradient.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCogDriverRankTCM layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint input_dim, uint query_dim, uint variables, uint memory_len, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(input_dim == 0 || query_dim == 0 || variables == 0 || memory_len == 0 || heads == 0 || query_dim % heads != 0) ReturnFalse; uint total_query = variables * query_dim; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, total_query, optimization_type, batch)) ReturnFalse; //--- iInputDim = input_dim; iQueryDim = query_dim; iVariables = variables; iHeads = heads; iHeadDim = query_dim / heads; iTotalQueryDim = total_query; iTotalHeads = variables * heads; iMemoryLen = memory_len; iMemoryCount = 0; bLastQueryValid = false; //--- if(!cQueryProjection.Init(0, 0, OpenCL, input_dim, input_dim, query_dim, variables, 1, optimization, iBatch)) ReturnFalse; cQueryProjection.SetActivationFunction(None); if(!cGateInput.Init(total_query, 1, OpenCL, 2 * total_query, optimization, iBatch)) ReturnFalse; cGateInput.SetActivationFunction(None); if(!cGateProjection.Init(0, 2, OpenCL, total_query, optimization, iBatch)) ReturnFalse; cGateProjection.SetActivationFunction(SIGMOID); if(!cMemoryContext.Init(0, 3, OpenCL, total_query, optimization, iBatch)) ReturnFalse; cMemoryContext.SetActivationFunction(None); if(!cSelfContext.Init(0, 4, OpenCL, total_query, optimization, iBatch)) ReturnFalse; cSelfContext.SetActivationFunction(None); if(!InitStateBuffers()) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Executes the insert pending query step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::InsertPendingQuery(void) { if(!bLastQueryValid) return(true); if(!OpenCL) ReturnFalse; CBufferFloat *pending = cQueryProjection.getOutput(); if(!pending) ReturnFalse; //--- insert the previous query independently for each variable uint kernel = def_k_CogDriverRankTCMInsertPending; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iVariables, iQueryDim}; uint local_work_size[2] = {1, iQueryDim}; setBuffer(kernel, def_k_cdrtcm_insert_memory, cMemoryQuery.GetIndex()) setBuffer(kernel, def_k_cdrtcm_insert_age, cMemoryAge.GetIndex()) setBuffer(kernel, def_k_cdrtcm_insert_score, cMemoryScore.GetIndex()) setBuffer(kernel, def_k_cdrtcm_insert_valid, cMemoryValid.GetIndex()) setBuffer(kernel, def_k_cdrtcm_insert_pending_query, pending.GetIndex()) setArgument(kernel, def_k_cdrtcm_insert_memory_len, (int)iMemoryLen) setArgument(kernel, def_k_cdrtcm_insert_query_dim, (int)iQueryDim) setArgument(kernel, def_k_cdrtcm_insert_variables, (int)iVariables) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) iMemoryCount = (uint)MathMin(iMemoryLen, iMemoryCount + 1); #ifdef _DEBUG if(!cMemoryQuery.BufferRead() || !cMemoryAge.BufferRead() || !cMemoryScore.BufferRead() || !cMemoryValid.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build memory kv step of the CNeuronCogDriverRankTCM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::BuildMemoryKV(void) { if(!OpenCL || iMemoryCount == 0) ReturnFalse; //--- uint kernel = def_k_CogDriverRankTCMBuildKV; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iMemoryCount, iVariables, iQueryDim}; setBuffer(kernel, def_k_cdrtcm_kv_memory, cMemoryQuery.GetIndex()) setBuffer(kernel, def_k_cdrtcm_kv_age, cMemoryAge.GetIndex()) setBuffer(kernel, def_k_cdrtcm_kv_valid, cMemoryValid.GetIndex()) setBuffer(kernel, def_k_cdrtcm_kv_out, cMemoryKV.GetIndex()) setArgument(kernel, def_k_cdrtcm_kv_memory_count, (int)iMemoryCount) setArgument(kernel, def_k_cdrtcm_kv_variables, (int)iVariables) setArgument(kernel, def_k_cdrtcm_kv_heads, (int)iHeads) setArgument(kernel, def_k_cdrtcm_kv_head_dim, (int)iHeadDim) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cMemoryKV.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the flash attention forward step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::FlashAttentionForward(CBufferFloat *query, CBufferFloat *out) { if(!OpenCL || !query || !out || iMemoryCount == 0) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = {1, (uint)MathMin(MathMax(iMemoryCount, iHeadDim), OpenCL.GetMaxLocalSize(1)), iTotalHeads }; uint local_work_size[] = {1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttention; setBuffer(kernel, def_k_mhflat_q, query.GetIndex()) setBuffer(kernel, def_k_mhflat_kv, cMemoryKV.GetIndex()) setBuffer(kernel, def_k_mhflat_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflat_out, out.GetIndex()) setArgument(kernel, def_k_mhflat_dimension, (int)iHeadDim) setArgument(kernel, def_k_mhflat_total_kv, (int)iMemoryCount) setArgument(kernel, def_k_mhflat_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the flash attention backward step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::FlashAttentionBackward(CBufferFloat *query, CBufferFloat *query_gr, CBufferFloat *out, CBufferFloat *out_gr) { if(!OpenCL || !query || !query_gr || !out || !out_gr || iMemoryCount == 0) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = {MathMax((int)1, (int)iMemoryCount), iHeadDim, iTotalHeads}; uint local_work_size[] = {1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttentionGrad; setBuffer(kernel, def_k_mhflatg_q, query.GetIndex()) setBuffer(kernel, def_k_mhflatg_q_gr, query_gr.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv, cMemoryKV.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv_gr, cMemoryKVGradient.GetIndex()) setBuffer(kernel, def_k_mhflatg_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflatg_out, out.GetIndex()) setBuffer(kernel, def_k_mhflatg_out_gr, out_gr.GetIndex()) setArgument(kernel, def_k_mhflatg_dimension, (int)iHeadDim) setArgument(kernel, def_k_mhflatg_total_q, 1) setArgument(kernel, def_k_mhflatg_total_kv, (int)iMemoryCount) setArgument(kernel, def_k_mhflatg_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!query_gr.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build self kv step of the CNeuronCogDriverRankTCM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::BuildSelfKV(CBufferFloat *source) { if(!OpenCL || !source) ReturnFalse; //--- uint kernel = def_k_CogDriverRankTCMSelfKV; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iVariables, iTotalQueryDim}; setBuffer(kernel, def_k_cdrtcm_selfkv_source, source.GetIndex()) setBuffer(kernel, def_k_cdrtcm_selfkv_out, cSelfKV.GetIndex()) setArgument(kernel, def_k_cdrtcm_selfkv_query_dim, (int)iQueryDim) setArgument(kernel, def_k_cdrtcm_selfkv_variables, (int)iVariables) setArgument(kernel, def_k_cdrtcm_selfkv_heads, (int)iHeads) setArgument(kernel, def_k_cdrtcm_selfkv_head_dim, (int)iHeadDim) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cSelfKV.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the self attention forward step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::SelfAttentionForward(CBufferFloat *query, CBufferFloat *out) { if(!OpenCL || !query || !out || iVariables == 0) ReturnFalse; if(!BuildSelfKV(query)) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = {1, (uint)MathMin(MathMax(iVariables, iHeadDim), OpenCL.GetMaxLocalSize(1)), iTotalHeads }; uint local_work_size[] = {1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttention; setBuffer(kernel, def_k_mhflat_q, query.GetIndex()) setBuffer(kernel, def_k_mhflat_kv, cSelfKV.GetIndex()) setBuffer(kernel, def_k_mhflat_logsumexp, cSelfLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflat_out, out.GetIndex()) setArgument(kernel, def_k_mhflat_dimension, (int)iHeadDim) setArgument(kernel, def_k_mhflat_total_kv, (int)iVariables) setArgument(kernel, def_k_mhflat_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the self attention backward step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::SelfAttentionBackward(CBufferFloat *query, CBufferFloat *query_gr, CBufferFloat *out, CBufferFloat *out_gr) { if(!OpenCL || !query || !query_gr || !out || !out_gr || iVariables == 0) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = {MathMax((int)1, (int)iVariables), iHeadDim, iTotalHeads}; uint local_work_size[] = {1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttentionGrad; setBuffer(kernel, def_k_mhflatg_q, query.GetIndex()) setBuffer(kernel, def_k_mhflatg_q_gr, query_gr.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv, cSelfKV.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv_gr, cSelfKVGradient.GetIndex()) setBuffer(kernel, def_k_mhflatg_logsumexp, cSelfLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflatg_out, out.GetIndex()) setBuffer(kernel, def_k_mhflatg_out_gr, out_gr.GetIndex()) setArgument(kernel, def_k_mhflatg_dimension, (int)iHeadDim) setArgument(kernel, def_k_mhflatg_total_q, 1) setArgument(kernel, def_k_mhflatg_total_kv, (int)iVariables) setArgument(kernel, def_k_mhflatg_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!query_gr.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the mean flash scores step of the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::MeanFlashScores(CBufferFloat *query) { if(!OpenCL || !query || iMemoryCount == 0) ReturnFalse; //--- uint kernel = def_k_CogDriverRankTCMMeanFlashScore; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iMemoryCount, iVariables}; setBuffer(kernel, def_k_cdrtcm_score_query, query.GetIndex()) setBuffer(kernel, def_k_cdrtcm_score_kv, cMemoryKV.GetIndex()) setBuffer(kernel, def_k_cdrtcm_score_valid, cMemoryValid.GetIndex()) setBuffer(kernel, def_k_cdrtcm_score_out, cMemoryScore.GetIndex()) setBuffer(kernel, def_k_cdrtcm_score_logsumexp, cLogSumExp.GetIndex()) setArgument(kernel, def_k_cdrtcm_score_memory_count, (int)iMemoryCount) setArgument(kernel, def_k_cdrtcm_score_variables, (int)iVariables) setArgument(kernel, def_k_cdrtcm_score_heads, (int)iHeads) setArgument(kernel, def_k_cdrtcm_score_head_dim, (int)iHeadDim) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cMemoryScore.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCogDriverRankTCM feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < int(iInputDim * iVariables)) ReturnFalse; //--- Step 1: deferred memory mutation from the previous pass if(!InsertPendingQuery()) ReturnFalse; //--- Step 2: current query if(!cQueryProjection.FeedForward(NeuronOCL)) ReturnFalse; CBufferFloat *query = cQueryProjection.getOutput(); if(!query) ReturnFalse; //--- Step 3-5: attention, score, and fusion if(iMemoryCount == 0) { if(!SelfAttentionForward(query, Output)) ReturnFalse; bLastQueryValid = true; return(true); } if(!BuildMemoryKV()) ReturnFalse; if(!FlashAttentionForward(query, cMemoryContext.getOutput())) ReturnFalse; if(!SelfAttentionForward(cMemoryContext.getOutput(), cSelfContext.getOutput())) ReturnFalse; if(!MeanFlashScores(query)) ReturnFalse; if(!Concat(query, cSelfContext.getOutput(), cGateInput.getOutput(), (int)iTotalQueryDim, (int)iTotalQueryDim, 1)) ReturnFalse; if(!cGateProjection.FeedForward(cGateInput.AsObject())) ReturnFalse; if(!GateElementMult(cSelfContext.getOutput(), query, cGateProjection.getOutput(), Output)) ReturnFalse; //--- cQueryProjection output is the pending query for the next forward pass. bLastQueryValid = true; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; CBufferFloat *query = cQueryProjection.getOutput(); CBufferFloat *query_gr = cQueryProjection.getGradient(); if(!query || !query_gr) ReturnFalse; if(!query_gr.Fill(0)) ReturnFalse; //--- if(iMemoryCount == 0) { if(!cSelfKVGradient.Fill(0)) ReturnFalse; if(!SelfAttentionBackward(query, query_gr, Output, Gradient)) ReturnFalse; } else { if(!cMemoryContext.getGradient().Fill(0) || !cSelfContext.getGradient().Fill(0) || !cGateProjection.getGradient().Fill(0) || !cGateInput.getGradient().Fill(0)) ReturnFalse; if(!GateElementMultGrad(cSelfContext.getOutput(), cSelfContext.getGradient(), query, query_gr, cGateProjection.getOutput(), cGateProjection.getGradient(), Gradient, None, None, SIGMOID)) ReturnFalse; if(!cGateInput.CalcHiddenGradients(cGateProjection.AsObject())) ReturnFalse; if(!DeConcat(GetPointer(cGateQueryGradient), GetPointer(cGateContextGradient), cGateInput.getGradient(), (int)iTotalQueryDim, (int)iTotalQueryDim, 1)) ReturnFalse; if(!SumAndNormalize(query_gr, GetPointer(cGateQueryGradient), query_gr, (int)iTotalQueryDim, false, 0, 0, 0, 1.0f)) ReturnFalse; if(!SumAndNormalize(cSelfContext.getGradient(), GetPointer(cGateContextGradient), cSelfContext.getGradient(), (int)iTotalQueryDim, false, 0, 0, 0, 1.0f)) ReturnFalse; CBufferFloat *gate_gr = query_gr; CBufferFloat *flash_gr = GetPointer(cQueryGateGradient); if(!flash_gr || !flash_gr.Fill(0)) ReturnFalse; //--- Keep the query projection owner intact: batch-one getters alias cConv. if(!cSelfKVGradient.Fill(0)) ReturnFalse; if(!SelfAttentionBackward(cMemoryContext.getOutput(), cMemoryContext.getGradient(), cSelfContext.getOutput(), cSelfContext.getGradient())) ReturnFalse; if(!cMemoryKVGradient.Fill(0)) ReturnFalse; if(!FlashAttentionBackward(query, flash_gr, cMemoryContext.getOutput(), cMemoryContext.getGradient())) ReturnFalse; if(!SumAndNormalize(flash_gr, gate_gr, gate_gr, (int)iTotalQueryDim, false, 0, 0, 0, 1.0f)) ReturnFalse; query_gr = gate_gr; } //--- if(!NeuronOCL.CalcHiddenGradients(cQueryProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCogDriverRankTCM trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cGateProjection.UpdateInputWeights(cGateInput.AsObject())) ReturnFalse; if(!cQueryProjection.UpdateInputWeights(NeuronOCL.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCogDriverRankTCM state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iInputDim); dFileWriteUInt(file_handle, iQueryDim); dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iHeadDim); dFileWriteUInt(file_handle, iTotalQueryDim); dFileWriteUInt(file_handle, iTotalHeads); dFileWriteUInt(file_handle, iMemoryLen); dFileWriteUInt(file_handle, iMemoryCount); dFileWriteUInt(file_handle, uint(bLastQueryValid)); if(!cQueryProjection.Save(file_handle)) ReturnFalse; if(!cGateInput.Save(file_handle)) ReturnFalse; if(!cGateProjection.Save(file_handle)) ReturnFalse; if(!cMemoryContext.Save(file_handle)) ReturnFalse; if(!cSelfContext.Save(file_handle)) ReturnFalse; if(!cMemoryQuery.Save(file_handle) || !cMemoryAge.Save(file_handle) || !cMemoryScore.Save(file_handle) || !cMemoryValid.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Append trainable RankTCM projections only. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronBaseOCL::AppendParameterFingerprint(fingerprint) || !cQueryProjection.AppendParameterFingerprint(fingerprint) || !cGateProjection.AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCogDriverRankTCM state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; uint last_valid = 0; dFileReadUInt(file_handle, iInputDim); dFileReadUInt(file_handle, iQueryDim); dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iHeadDim); dFileReadUInt(file_handle, iTotalQueryDim); dFileReadUInt(file_handle, iTotalHeads); dFileReadUInt(file_handle, iMemoryLen); dFileReadUInt(file_handle, iMemoryCount); dFileReadUInt(file_handle, last_valid); if(OpenCL && OpenCL.InferenceRuntimeLoadActive()) { if(last_valid > 1 || iInputDim == 0 || iQueryDim == 0 || iVariables == 0 || iHeads == 0 || iMemoryLen == 0 || iQueryDim % iHeads != 0 || iHeadDim != iQueryDim / iHeads || (ulong)iVariables * iQueryDim != iTotalQueryDim || (ulong)iVariables * iHeads != iTotalHeads || iMemoryCount > iMemoryLen || 2 * (ulong)iMemoryLen * iTotalQueryDim > (ulong)INT_MAX || 2 * (ulong)iVariables * iTotalQueryDim > (ulong)INT_MAX) ReturnFalse; } bLastQueryValid = (last_valid != 0); if(!LoadInsideLayer(file_handle, cQueryProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGateInput.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGateProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMemoryContext.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfContext.AsObject())) ReturnFalse; if(!(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cMemoryQuery.LoadInferenceFinite(file_handle, OpenCL) : cMemoryQuery.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cMemoryAge.LoadInferenceFinite(file_handle, OpenCL) : cMemoryAge.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cMemoryScore.LoadInferenceFinite(file_handle, OpenCL) : cMemoryScore.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cMemoryValid.LoadInferenceFinite(file_handle, OpenCL) : cMemoryValid.Load(file_handle))) ReturnFalse; //--- transient buffers are rebuilt from dimensions if(!cMemoryKV.BufferInit(2 * iMemoryLen * iTotalQueryDim, 0) || !cMemoryKV.BufferCreate(OpenCL)) ReturnFalse; if(!cMemoryKVGradient.BufferInit(2 * iMemoryLen * iTotalQueryDim, 0) || !cMemoryKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfKV.BufferInit(2 * iVariables * iTotalQueryDim, 0) || !cSelfKV.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfKVGradient.BufferInit(2 * iVariables * iTotalQueryDim, 0) || !cSelfKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cLogSumExp.BufferInit(iTotalHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; if(!cSelfLogSumExp.BufferInit(iTotalHeads, 0) || !cSelfLogSumExp.BufferCreate(OpenCL)) ReturnFalse; if(!cQueryGateGradient.BufferInit(iTotalQueryDim, 0) || !cQueryGateGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cGateQueryGradient.BufferInit(iTotalQueryDim, 0) || !cGateQueryGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cGateContextGradient.BufferInit(iTotalQueryDim, 0) || !cGateContextGradient.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCogDriverRankTCM GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronCogDriverRankTCM::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cQueryProjection.SetOpenCL(OpenCL); cGateInput.SetOpenCL(OpenCL); cGateProjection.SetOpenCL(OpenCL); cMemoryContext.SetOpenCL(OpenCL); cSelfContext.SetOpenCL(OpenCL); cMemoryQuery.BufferCreate(OpenCL); cMemoryAge.BufferCreate(OpenCL); cMemoryScore.BufferCreate(OpenCL); cMemoryValid.BufferCreate(OpenCL); cMemoryKV.BufferCreate(OpenCL); cMemoryKVGradient.BufferCreate(OpenCL); cSelfKV.BufferCreate(OpenCL); cSelfKVGradient.BufferCreate(OpenCL); cLogSumExp.BufferCreate(OpenCL); cSelfLogSumExp.BufferCreate(OpenCL); cQueryGateGradient.BufferCreate(OpenCL); cGateQueryGradient.BufferCreate(OpenCL); cGateContextGradient.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Checks the complete RankTCM ownership tree. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::SupportsOpenCLChecked(void) const { return(Type() == defNeuronCogDriverRankTCM && cQueryProjection.SupportsOpenCLChecked() && cGateInput.SupportsOpenCLChecked() && cGateProjection.SupportsOpenCLChecked() && cMemoryContext.SupportsOpenCLChecked() && cSelfContext.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers RankTCM state before publishing the base context. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::SetOpenCLChecked(COpenCLMy *obj) { //--- Reject unsupported ownership before changing any resource. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or RankTCM tree"); //--- Preserve foreign direct state before any buffer is recreated. if(cMemoryQuery.Total() > 0 && cMemoryQuery.GetIndex() >= 0 && cMemoryQuery.GetOpenCL() != obj && !cMemoryQuery.BufferRead()) ReturnFalseEx("cMemoryQuery read"); if(cMemoryAge.Total() > 0 && cMemoryAge.GetIndex() >= 0 && cMemoryAge.GetOpenCL() != obj && !cMemoryAge.BufferRead()) ReturnFalseEx("cMemoryAge read"); if(cMemoryScore.Total() > 0 && cMemoryScore.GetIndex() >= 0 && cMemoryScore.GetOpenCL() != obj && !cMemoryScore.BufferRead()) ReturnFalseEx("cMemoryScore read"); if(cMemoryValid.Total() > 0 && cMemoryValid.GetIndex() >= 0 && cMemoryValid.GetOpenCL() != obj && !cMemoryValid.BufferRead()) ReturnFalseEx("cMemoryValid read"); if(cMemoryKV.Total() > 0 && cMemoryKV.GetIndex() >= 0 && cMemoryKV.GetOpenCL() != obj && !cMemoryKV.BufferRead()) ReturnFalseEx("cMemoryKV read"); if(cMemoryKVGradient.Total() > 0 && cMemoryKVGradient.GetIndex() >= 0 && cMemoryKVGradient.GetOpenCL() != obj && !cMemoryKVGradient.BufferRead()) ReturnFalseEx("cMemoryKVGradient read"); if(cSelfKV.Total() > 0 && cSelfKV.GetIndex() >= 0 && cSelfKV.GetOpenCL() != obj && !cSelfKV.BufferRead()) ReturnFalseEx("cSelfKV read"); if(cSelfKVGradient.Total() > 0 && cSelfKVGradient.GetIndex() >= 0 && cSelfKVGradient.GetOpenCL() != obj && !cSelfKVGradient.BufferRead()) ReturnFalseEx("cSelfKVGradient read"); if(cLogSumExp.Total() > 0 && cLogSumExp.GetIndex() >= 0 && cLogSumExp.GetOpenCL() != obj && !cLogSumExp.BufferRead()) ReturnFalseEx("cLogSumExp read"); if(cSelfLogSumExp.Total() > 0 && cSelfLogSumExp.GetIndex() >= 0 && cSelfLogSumExp.GetOpenCL() != obj && !cSelfLogSumExp.BufferRead()) ReturnFalseEx("cSelfLogSumExp read"); if(cQueryGateGradient.Total() > 0 && cQueryGateGradient.GetIndex() >= 0 && cQueryGateGradient.GetOpenCL() != obj && !cQueryGateGradient.BufferRead()) ReturnFalseEx("cQueryGateGradient read"); if(cGateQueryGradient.Total() > 0 && cGateQueryGradient.GetIndex() >= 0 && cGateQueryGradient.GetOpenCL() != obj && !cGateQueryGradient.BufferRead()) ReturnFalseEx("cGateQueryGradient read"); if(cGateContextGradient.Total() > 0 && cGateContextGradient.GetIndex() >= 0 && cGateContextGradient.GetOpenCL() != obj && !cGateContextGradient.BufferRead()) ReturnFalseEx("cGateContextGradient read"); //--- Move nested owners while the source context remains alive. if(!cQueryProjection.SetOpenCLChecked(obj)) ReturnFalseEx("cQueryProjection context"); if(!cGateInput.SetOpenCLChecked(obj)) ReturnFalseEx("cGateInput context"); if(!cGateProjection.SetOpenCLChecked(obj)) ReturnFalseEx("cGateProjection context"); if(!cMemoryContext.SetOpenCLChecked(obj)) ReturnFalseEx("cMemoryContext context"); if(!cSelfContext.SetOpenCLChecked(obj)) ReturnFalseEx("cSelfContext context"); //--- Preserve already-target GPU state when retrying a partial transfer. if(cMemoryQuery.Total() > 0 && (cMemoryQuery.GetOpenCL() != obj || cMemoryQuery.GetIndex() < 0) && !cMemoryQuery.BufferCreate(obj)) ReturnFalseEx("cMemoryQuery create"); if(cMemoryAge.Total() > 0 && (cMemoryAge.GetOpenCL() != obj || cMemoryAge.GetIndex() < 0) && !cMemoryAge.BufferCreate(obj)) ReturnFalseEx("cMemoryAge create"); if(cMemoryScore.Total() > 0 && (cMemoryScore.GetOpenCL() != obj || cMemoryScore.GetIndex() < 0) && !cMemoryScore.BufferCreate(obj)) ReturnFalseEx("cMemoryScore create"); if(cMemoryValid.Total() > 0 && (cMemoryValid.GetOpenCL() != obj || cMemoryValid.GetIndex() < 0) && !cMemoryValid.BufferCreate(obj)) ReturnFalseEx("cMemoryValid create"); if(cMemoryKV.Total() > 0 && (cMemoryKV.GetOpenCL() != obj || cMemoryKV.GetIndex() < 0) && !cMemoryKV.BufferCreate(obj)) ReturnFalseEx("cMemoryKV create"); if(cMemoryKVGradient.Total() > 0 && (cMemoryKVGradient.GetOpenCL() != obj || cMemoryKVGradient.GetIndex() < 0) && !cMemoryKVGradient.BufferCreate(obj)) ReturnFalseEx("cMemoryKVGradient create"); if(cSelfKV.Total() > 0 && (cSelfKV.GetOpenCL() != obj || cSelfKV.GetIndex() < 0) && !cSelfKV.BufferCreate(obj)) ReturnFalseEx("cSelfKV create"); if(cSelfKVGradient.Total() > 0 && (cSelfKVGradient.GetOpenCL() != obj || cSelfKVGradient.GetIndex() < 0) && !cSelfKVGradient.BufferCreate(obj)) ReturnFalseEx("cSelfKVGradient create"); if(cLogSumExp.Total() > 0 && (cLogSumExp.GetOpenCL() != obj || cLogSumExp.GetIndex() < 0) && !cLogSumExp.BufferCreate(obj)) ReturnFalseEx("cLogSumExp create"); if(cSelfLogSumExp.Total() > 0 && (cSelfLogSumExp.GetOpenCL() != obj || cSelfLogSumExp.GetIndex() < 0) && !cSelfLogSumExp.BufferCreate(obj)) ReturnFalseEx("cSelfLogSumExp create"); if(cQueryGateGradient.Total() > 0 && (cQueryGateGradient.GetOpenCL() != obj || cQueryGateGradient.GetIndex() < 0) && !cQueryGateGradient.BufferCreate(obj)) ReturnFalseEx("cQueryGateGradient create"); if(cGateQueryGradient.Total() > 0 && (cGateQueryGradient.GetOpenCL() != obj || cGateQueryGradient.GetIndex() < 0) && !cGateQueryGradient.BufferCreate(obj)) ReturnFalseEx("cGateQueryGradient create"); if(cGateContextGradient.Total() > 0 && (cGateContextGradient.GetOpenCL() != obj || cGateContextGradient.GetIndex() < 0) && !cGateContextGradient.BufferCreate(obj)) ReturnFalseEx("cGateContextGradient create"); //--- Publish the inherited context only after all RankTCM owners succeed. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("inherited base context"); //--- Retain the pending query and memory count for continued execution. return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronCogDriverRankTCM layer. | //+------------------------------------------------------------------+ void CNeuronCogDriverRankTCM::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cQueryProjection.TrainMode(bTrain); cGateInput.TrainMode(bTrain); cGateProjection.TrainMode(bTrain); cMemoryContext.TrainMode(bTrain); cSelfContext.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronCogDriverRankTCM weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronCogDriverRankTCM *Source = source; if(!cQueryProjection.WeightsUpdate(Source.cQueryProjection.AsObject(), tau)) ReturnFalse; if(!cGateInput.WeightsUpdate(Source.cGateInput.AsObject(), tau)) ReturnFalse; if(!cGateProjection.WeightsUpdate(Source.cGateProjection.AsObject(), tau)) ReturnFalse; if(!cMemoryContext.WeightsUpdate(Source.cMemoryContext.AsObject(), tau)) ReturnFalse; if(!cSelfContext.WeightsUpdate(Source.cSelfContext.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronCogDriverRankTCM execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverRankTCM::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cQueryProjection.Clear()) ReturnFalse; if(!cGateInput.Clear()) ReturnFalse; if(!cGateProjection.Clear()) ReturnFalse; if(!cMemoryContext.Clear()) ReturnFalse; if(!cSelfContext.Clear()) ReturnFalse; if(!cMemoryQuery.Fill(0) || !cMemoryAge.Fill(0) || !cMemoryScore.Fill(0) || !cMemoryValid.Fill(0) || !cMemoryKV.Fill(0) || !cMemoryKVGradient.Fill(0) || !cSelfKV.Fill(0) || !cSelfKVGradient.Fill(0) || !cLogSumExp.Fill(0) || !cSelfLogSumExp.Fill(0) || !cQueryGateGradient.Fill(0) || !cGateQueryGradient.Fill(0) || !cGateContextGradient.Fill(0)) ReturnFalse; iMemoryCount = 0; bLastQueryValid = false; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the shift state step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::ShiftState(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastShiftState; uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits * iWindow}; setBuffer(kernel, def_k_cdfh_shift_plan, Output.GetIndex()) setBuffer(kernel, def_k_cdfh_shift_confidence, cConfidence.GetIndex()) setBuffer(kernel, def_k_cdfh_shift_shifted_plan, cPlanInput.getOutputIndex()) setBuffer(kernel, def_k_cdfh_shift_shifted_confidence, cShiftedConfidence.GetIndex()) setBuffer(kernel, def_k_cdfh_shift_delta_mask, cDeltaMask.GetIndex()) setArgument(kernel, def_k_cdfh_shift_query_dim, (int)iWindow) setArgument(kernel, def_k_cdfh_shift_horizon, (int)iUnits) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cPlanInput.getOutput().BufferRead() || !cShiftedConfidence.BufferRead() || !cDeltaMask.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build context kv step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildContextKV(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastBuildKV; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iVariables, iUnits, iWindow}; setBuffer(kernel, def_k_cdfh_kv_context, cContextProjection.getOutputIndex()) setBuffer(kernel, def_k_cdfh_kv_out, cContextKV.GetIndex()) setArgument(kernel, def_k_cdfh_kv_dimension, (int)iWindow) setArgument(kernel, def_k_cdfh_kv_horizon, (int)iUnits) setArgument(kernel, def_k_cdfh_kv_variables, (int)iVariables) setArgument(kernel, def_k_cdfh_kv_heads, (int)iHeads) setArgument(kernel, def_k_cdfh_kv_head_dim, (int)iWindowKey) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cContextKV.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build context projection gradient step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildContextProjectionGradient(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastBuildKVGrad; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iVariables, iUnits, iWindow}; setBuffer(kernel, def_k_cdfh_kvg_kv_gr, cContextKVGradient.GetIndex()) setBuffer(kernel, def_k_cdfh_kvg_context_gr, cContextProjection.getGradientIndex()) setArgument(kernel, def_k_cdfh_kvg_dimension, (int)iWindow) setArgument(kernel, def_k_cdfh_kvg_horizon, (int)iUnits) setArgument(kernel, def_k_cdfh_kvg_variables, (int)iVariables) setArgument(kernel, def_k_cdfh_kvg_heads, (int)iHeads) setArgument(kernel, def_k_cdfh_kvg_head_dim, (int)iWindowKey) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cContextProjection.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build error input step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildErrorInput(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < int(iVariables * iWindow)) ReturnFalse; if(!ConcatVecMatrix(NeuronOCL.getOutput(), Output, cErrorInput.getOutput(), iVariables * iWindow, iUnits, iWindow, 1, false)) ReturnFalse; //--- return(cErrorHead.FeedForward(cErrorInput.AsObject())); } //+------------------------------------------------------------------+ //| Executes the build error target step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildErrorTarget(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastGradientRMS; uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits}; setBuffer(kernel, def_k_cdfh_grms_gradient, getGradientIndex()) setBuffer(kernel, def_k_cdfh_grms_error_target, cErrorTarget.GetIndex()) setArgument(kernel, def_k_cdfh_grms_dimension, (int)iWindow) setArgument(kernel, def_k_cdfh_grms_horizon, (int)iUnits) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cErrorTarget.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build confidence step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildConfidence(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastConfidence; uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits}; setBuffer(kernel, def_k_cdfh_conf_expected_error, cErrorHead.getOutputIndex()) setBuffer(kernel, def_k_cdfh_conf_confidence, cConfidence.GetIndex()) setArgument(kernel, def_k_cdfh_conf_total, (int)iUnits) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cConfidence.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Executes the build delta norm step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::BuildDeltaNorm(void) { if(!OpenCL) ReturnFalse; //--- uint kernel = def_k_CogDriverForecastDeltaNorm; uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits}; setBuffer(kernel, def_k_cdfh_delta_plan, Output.GetIndex()) setBuffer(kernel, def_k_cdfh_delta_shifted_plan, cPlanInput.getOutputIndex()) setBuffer(kernel, def_k_cdfh_delta_plan_delta, cPlanDelta.GetIndex()) setBuffer(kernel, def_k_cdfh_delta_plan_delta_norm, cPlanDeltaNorm.GetIndex()) setArgument(kernel, def_k_cdfh_delta_query_dim, (int)iWindow) setArgument(kernel, def_k_cdfh_delta_horizon, (int)iUnits) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cPlanDelta.BufferRead() || !cPlanDeltaNorm.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCogDriverForecastHead layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint horizon, uint variables, uint forecast_blocks, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension == 0 || horizon == 0 || variables == 0 || forecast_blocks == 0 || heads == 0 || dimension % heads != 0) ReturnFalse; uint head_dim = dimension / heads; if(!CNeuronMLMHAttentionOCL::Init(numOutputs, myIndex, open_cl, dimension, head_dim, heads, horizon, forecast_blocks, optimization_type, batch)) ReturnFalse; //--- iVariables = variables; SetActivationFunction(None); //--- wrappers and trainable context projection if(!cPlanInput.Init(0, 0, OpenCL, iUnits * iWindow, optimization, iBatch)) ReturnFalse; cPlanInput.SetActivationFunction(None); if(!cContextProjection.Init(0, 1, OpenCL, iWindow, iWindow, 2 * iUnits * iWindow, iVariables, optimization, iBatch)) ReturnFalse; cContextProjection.SetActivationFunction(None); if(!cQueryPE.Init(0, 2, OpenCL, iUnits * iWindow, optimization, iBatch)) ReturnFalse; cQueryPE.SetActivationFunction(None); //--- forecast blocks cCrossAttention.Clear(); cCrossResidual.Clear(); cCrossFF1.Clear(); cCrossFF2.Clear(); cCrossFFOutput.Clear(); cSelfOutput.Clear(); cCrossAttention.SetOpenCL(OpenCL); cCrossResidual.SetOpenCL(OpenCL); cCrossFF1.SetOpenCL(OpenCL); cCrossFF2.SetOpenCL(OpenCL); cCrossFFOutput.SetOpenCL(OpenCL); cSelfOutput.SetOpenCL(OpenCL); uint index = 3; uint plan_total = iUnits * iWindow; for(uint i = 0; i < iLayers; i++) { CCrossMHFlashAttention *cross = new CCrossMHFlashAttention(); if(!cross || !cross.Init(0, index++, OpenCL, iWindowKey, iHeads, false, iUnits, iVariables * iUnits, optimization, iBatch) || !cCrossAttention.Add(cross)) DeleteObjAndFalse(cross); CNeuronBaseOCL *holder = new CNeuronBaseOCL(); if(!holder || !holder.Init(0, index++, OpenCL, plan_total, optimization, iBatch) || !cCrossResidual.Add(holder)) DeleteObjAndFalse(holder); holder.SetActivationFunction(None); CNeuronConvOCL *ff1 = new CNeuronConvOCL(); if(!ff1 || !ff1.Init(0, index++, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization, iBatch) || !cCrossFF1.Add(ff1)) DeleteObjAndFalse(ff1); ff1.SetActivationFunction(GELU); CNeuronConvOCL *ff2 = new CNeuronConvOCL(); if(!ff2 || !ff2.Init(0, index++, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization, iBatch) || !cCrossFF2.Add(ff2)) DeleteObjAndFalse(ff2); ff2.SetActivationFunction(None); holder = new CNeuronBaseOCL(); if(!holder || !holder.Init(0, index++, OpenCL, plan_total, optimization, iBatch) || !cCrossFFOutput.Add(holder)) DeleteObjAndFalse(holder); holder.SetActivationFunction(None); holder = new CNeuronBaseOCL(); if(!holder || !holder.Init(0, index++, OpenCL, plan_total, optimization, iBatch) || !cSelfOutput.Add(holder)) DeleteObjAndFalse(holder); holder.SetActivationFunction(None); } uint error_width = (iVariables + 1) * iWindow; if(!cErrorInput.Init(0, index++, OpenCL, iUnits * error_width, optimization, iBatch)) ReturnFalse; cErrorInput.SetActivationFunction(None); if(!cErrorHead.Init(0, index++, OpenCL, error_width, error_width, 1, iUnits, optimization, iBatch)) ReturnFalse; cErrorHead.SetActivationFunction(SoftPlus); //--- state and diagnostics uint context_kv_total = 2 * iVariables * iUnits * iWindow; if(!cContextKV.BufferInit(context_kv_total, 0) || !cContextKV.BufferCreate(OpenCL)) ReturnFalse; if(!cContextKVGradient.BufferInit(context_kv_total, 0) || !cContextKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cShiftedConfidence.BufferInit(iUnits, 0) || !cShiftedConfidence.BufferCreate(OpenCL)) ReturnFalse; if(!cDeltaMask.BufferInit(iUnits, 0) || !cDeltaMask.BufferCreate(OpenCL)) ReturnFalse; if(!cPlanDelta.BufferInit(plan_total, 0) || !cPlanDelta.BufferCreate(OpenCL)) ReturnFalse; if(!cPlanDeltaNorm.BufferInit(iUnits, 0) || !cPlanDeltaNorm.BufferCreate(OpenCL)) ReturnFalse; if(!cConfidence.BufferInit(iUnits, 0) || !cConfidence.BufferCreate(OpenCL)) ReturnFalse; if(!cErrorTarget.BufferInit(iUnits, 0) || !cErrorTarget.BufferCreate(OpenCL)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Executes the cross layer forward step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::CrossLayerForward(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context) { if(!plan || !context || layer >= iLayers) ReturnFalse; CNeuronBaseOCL *query = plan; if(layer == 0) { if(!cQueryPE.FeedForward(plan)) ReturnFalse; query = cQueryPE.AsObject(); } CCrossMHFlashAttention *cross = cCrossAttention.At((int)layer); CNeuronBaseOCL *cross_res = cCrossResidual.At((int)layer); CNeuronConvOCL *ff1 = cCrossFF1.At((int)layer); CNeuronConvOCL *ff2 = cCrossFF2.At((int)layer); CNeuronBaseOCL *ff_out = cCrossFFOutput.At((int)layer); if(!cross || !cross_res || !ff1 || !ff2 || !ff_out) ReturnFalse; if(!cross.FeedForward(query, GetPointer(cContextKV))) ReturnFalse; if(!SumAndNormalize(cross.getOutput(), plan.getOutput(), cross_res.getOutput(), (int)iWindow)) ReturnFalse; if(!ff1.FeedForward(cross_res)) ReturnFalse; if(!ff2.FeedForward(ff1.AsObject())) ReturnFalse; if(!SumAndNormalize(ff2.getOutput(), cross_res.getOutput(), ff_out.getOutput(), (int)iWindow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the cross layer backward step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::CrossLayerBackward(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context, CBufferFloat *out_grad) { if(!plan || !context || !out_grad || layer >= iLayers) ReturnFalse; CNeuronConvOCL *ff1 = cCrossFF1.At((int)layer); CNeuronConvOCL *ff2 = cCrossFF2.At((int)layer); CNeuronBaseOCL *cross_res = cCrossResidual.At((int)layer); CCrossMHFlashAttention *cross = cCrossAttention.At((int)layer); if(!ff1 || !ff2 || !cross_res || !cross) ReturnFalse; CBufferFloat *ff2_gr = ff2.getGradient(); if(!ff2.SetGradient(out_grad, false)) ReturnFalse; if(!ff1.CalcHiddenGradients(ff2.AsObject())) { if(!ff2.SetGradient(ff2_gr, false)) ReturnFalse; ReturnFalse; } if(!ff2.SetGradient(ff2_gr, false)) ReturnFalse; if(!cross_res.CalcHiddenGradients(ff1.AsObject())) ReturnFalse; if(!SumAndNormalize(out_grad, cross_res.getGradient(), cross_res.getGradient(), (int)iWindow, false, 0, 0, 0, 1)) ReturnFalse; CBufferFloat *cross_gr = cross.getGradient(); if(!cross.SetGradient(cross_res.getGradient(), false)) ReturnFalse; if(!cContextKVGradient.Fill(0)) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } if(layer == 0) { if(!cPlanInput.getGradient().Fill(0)) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } if(!cQueryPE.CalcHiddenGradients(cross.AsObject(), GetPointer(cContextKV), GetPointer(cContextKVGradient), None)) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } if(!cPlanInput.CalcHiddenGradients(cQueryPE.AsObject())) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } } else { if(!plan.CalcHiddenGradients(cross, GetPointer(cContextKV), GetPointer(cContextKVGradient), None)) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } if(!SumAndNormalize(cross_res.getGradient(), plan.getGradient(), plan.getGradient(), (int)iWindow, false, 0, 0, 0, 1)) { if(!cross.SetGradient(cross_gr, false)) ReturnFalse; ReturnFalse; } } if(!cross.SetGradient(cross_gr, false)) ReturnFalse; //--- return(BuildContextProjectionGradient()); } //+------------------------------------------------------------------+ //| Executes the cross layer update weights step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::CrossLayerUpdateWeights(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context) { if(!plan || !context || layer >= iLayers) ReturnFalse; CNeuronBaseOCL *cross_res = cCrossResidual.At((int)layer); CNeuronConvOCL *ff1 = cCrossFF1.At((int)layer); CNeuronConvOCL *ff2 = cCrossFF2.At((int)layer); if(!cross_res || !ff1 || !ff2) ReturnFalse; if(layer == 0) if(!cQueryPE.UpdateInputWeights(cPlanInput.AsObject())) ReturnFalse; if(!ff1.UpdateInputWeights(cross_res)) ReturnFalse; if(!ff2.UpdateInputWeights(ff1.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the self layer forward step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::SelfLayerForward(uint layer, CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || layer >= iLayers) ReturnFalse; //--- Calculate Queries, Keys, Values CBufferFloat *inputs = NeuronOCL.getOutput(); CBufferFloat *qkv = QKV_Tensors.At(layer * 2); if(!ConvolutionForward(QKV_Weights.At(layer * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score and attention CBufferFloat *scores = S_Tensors.At(layer * 2); if(!AttentionScore(qkv, scores, false)) ReturnFalse; CBufferFloat *attn = AO_Tensors.At(layer * 2); if(!AttentionOut(qkv, scores, attn)) ReturnFalse; //--- Attention projection + residual/norm CBufferFloat *temp = FF_Tensors.At(layer * 6); if(!ConvolutionForward(FF_Weights.At(layer * (optimization == SGD ? 6 : 9)), attn, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; if(!SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Self feed-forward + residual/norm inputs = temp; temp = FF_Tensors.At(layer * 6 + 1); if(!ConvolutionForward(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; CBufferFloat *out = Output; if(layer + 1 < iLayers) { CNeuronBaseOCL *holder = cSelfOutput.At((int)layer); if(!holder) ReturnFalse; out = holder.getOutput(); } if(!ConvolutionForward(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; if(!SumAndNormalize(out, inputs, out, iWindow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the self layer backward step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::SelfLayerBackward(uint layer, CNeuronBaseOCL *NeuronOCL, CBufferFloat *out_grad) { if(!NeuronOCL || !out_grad || layer >= iLayers) ReturnFalse; //--- Passing gradient through self feed-forward layers if(!ConvolutionInputGradients(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(layer * 6 + 1), FF_Tensors.At(layer * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(layer * 6 + 3); if(!ConvolutionInputGradients(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(layer * 6 + 4), FF_Tensors.At(layer * 6), temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; if(!SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient through self attention if(!ConvolutionInputGradients(FF_Weights.At(layer * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(layer * 2), AO_Tensors.At(layer * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; if(!AttentionInsideGradients(QKV_Tensors.At(layer * 2), QKV_Tensors.At(layer * 2 + 1), S_Tensors.At(layer * 2), AO_Tensors.At(layer * 2 + 1))) ReturnFalse; temp = NeuronOCL.getGradient(); if(!ConvolutionInputGradients(QKV_Weights.At(layer * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(layer * 2 + 1), NeuronOCL.getOutput(), temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; if(!SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the self layer update weights step of the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::SelfLayerUpdateWeights(uint layer, CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || layer >= iLayers) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); if(!ConvolutuionUpdateWeights(QKV_Weights.At(layer * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(layer * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(layer * 2 + 1) : QKV_Weights.At(layer * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(layer * 3 + 2)), iWindow, 3 * iWindowKey * iHeads, 0, iHeads)) ReturnFalse; if(!ConvolutuionUpdateWeights(FF_Weights.At(layer * (optimization == SGD ? 6 : 9)), FF_Tensors.At(layer * 6 + 3), AO_Tensors.At(layer * 2), (optimization == SGD ? FF_Weights.At(layer * 6 + 3) : FF_Weights.At(layer * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(layer * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; if(!ConvolutuionUpdateWeights(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(layer * 6 + 4), FF_Tensors.At(layer * 6), (optimization == SGD ? FF_Weights.At(layer * 6 + 4) : FF_Weights.At(layer * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(layer * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; CBufferFloat *ff2_out_gr = Gradient; if(layer + 1 < iLayers) { CNeuronBaseOCL *holder = cSelfOutput.At((int)layer); if(!holder) ReturnFalse; ff2_out_gr = holder.getGradient(); } if(!ConvolutuionUpdateWeights(FF_Weights.At(layer * (optimization == SGD ? 6 : 9) + 2), ff2_out_gr, FF_Tensors.At(layer * 6 + 1), (optimization == SGD ? FF_Weights.At(layer * 6 + 5) : FF_Weights.At(layer * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(layer * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCogDriverForecastHead feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iVariables * iWindow)) ReturnFalse; //--- recurrent plan state is captured before Output is overwritten if(!ShiftState()) ReturnFalse; //--- variables x dimension context -> trainable K/V tokens for variables * horizon if(!cContextProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!BuildContextKV()) ReturnFalse; //--- forecast transformer blocks CNeuronBaseOCL *plan = cPlanInput.AsObject(); CNeuronBaseOCL *context = cContextProjection.AsObject(); for(uint i = 0; i < iLayers; i++) { CNeuronBaseOCL *ff_out = cCrossFFOutput.At((int)i); if(!ff_out) ReturnFalse; if(!CrossLayerForward(i, plan, context)) ReturnFalse; if(!SelfLayerForward(i, ff_out)) ReturnFalse; if(i + 1 < iLayers) plan = cSelfOutput.At((int)i); } //--- side outputs if(!BuildErrorInput(NeuronOCL)) ReturnFalse; if(!BuildConfidence()) ReturnFalse; if(!BuildDeltaNorm()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCogDriverForecastHead layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- expected-error head learns decoder-gradient magnitude without feeding it back into z_pred float error = 0; if(!BuildErrorTarget()) ReturnFalse; if(!cErrorHead.calcOutputGradients(GetPointer(cErrorTarget), error)) ReturnFalse; //--- gradients through forecast blocks; shifted plan branch is detached if(!cContextProjection.getGradient().Fill(0)) ReturnFalse; CNeuronBaseOCL *context = cContextProjection.AsObject(); for(int i = int(iLayers) - 1; i >= 0; i--) { CNeuronBaseOCL *self_input = cCrossFFOutput.At(i); CNeuronBaseOCL *self_out = cSelfOutput.At(i); if(!self_input || !self_out) ReturnFalse; CBufferFloat *out_grad = (i == int(iLayers) - 1 ? Gradient : self_out.getGradient()); if(!SelfLayerBackward((uint)i, self_input, out_grad)) ReturnFalse; CNeuronBaseOCL *plan = (i == 0 ? cPlanInput.AsObject() : cSelfOutput.At(i - 1)); if(!CrossLayerBackward((uint)i, plan, context, self_input.getGradient())) ReturnFalse; } //--- accumulated KV gradient -> projection gradient -> RankTCM output gradient if(!NeuronOCL.CalcHiddenGradients(cContextProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCogDriverForecastHead trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *plan = cPlanInput.AsObject(); CNeuronBaseOCL *context = cContextProjection.AsObject(); for(uint i = 0; i < iLayers; i++) { CNeuronBaseOCL *ff_out = cCrossFFOutput.At((int)i); if(!ff_out) ReturnFalse; if(!CrossLayerUpdateWeights(i, plan, context)) ReturnFalse; if(!SelfLayerUpdateWeights(i, ff_out)) ReturnFalse; if(i + 1 < iLayers) plan = cSelfOutput.At((int)i); } if(!cContextProjection.UpdateInputWeights(NeuronOCL.AsObject())) ReturnFalse; if(!cErrorHead.UpdateInputWeights(cErrorInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCogDriverForecastHead state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iVariables); if(!cPlanInput.Save(file_handle) || !cContextProjection.Save(file_handle) || !cQueryPE.Save(file_handle) || !cCrossAttention.Save(file_handle) || !cCrossResidual.Save(file_handle) || !cCrossFF1.Save(file_handle) || !cCrossFF2.Save(file_handle) || !cCrossFFOutput.Save(file_handle) || !cSelfOutput.Save(file_handle) || !cErrorInput.Save(file_handle) || !cErrorHead.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCogDriverForecastHead state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iVariables); if(!LoadInsideLayer(file_handle, cPlanInput.AsObject()) || !LoadInsideLayer(file_handle, cContextProjection.AsObject()) || !LoadInsideLayer(file_handle, cQueryPE.AsObject()) || !cCrossAttention.Load(file_handle) || !cCrossResidual.Load(file_handle) || !cCrossFF1.Load(file_handle) || !cCrossFF2.Load(file_handle) || !cCrossFFOutput.Load(file_handle) || !cSelfOutput.Load(file_handle) || !LoadInsideLayer(file_handle, cErrorInput.AsObject()) || !LoadInsideLayer(file_handle, cErrorHead.AsObject())) ReturnFalse; cPlanInput.SetOpenCL(OpenCL); cContextProjection.SetOpenCL(OpenCL); cQueryPE.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cCrossResidual.SetOpenCL(OpenCL); cCrossFF1.SetOpenCL(OpenCL); cCrossFF2.SetOpenCL(OpenCL); cCrossFFOutput.SetOpenCL(OpenCL); cSelfOutput.SetOpenCL(OpenCL); cErrorInput.SetOpenCL(OpenCL); cErrorHead.SetOpenCL(OpenCL); //--- transient state is rebuilt from dimensions uint plan_total = iUnits * iWindow; uint context_kv_total = 2 * iVariables * iUnits * iWindow; if(!cContextKV.BufferInit(context_kv_total, 0) || !cContextKV.BufferCreate(OpenCL)) ReturnFalse; if(!cContextKVGradient.BufferInit(context_kv_total, 0) || !cContextKVGradient.BufferCreate(OpenCL)) ReturnFalse; if(!cShiftedConfidence.BufferInit(iUnits, 0) || !cShiftedConfidence.BufferCreate(OpenCL)) ReturnFalse; if(!cDeltaMask.BufferInit(iUnits, 0) || !cDeltaMask.BufferCreate(OpenCL)) ReturnFalse; if(!cPlanDelta.BufferInit(plan_total, 0) || !cPlanDelta.BufferCreate(OpenCL)) ReturnFalse; if(!cPlanDeltaNorm.BufferInit(iUnits, 0) || !cPlanDeltaNorm.BufferCreate(OpenCL)) ReturnFalse; if(!cConfidence.BufferInit(iUnits, 0) || !cConfidence.BufferCreate(OpenCL)) ReturnFalse; if(!cErrorTarget.BufferInit(iUnits, 0) || !cErrorTarget.BufferCreate(OpenCL)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCogDriverForecastHead GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronCogDriverForecastHead::SetOpenCL(COpenCLMy *obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cPlanInput.SetOpenCL(OpenCL); cContextProjection.SetOpenCL(OpenCL); cQueryPE.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cCrossResidual.SetOpenCL(OpenCL); cCrossFF1.SetOpenCL(OpenCL); cCrossFF2.SetOpenCL(OpenCL); cCrossFFOutput.SetOpenCL(OpenCL); cSelfOutput.SetOpenCL(OpenCL); cErrorInput.SetOpenCL(OpenCL); cErrorHead.SetOpenCL(OpenCL); cContextKV.BufferCreate(OpenCL); cContextKVGradient.BufferCreate(OpenCL); cShiftedConfidence.BufferCreate(OpenCL); cDeltaMask.BufferCreate(OpenCL); cPlanDelta.BufferCreate(OpenCL); cPlanDeltaNorm.BufferCreate(OpenCL); cConfidence.BufferCreate(OpenCL); cErrorTarget.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronCogDriverForecastHead | //| layer. | //+------------------------------------------------------------------+ void CNeuronCogDriverForecastHead::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cPlanInput.TrainMode(bTrain); cContextProjection.TrainMode(bTrain); cQueryPE.TrainMode(bTrain); cCrossAttention.TrainMode(bTrain); cCrossResidual.TrainMode(bTrain); cCrossFF1.TrainMode(bTrain); cCrossFF2.TrainMode(bTrain); cCrossFFOutput.TrainMode(bTrain); cSelfOutput.TrainMode(bTrain); cErrorInput.TrainMode(bTrain); cErrorHead.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronCogDriverForecastHead weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; if(!CNeuronMLMHAttentionOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronCogDriverForecastHead *Source = source; if(!cPlanInput.WeightsUpdate(Source.cPlanInput.AsObject(), tau) || !cContextProjection.WeightsUpdate(Source.cContextProjection.AsObject(), tau) || !cQueryPE.WeightsUpdate(Source.cQueryPE.AsObject(), tau) || !cCrossAttention.WeightsUpdate(GetPointer(Source.cCrossAttention), tau) || !cCrossResidual.WeightsUpdate(GetPointer(Source.cCrossResidual), tau) || !cCrossFF1.WeightsUpdate(GetPointer(Source.cCrossFF1), tau) || !cCrossFF2.WeightsUpdate(GetPointer(Source.cCrossFF2), tau) || !cCrossFFOutput.WeightsUpdate(GetPointer(Source.cCrossFFOutput), tau) || !cSelfOutput.WeightsUpdate(GetPointer(Source.cSelfOutput), tau) || !cErrorInput.WeightsUpdate(Source.cErrorInput.AsObject(), tau) || !cErrorHead.WeightsUpdate(Source.cErrorHead.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronCogDriverForecastHead execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastHead::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cPlanInput.Clear() || !cContextProjection.Clear() || !cQueryPE.Clear() || !cCrossAttention.ClearStates() || !cCrossResidual.ClearStates() || !cCrossFF1.ClearStates() || !cCrossFF2.ClearStates() || !cCrossFFOutput.ClearStates() || !cSelfOutput.ClearStates() || !cErrorInput.Clear() || !cErrorHead.Clear()) ReturnFalse; if(!cContextKV.Fill(0) || !cContextKVGradient.Fill(0) || !cShiftedConfidence.Fill(0) || !cDeltaMask.Fill(0) || !cPlanDelta.Fill(0) || !cPlanDeltaNorm.Fill(0) || !cConfidence.Fill(0) || !cErrorTarget.Fill(0)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCogDriverForecastToken layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint horizon, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension == 0 || horizon == 0 || dimension % 2 != 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, horizon * (dimension + 1), optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iHorizon = horizon; SetActivationFunction(None); if(!cRoPE.Init(0, myIndex + 1, OpenCL, iHorizon, iDimension, 1, optimization, iBatch)) ReturnFalse; cRoPE.SetActivationFunction(None); if(!cConfidenceGradient.BufferInit(iHorizon, 0) || !cConfidenceGradient.BufferCreate(OpenCL)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Runs the CNeuronCogDriverForecastToken feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Type() != defNeuronCogDriverForecastHead || NeuronOCL.Neurons() != int(iHorizon * iDimension)) ReturnFalse; //--- CNeuronCogDriverForecastHead *forecast = NeuronOCL; CBufferFloat *confidence = forecast.GetConfidence(); if(!confidence || confidence.Total() < int(iHorizon)) ReturnFalse; if(!cRoPE.FeedForward(NeuronOCL.AsObject())) ReturnFalse; if(!Concat(cRoPE.getOutput(), confidence, Output, (int)iDimension, 1, (int)iHorizon)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCogDriverForecastToken layer. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Type() != defNeuronCogDriverForecastHead) ReturnFalse; if(!DeConcat(cRoPE.getGradient(), GetPointer(cConfidenceGradient), Gradient, (int)iDimension, 1, (int)iHorizon)) ReturnFalse; if(!cRoPE.CalcHiddenGradients(NeuronOCL.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCogDriverForecastToken state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iHorizon); if(!cRoPE.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCogDriverForecastToken state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, iHorizon); if(!LoadInsideLayer(file_handle, cRoPE.AsObject())) ReturnFalse; cRoPE.SetOpenCL(OpenCL); if(!cConfidenceGradient.BufferInit(iHorizon, 0) || !cConfidenceGradient.BufferCreate(OpenCL)) ReturnFalse; //--- return(Clear()); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCogDriverForecastToken GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronCogDriverForecastToken::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cRoPE.SetOpenCL(OpenCL); cConfidenceGradient.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronCogDriverForecastToken | //| layer. | //+------------------------------------------------------------------+ void CNeuronCogDriverForecastToken::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cRoPE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Clears the CNeuronCogDriverForecastToken execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCogDriverForecastToken::Clear(void) { if(!CNeuronBaseOCL::Clear() || !cRoPE.Clear() || !cConfidenceGradient.Fill(0)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUncAD_MUE layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint variables, uint stack_size, uint quantiles, float range, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, window + window_out, window + window_out, window_out, 2 * quantiles + 1, variables, optimization_type, batch)) ReturnFalse; //--- cMSDFlow.Clear(); cUncertainty.Clear(); cMSDFlow.SetOpenCL(OpenCL); cUncertainty.SetOpenCL(OpenCL); //--- MSD Flow uint index = 0; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, window * variables, iBatch, optimization) || !cMSDFlow.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; CNeuronAddToStack* stack = new CNeuronAddToStack(); if(!stack || !stack.Init(0, index, OpenCL, stack_size, window, variables, optimization, iBatch) || !cMSDFlow.Add(stack)) DeleteObjAndFalse(stack); index++; CNeuronUncAD_MSD* msd = new CNeuronUncAD_MSD(); if(!msd || !msd.Init(0, index, OpenCL, window, stack_size, variables, quantiles, range, optimization, iBatch) || !cMSDFlow.Add(msd)) DeleteObjAndFalse(msd); //--- Uncertainty Flow index++; CNeuronMultiMixAttention* att = new CNeuronMultiMixAttention(); if(!att || !att.Init(0, index, OpenCL, window * (2 * quantiles + 1), variables, 2 * quantiles + 1, quantiles, 5, 2, optimization, iBatch) || !cUncertainty.Add(att)) DeleteObjAndFalse(att); index++; CNeuronDictionaryCrossAtt* dic = new CNeuronDictionaryCrossAtt(); if(!dic || !dic.Init(0, index, OpenCL, window * (2 * quantiles + 1), variables, 4, stack_size / 2, optimization, iBatch) || !cUncertainty.Add(dic)) DeleteObjAndFalse(dic); index++; CNeuronBaseOCL* concat = new CNeuronBaseOCL(); if(!concat || !concat.Init(0, index, OpenCL, 2 * dic.Neurons(), optimization, iBatch) || !cUncertainty.Add(concat)) DeleteObjAndFalse(concat); concat.SetActivationFunction(None); index++; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 2 * window, 2 * window, window_out, 2 * quantiles + 1, variables, optimization, iBatch) || !cUncertainty.Add(conv)) DeleteObjAndFalse(conv); //--- index++; if(!cConcatenated.Init(0, index, OpenCL, msd.Neurons() + conv.Neurons(), optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUncAD_MUE feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- MSD Flow for(int i = 0; i < cMSDFlow.Total(); i++) { curr = cMSDFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Uncertainty Flow for(int i = 0; i < cUncertainty.Total(); i++) { curr = cUncertainty[i]; if(!curr) ReturnFalse; if(prev.Type() != defNeuronDictionaryCrossAtt) { if(!curr.FeedForward(prev)) ReturnFalse; } else { CNeuronDictionaryCrossAtt* dic = prev; uint count = dic.GetUnits() * dic.GetVariables(); uint dimension = dic.GetFilters(); prev = dic.GetSimilarity(); if(!prev || !Concat(dic.getOutput(), prev.getOutput(), curr.getOutput(), dimension, dimension, count)) ReturnFalse; } prev = curr; } //--- prev = cMSDFlow[-1]; uint count = cConv.GetUnits() * cConv.GetVariables(); uint dim_msd = prev.Neurons() / count; uint dim_unc = curr.Neurons() / count; if(!Concat(prev.getOutput(), curr.getOutput(), cConcatenated.getOutput(), dim_msd, dim_unc, count)) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cConcatenated.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronUncAD_MUE | //| layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || cMSDFlow.Total() < 1 || cUncertainty.Total() < 1) ReturnFalse; //--- CNeuronBaseOCL* msd = cMSDFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* next = cUncertainty[-1]; if(!msd || !next) ReturnFalse; //--- Final block gradient if(!CNeuronSpikeConvBlock::calcInputGradients(cConcatenated.AsObject())) ReturnFalse; //--- Split final concatenation and apply NLL gradient uint count = cConv.GetUnits() * cConv.GetVariables(); if(count == 0) ReturnFalse; uint dim_msd = msd.Neurons() / count; uint dim_unc = next.Neurons() / count; if(dim_msd == 0 || dim_unc == 0) ReturnFalse; if(!DeConcat(msd.getGradient(), next.getGradient(), cConcatenated.getGradient(), dim_msd, dim_unc, count)) ReturnFalse; if(!NLLGrad(next.getOutput(), msd.getGradient(), msd.getGradient(), next.getGradient())) ReturnFalse; //--- Uncertainty flow gradient for(int i = cUncertainty.Total() - 2; i >= 0; i--) { curr = cUncertainty[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronDictionaryCrossAtt) { CNeuronDictionaryCrossAtt* dictionary = curr; CNeuronBaseOCL* similarity = dictionary.GetSimilarity(); if(!similarity) ReturnFalse; count = dictionary.GetUnits() * dictionary.GetVariables(); uint dimension = dictionary.GetFilters(); if(count == 0 || dimension == 0) ReturnFalse; if(!DeConcat(dictionary.getGradient(), similarity.getGradient(), next.getGradient(), dimension, dimension, count)) ReturnFalse; } else if(!curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- MSD flow intentionally does not propagate gradients to source if(!NeuronOCL.getGradient().Fill(0)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUncAD_MUE trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || cMSDFlow.Total() < 1 || cUncertainty.Total() < 1) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* prev = cMSDFlow[-1]; if(!prev) ReturnFalse; //--- Update only trainable uncertainty path and final block for(int i = 0; i < cUncertainty.Total(); i++) { curr = cUncertainty[i]; if(!curr || !prev) ReturnFalse; if(prev.Type() != defNeuronDictionaryCrossAtt) { if(!curr.UpdateInputWeights(prev)) ReturnFalse; } prev = curr; } if(!CNeuronSpikeConvBlock::updateInputWeights(cConcatenated.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUncAD_MUE state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cMSDFlow.Save(file_handle)) ReturnFalse; if(!cUncertainty.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUncAD_MUE state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(!cMSDFlow.Load(file_handle)) ReturnFalse; if(!cUncertainty.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL* msd = cMSDFlow[-1]; CNeuronBaseOCL* uncertainty = cUncertainty[-1]; if(!msd || !uncertainty) ReturnFalse; uint index = cMSDFlow.Total() + cUncertainty.Total(); if(!cConcatenated.Init(0, index, OpenCL, msd.Neurons() + uncertainty.Neurons(), optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUncAD_MUE GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUncAD_MUE::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cMSDFlow.SetOpenCL(OpenCL); cUncertainty.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUncAD_MUE layer. | //+------------------------------------------------------------------+ void CNeuronUncAD_MUE::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cMSDFlow.TrainMode(bTrain); cUncertainty.TrainMode(bTrain); cConcatenated.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronUncAD_MUE weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUncAD_MUE* Source = source; dWeightsUpdate(cMSDFlow, Source, tau); dWeightsUpdate(cUncertainty, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronUncAD_MUE execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_MUE::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cMSDFlow.ClearStates()) ReturnFalse; if(!cUncertainty.ClearStates()) ReturnFalse; if(!cConcatenated.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUncAD_UGP layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint context_window, uint context_units, uint heads, uint key_dimension, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window == 0 || units == 0 || context_window == 0 || context_units == 0 || heads == 0 || key_dimension == 0 || candidates == 0 || topK == 0) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::Init(numOutputs, myIndex, open_cl, window, key_dimension, heads, units, context_window, context_units, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cSelfAttention.Init(0, index, OpenCL, window, units, heads, key_dimension, candidates, topK, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUncAD_UGP feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::feedForward(cSelfAttention.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronUncAD_UGP | //| layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronSpikeMHCrossAttention::calcInputGradients(cSelfAttention.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSelfAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUncAD_UGP trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; //--- if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::updateInputWeights(cSelfAttention.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUncAD_UGP state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::Save(const int file_handle) { if(!CNeuronSpikeMHCrossAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUncAD_UGP state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::Load(const int file_handle) { if(!CNeuronSpikeMHCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUncAD_UGP GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUncAD_UGP::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeMHCrossAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUncAD_UGP layer. | //+------------------------------------------------------------------+ void CNeuronUncAD_UGP::TrainMode(bool flag) { CNeuronSpikeMHCrossAttention::TrainMode(flag); cSelfAttention.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronUncAD_UGP weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeMHCrossAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUncAD_UGP* Source = source; dWeightsUpdate(cSelfAttention, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronUncAD_UGP execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGP::Clear(void) { if(!CNeuronSpikeMHCrossAttention::Clear()) ReturnFalse; if(!cSelfAttention.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUncAD_UGPL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint context_window, uint context_units, uint mode_units, uint heads, uint key_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window == 0 || units == 0 || context_window == 0 || context_units == 0 || mode_units == 0 || heads == 0 || key_dimension == 0) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::Init(numOutputs, myIndex, open_cl, window, key_dimension, heads, units, context_window, context_units, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cMode.Init(0, index, OpenCL, window * mode_units, optimization, iBatch)) ReturnFalse; cMode.SetActivationFunction(None); index++; if(!cModeFusion.Init(0, index, OpenCL, window, key_dimension, heads, units, window, mode_units, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUncAD_UGPL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- if(bTrain) if(!cMode.FeedForward()) ReturnFalse; if(!cModeFusion.FeedForward(NeuronOCL, cMode.getOutput())) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::feedForward(cModeFusion.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronUncAD_UGPL layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronSpikeMHCrossAttention::calcInputGradients(cModeFusion.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cModeFusion.AsObject(), cMode.getOutput(), cMode.getGradient(), None)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUncAD_UGPL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; //--- if(!cModeFusion.UpdateInputWeights(NeuronOCL, cMode.getOutput())) ReturnFalse; if(!cMode.UpdateInputWeights()) ReturnFalse; if(!CNeuronSpikeMHCrossAttention::updateInputWeights(cModeFusion.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUncAD_UGPL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::Save(const int file_handle) { if(!CNeuronSpikeMHCrossAttention::Save(file_handle)) ReturnFalse; if(!cMode.Save(file_handle)) ReturnFalse; if(!cModeFusion.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUncAD_UGPL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::Load(const int file_handle) { if(!CNeuronSpikeMHCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMode.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cModeFusion.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUncAD_UGPL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUncAD_UGPL::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeMHCrossAttention::SetOpenCL(obj); cMode.SetOpenCL(OpenCL); cModeFusion.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUncAD_UGPL layer. | //+------------------------------------------------------------------+ void CNeuronUncAD_UGPL::TrainMode(bool flag) { CNeuronSpikeMHCrossAttention::TrainMode(flag); cMode.TrainMode(bTrain); cModeFusion.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronUncAD_UGPL weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeMHCrossAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUncAD_UGPL* Source = source; dWeightsUpdate(cMode, Source, tau); dWeightsUpdate(cModeFusion, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronUncAD_UGPL execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD_UGPL::Clear(void) { if(!CNeuronSpikeMHCrossAttention::Clear()) ReturnFalse; if(!cMode.Clear()) ReturnFalse; if(!cModeFusion.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUncAD layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint quantiles, uint embed_size, uint candidates, uint topK, uint latent_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimensions.Size() < 2 || units_s == 0 || heads == 0 || stack_size == 0 || quantiles == 0 || embed_size == 0 || candidates < 2 || topK == 0 || topK > candidates || latent_count == 0 || embed_size % heads != 0) ReturnFalse; //--- iInputDim = dimensions[0] * units_s; for(uint i = 1; i < dimensions.Size(); i++) { if(dimensions[i] == 0) ReturnFalse; iInputDim += dimensions[i]; } if(dimensions[0] == 0) ReturnFalse; iAgentUnits = units_s; iContextUnits = dimensions.Size() - 1; iEmbedSize = embed_size; iQuantiles = quantiles; iCandidates = candidates; iTopK = topK; //--- uint agent_dim = iAgentUnits * iEmbedSize; uint context_dim = iContextUnits * iEmbedSize; uint qm_size = iAgentUnits * iEmbedSize * (2 * iQuantiles + 1); uint fusion_size = agent_dim + context_dim + qm_size; if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, fusion_size, fusion_size, latent_count, 1, 1, optimization_type, batch)) ReturnFalse; //--- uint token_units = iAgentUnits + iContextUnits; uint all_dimensions[]; if(ArrayResize(all_dimensions, token_units) < int(token_units)) ReturnFalse; for(uint i = 0; i < iAgentUnits; i++) all_dimensions[i] = dimensions[0]; for(uint i = 0; i < iContextUnits; i++) all_dimensions[iAgentUnits + i] = dimensions[i + 1]; //--- uint index = 0; if(!cTokenizer.Init(0, index, OpenCL, all_dimensions, token_units, iEmbedSize, token_units, iCandidates, iTopK, optimization, iBatch)) ReturnFalse; index++; if(!cAgents.Init(0, index, OpenCL, agent_dim, optimization, iBatch)) ReturnFalse; cAgents.SetActivationFunction(None); index++; if(!cContext.Init(0, index, OpenCL, context_dim, optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); index++; if(!cMUE.Init(0, index, OpenCL, iEmbedSize, iEmbedSize, iAgentUnits, stack_size, iQuantiles, 1.0f, optimization, iBatch)) ReturnFalse; index++; if(!cUGP.Init(0, index, OpenCL, iEmbedSize, iAgentUnits, iEmbedSize, iAgentUnits * (2 * iQuantiles + 1), heads, iEmbedSize / heads, candidates, topK, optimization, iBatch)) ReturnFalse; index++; if(!cUGPL.Init(0, index, OpenCL, iEmbedSize, iContextUnits, iEmbedSize, iAgentUnits, candidates, heads, iEmbedSize / heads, optimization, iBatch)) ReturnFalse; index++; if(!cMapInteraction.Init(0, index, OpenCL, iEmbedSize, iEmbedSize / heads, heads, iContextUnits, iEmbedSize, iAgentUnits * (2 * iQuantiles + 1), optimization, iBatch)) ReturnFalse; //--- tau a index++; if(!cTauAValues.Init(0, index, OpenCL, iEmbedSize, iEmbedSize, iCandidates * iEmbedSize, iAgentUnits, 1, optimization, iBatch)) ReturnFalse; cTauAValues.SetActivationFunction(SoftPlus); index++; if(!cTauAScores.Init(0, index, OpenCL, iEmbedSize, iEmbedSize, iCandidates, iAgentUnits, 1, optimization, iBatch)) ReturnFalse; cTauAScores.SetActivationFunction(None); index++; if(!cTauASoftMax.Init(0, index, OpenCL, iAgentUnits, iCandidates, iTopK, optimization, iBatch)) ReturnFalse; index++; if(!cTauAWeighted.Init(0, index, OpenCL, agent_dim, optimization, iBatch)) ReturnFalse; cTauAWeighted.SetActivationFunction(None); //--- tau e index++; if(!cTauEValues.Init(0, index, OpenCL, iEmbedSize, iEmbedSize, iCandidates * iEmbedSize, iContextUnits, 1, optimization, iBatch)) ReturnFalse; cTauEValues.SetActivationFunction(SoftPlus); index++; if(!cTauEScores.Init(0, index, OpenCL, iEmbedSize, iEmbedSize, iCandidates, iContextUnits, 1, optimization, iBatch)) ReturnFalse; cTauEScores.SetActivationFunction(None); index++; if(!cTauESoftMax.Init(0, index, OpenCL, iContextUnits, iCandidates, iTopK, optimization, iBatch)) ReturnFalse; index++; if(!cTauEWeighted.Init(0, index, OpenCL, iContextUnits * iEmbedSize, optimization, iBatch)) ReturnFalse; cTauEWeighted.SetActivationFunction(None); //--- fusion index++; if(!cFusionInput.Init(0, index, OpenCL, cTauAWeighted.Neurons() + cTauEWeighted.Neurons() + qm_size, optimization, iBatch)) ReturnFalse; cFusionInput.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Executes the tau a forward step of the CNeuronUncAD layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD::TauAForward(void) { if(!cTauAValues.FeedForward(cUGP.AsObject()) || !cTauAScores.FeedForward(cUGP.AsObject()) || !cTauASoftMax.FeedForward(cTauAScores.AsObject())) ReturnFalse; if(!SparseMatMul(cTauASoftMax.GetIndexes(), cTauASoftMax.getOutput(), cTauAValues.getOutput(), cTauAWeighted.getOutput(), iAgentUnits, iTopK, iCandidates, iEmbedSize)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the tau e forward step of the CNeuronUncAD layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD::TauEForward(void) { if(!cTauEValues.FeedForward(cMapInteraction.AsObject()) || !cTauEScores.FeedForward(cMapInteraction.AsObject()) || !cTauESoftMax.FeedForward(cTauEScores.AsObject())) ReturnFalse; if(!SparseMatMul(cTauESoftMax.GetIndexes(), cTauESoftMax.getOutput(), cTauEValues.getOutput(), cTauEWeighted.getOutput(), iContextUnits, iTopK, iCandidates, iEmbedSize)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUncAD feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronUncAD::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < int(iInputDim)) ReturnFalse; //--- Unified tokenization: market agents first, context tokens second if(!cTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cAgents.getOutput(), cContext.getOutput(), cTokenizer.getOutput(), cAgents.Neurons(), cContext.Neurons(), 1)) ReturnFalse; //--- Uncertainty-guided prediction/planning graph if(!cMUE.FeedForward(cAgents.AsObject())) ReturnFalse; if(!cUGP.FeedForward(cAgents.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cUGPL.FeedForward(cContext.AsObject(), cUGP.getOutput())) ReturnFalse; if(!cMapInteraction.FeedForward(cUGPL.AsObject(), cMUE.getOutput())) ReturnFalse; //--- Differentiable tau proposals if(!TauAForward() || !TauEForward()) ReturnFalse; //--- Proposal fusion for critic/actor head if(!Concat(cTauAWeighted.getOutput(), cTauEWeighted.getOutput(), cMUE.getOutput(), cFusionInput.getOutput(), cTauAWeighted.Neurons(), cTauEWeighted.Neurons(), cMUE.Neurons(), 1)) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cFusionInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the tau a gradients step of the CNeuronUncAD layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD::TauAGradients(void) { if(!SparseMatMulGrad(cTauASoftMax.GetIndexes(), cTauASoftMax.getOutput(), cTauASoftMax.getGradient(), cTauAValues.getOutput(), cTauAValues.getGradient(), cTauAWeighted.getGradient(), iAgentUnits, iTopK, iCandidates, iEmbedSize)) ReturnFalse; if(!cTauAScores.CalcHiddenGradients(cTauASoftMax.AsObject())) ReturnFalse; //--- Fan-in to Q_a'' if(!cUGP.CalcHiddenGradients(cTauAValues.AsObject())) ReturnFalse; CBufferFloat* qaa_gr = cUGP.getGradient(); if(!cUGP.SetGradient(cUGP.getPrevOutput(), false)) ReturnFalse; if(!cUGP.CalcHiddenGradients(cTauAScores.AsObject())) ReturnFalse; if(!SumAndNormalize(qaa_gr, cUGP.getGradient(), qaa_gr, 1, false, 0, 0, 0, 1) || !cUGP.SetGradient(qaa_gr, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the tau e gradients step of the CNeuronUncAD layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD::TauEGradients(void) { if(!SparseMatMulGrad(cTauESoftMax.GetIndexes(), cTauESoftMax.getOutput(), cTauESoftMax.getGradient(), cTauEValues.getOutput(), cTauEValues.getGradient(), cTauEWeighted.getGradient(), iContextUnits, iTopK, iCandidates, iEmbedSize)) ReturnFalse; if(!cTauEScores.CalcHiddenGradients(cTauESoftMax.AsObject())) ReturnFalse; //--- Fan-in to Q_e''' if(!cMapInteraction.CalcHiddenGradients(cTauEValues.AsObject())) ReturnFalse; CBufferFloat* qeee_gr = cMapInteraction.getGradient(); if(!cMapInteraction.SetGradient(cMapInteraction.getPrevOutput(), false)) ReturnFalse; if(!cMapInteraction.CalcHiddenGradients(cTauEScores.AsObject())) ReturnFalse; if(!SumAndNormalize(qeee_gr, cMapInteraction.getGradient(), qeee_gr, 1, false, 0, 0, 0, 1) || !cMapInteraction.SetGradient(qeee_gr, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronUncAD | //| layer. | //+------------------------------------------------------------------+ bool CNeuronUncAD::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Output fusion if(!CNeuronSpikeConvBlock::calcInputGradients(cFusionInput.AsObject())) ReturnFalse; if(!DeConcat(cTauAWeighted.getGradient(), cTauEWeighted.getGradient(), cMUE.getGradient(), cFusionInput.getGradient(), cTauAWeighted.Neurons(), cTauEWeighted.Neurons(), cMUE.Neurons(), 1)) ReturnFalse; //--- tau heads if(!TauAGradients() || !TauEGradients()) ReturnFalse; //--- Q_e''' = CrossAttention(Q_e'', Q_m') CBufferFloat* qm_gr = cMUE.getGradient(); if(!cMUE.SetGradient(cMUE.getPrevOutput(), false)) ReturnFalse; if(!cUGPL.CalcHiddenGradients(cMapInteraction.AsObject(), cMUE.getOutput(), cMUE.getGradient(), None)) ReturnFalse; if(!SumAndNormalize(qm_gr, cMUE.getGradient(), qm_gr, 1, false, 0, 0, 0, 1) || !cMUE.SetGradient(qm_gr, false)) ReturnFalse; //--- Q_e'' = UGPL(Q_e, Q_a'') CBufferFloat* qaa_gr = cUGP.getGradient(); if(!cUGP.SetGradient(cUGP.getPrevOutput(), false)) ReturnFalse; if(!cContext.CalcHiddenGradients(cUGPL.AsObject(), cUGP.getOutput(), cUGP.getGradient(), None)) ReturnFalse; if(!SumAndNormalize(qaa_gr, cUGP.getGradient(), qaa_gr, 1, false, 0, 0, 0, 1) || !cUGP.SetGradient(qaa_gr, false)) ReturnFalse; //--- Q_a'' = UGP(Q_a, Q_m') qm_gr = cMUE.getGradient(); if(!cMUE.SetGradient(cMUE.getPrevOutput(), false)) ReturnFalse; if(!cAgents.CalcHiddenGradients(cUGP.AsObject(), cMUE.getOutput(), cMUE.getGradient(), None)) ReturnFalse; if(!SumAndNormalize(qm_gr, cMUE.getGradient(), qm_gr, 1, false, 0, 0, 0, 1) || !cMUE.SetGradient(qm_gr, false)) ReturnFalse; //--- Q_m' = MUE(Q_a). MUE intentionally does not propagate source gradient. CBufferFloat* qa_gr = cAgents.getGradient(); if(!cAgents.SetGradient(cAgents.getPrevOutput(), false)) ReturnFalse; if(!cAgents.CalcHiddenGradients(cMUE.AsObject())) ReturnFalse; if(!cAgents.SetGradient(qa_gr, false)) ReturnFalse; //--- Unified tokenizer if(!Concat(cAgents.getGradient(), cContext.getGradient(), cTokenizer.getGradient(), cAgents.Neurons(), cContext.Neurons(), 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTokenizer.AsObject())) ReturnFalse; Deactivation(NeuronOCL) //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUncAD trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronUncAD::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMUE.UpdateInputWeights(cAgents.AsObject())) ReturnFalse; if(!cUGP.UpdateInputWeights(cAgents.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cUGPL.UpdateInputWeights(cContext.AsObject(), cUGP.getOutput())) ReturnFalse; if(!cMapInteraction.UpdateInputWeights(cUGPL.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cTauAValues.UpdateInputWeights(cUGP.AsObject())) ReturnFalse; if(!cTauAScores.UpdateInputWeights(cUGP.AsObject())) ReturnFalse; if(!cTauEValues.UpdateInputWeights(cMapInteraction.AsObject())) ReturnFalse; if(!cTauEScores.UpdateInputWeights(cMapInteraction.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cFusionInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUncAD state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iInputDim, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iAgentUnits, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iContextUnits, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iEmbedSize, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iQuantiles, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iCandidates, INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, (int)iTopK, INT_VALUE) < INT_VALUE) ReturnFalse; //--- if(!cTokenizer.Save(file_handle) || !cAgents.Save(file_handle) || !cContext.Save(file_handle) || !cMUE.Save(file_handle) || !cUGP.Save(file_handle) || !cUGPL.Save(file_handle) || !cMapInteraction.Save(file_handle) || !cTauAValues.Save(file_handle) || !cTauAScores.Save(file_handle) || !cTauASoftMax.Save(file_handle) || !cTauAWeighted.Save(file_handle) || !cTauEValues.Save(file_handle) || !cTauEScores.Save(file_handle) || !cTauESoftMax.Save(file_handle) || !cTauEWeighted.Save(file_handle) || !cFusionInput.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUncAD state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUncAD::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iInputDim = (uint)FileReadInteger(file_handle, INT_VALUE); iAgentUnits = (uint)FileReadInteger(file_handle, INT_VALUE); iContextUnits = (uint)FileReadInteger(file_handle, INT_VALUE); iEmbedSize = (uint)FileReadInteger(file_handle, INT_VALUE); iQuantiles = (uint)FileReadInteger(file_handle, INT_VALUE); iCandidates = (uint)FileReadInteger(file_handle, INT_VALUE); iTopK = (uint)FileReadInteger(file_handle, INT_VALUE); //--- if(!LoadInsideLayer(file_handle, cTokenizer.AsObject()) || !LoadInsideLayer(file_handle, cAgents.AsObject()) || !LoadInsideLayer(file_handle, cContext.AsObject()) || !LoadInsideLayer(file_handle, cMUE.AsObject()) || !LoadInsideLayer(file_handle, cUGP.AsObject()) || !LoadInsideLayer(file_handle, cUGPL.AsObject()) || !LoadInsideLayer(file_handle, cMapInteraction.AsObject()) || !LoadInsideLayer(file_handle, cTauAValues.AsObject()) || !LoadInsideLayer(file_handle, cTauAScores.AsObject()) || !LoadInsideLayer(file_handle, cTauASoftMax.AsObject()) || !LoadInsideLayer(file_handle, cTauAWeighted.AsObject()) || !LoadInsideLayer(file_handle, cTauEValues.AsObject()) || !LoadInsideLayer(file_handle, cTauEScores.AsObject()) || !LoadInsideLayer(file_handle, cTauESoftMax.AsObject()) || !LoadInsideLayer(file_handle, cTauEWeighted.AsObject()) || !LoadInsideLayer(file_handle, cFusionInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUncAD GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronUncAD::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cTokenizer.SetOpenCL(OpenCL); cAgents.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cMUE.SetOpenCL(OpenCL); cUGP.SetOpenCL(OpenCL); cUGPL.SetOpenCL(OpenCL); cMapInteraction.SetOpenCL(OpenCL); cTauAValues.SetOpenCL(OpenCL); cTauAScores.SetOpenCL(OpenCL); cTauASoftMax.SetOpenCL(OpenCL); cTauAWeighted.SetOpenCL(OpenCL); cTauEValues.SetOpenCL(OpenCL); cTauEScores.SetOpenCL(OpenCL); cTauESoftMax.SetOpenCL(OpenCL); cTauEWeighted.SetOpenCL(OpenCL); cFusionInput.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUncAD layer. | //+------------------------------------------------------------------+ void CNeuronUncAD::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cTokenizer.TrainMode(bTrain); cAgents.TrainMode(bTrain); cContext.TrainMode(bTrain); cMUE.TrainMode(bTrain); cUGP.TrainMode(bTrain); cUGPL.TrainMode(bTrain); cMapInteraction.TrainMode(bTrain); cTauAValues.TrainMode(bTrain); cTauAScores.TrainMode(bTrain); cTauASoftMax.TrainMode(bTrain); cTauAWeighted.TrainMode(bTrain); cTauEValues.TrainMode(bTrain); cTauEScores.TrainMode(bTrain); cTauESoftMax.TrainMode(bTrain); cTauEWeighted.TrainMode(bTrain); cFusionInput.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronUncAD weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronUncAD::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUncAD* Source = source; dWeightsUpdate(cTokenizer, Source, tau); dWeightsUpdate(cMUE, Source, tau); dWeightsUpdate(cUGP, Source, tau); dWeightsUpdate(cUGPL, Source, tau); dWeightsUpdate(cMapInteraction, Source, tau); dWeightsUpdate(cTauAValues, Source, tau); dWeightsUpdate(cTauAScores, Source, tau); dWeightsUpdate(cTauEValues, Source, tau); dWeightsUpdate(cTauEScores, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronUncAD execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUncAD::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cTokenizer.Clear() || !cAgents.Clear() || !cContext.Clear() || !cMUE.Clear() || !cUGP.Clear() || !cUGPL.Clear() || !cMapInteraction.Clear() || !cTauAValues.Clear() || !cTauAScores.Clear() || !cTauASoftMax.Clear() || !cTauAWeighted.Clear() || !cTauEValues.Clear() || !cTauEScores.Clear() || !cTauESoftMax.Clear() || !cTauEWeighted.Clear() || !cFusionInput.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the detach tau e from mpi step of the CNeuronMomAD | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMomAD::DetachTauEFromMPI(void) { if(!bTauELinked) return(true); //--- int output_total = cTauEWeighted.Neurons(); int gradient_total = cTauEWeighted.getGradient().Total(); if(output_total <= 0 || gradient_total <= 0) ReturnFalse; //--- CBufferFloat *output = new CBufferFloat(); CBufferFloat *gradient = new CBufferFloat(); if(!output || !gradient) { DeleteObj(output); DeleteObj(gradient); ReturnFalse; } if(!output.BufferInit(output_total, 0) || !gradient.BufferInit(gradient_total, 0)) { DeleteObj(output); DeleteObj(gradient); ReturnFalse; } if(!cTauEWeighted.SetOutput(output, false)) { DeleteObj(output); DeleteObj(gradient); ReturnFalse; } if(!cTauEWeighted.SetGradient(gradient, false)) { DeleteObj(gradient); ReturnFalse; } bTauELinked = false; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the link tau e to mpi step of the CNeuronMomAD layer. | //+------------------------------------------------------------------+ bool CNeuronMomAD::LinkTauEToMPI(void) { if(bTauELinked) return(true); if(cMPI.Neurons() != cTauEWeighted.Neurons() || cMPI.getGradient().Total() != cTauEWeighted.getGradient().Total()) ReturnFalse; //--- if(!cTauEWeighted.SetOutput(cMPI.getOutput(), true) || !cTauEWeighted.SetGradient(cMPI.getGradient(), true)) ReturnFalse; bTauELinked = true; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMomAD layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMomAD::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint heads, uint stack_size, uint quantiles, uint embed_size, uint candidates, uint topK, uint latent_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!DetachTauEFromMPI()) ReturnFalse; if(!CNeuronUncAD::Init(numOutputs, myIndex, open_cl, dimensions, units_s, heads, stack_size, quantiles, embed_size, candidates, topK, latent_count, optimization_type, batch)) ReturnFalse; //--- uint mpi_latent = (uint)cTauEWeighted.Neurons(); uint mpi_market_dim = iEmbedSize; uint mpi_market_units = iAgentUnits * (2 * iQuantiles + 1); if(mpi_latent == 0 || mpi_market_dim == 0 || mpi_market_units == 0 || heads == 0 || (mpi_latent % heads) != 0) ReturnFalse; if(!cMPI.Init(0, 0, OpenCL, mpi_latent, iCandidates, mpi_market_dim, mpi_market_units, 5, topK, heads, mpi_latent / heads, optimization, iBatch)) ReturnFalse; //--- return(LinkTauEToMPI()); } //+------------------------------------------------------------------+ //| Executes the tau e forward step of the CNeuronMomAD layer. | //+------------------------------------------------------------------+ bool CNeuronMomAD::TauEForward(void) { if(!bTauELinked && !LinkTauEToMPI()) ReturnFalse; if(!cTauEValues.FeedForward(cMapInteraction.AsObject())) ReturnFalse; if(!cMPI.FeedForward(cTauEValues.AsObject(), cMUE.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Executes the tau e gradients step of the CNeuronMomAD layer. | //+------------------------------------------------------------------+ bool CNeuronMomAD::TauEGradients(void) { if(!bTauELinked && !LinkTauEToMPI()) ReturnFalse; //--- Preserve the fusion gradient into MUE and add MPI's second-input gradient. CBufferFloat *qm_gr = cMUE.getGradient(); if(!cMUE.SetGradient(cMUE.getPrevOutput(), false)) ReturnFalse; if(!cTauEValues.CalcHiddenGradients(cMPI.AsObject(), cMUE.getOutput(), cMUE.getGradient(), (ENUM_ACTIVATION)cMUE.Activation())) ReturnFalse; if(!SumAndNormalize(qm_gr, cMUE.getGradient(), qm_gr, 1, false, 0, 0, 0, 1) || !cMUE.SetGradient(qm_gr, false)) ReturnFalse; if(!cMapInteraction.CalcHiddenGradients(cTauEValues.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMomAD trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMomAD::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMUE.UpdateInputWeights(cAgents.AsObject())) ReturnFalse; if(!cUGP.UpdateInputWeights(cAgents.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cUGPL.UpdateInputWeights(cContext.AsObject(), cUGP.getOutput())) ReturnFalse; if(!cMapInteraction.UpdateInputWeights(cUGPL.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cMPI.UpdateInputWeights(cMapInteraction.AsObject(), cMUE.getOutput())) ReturnFalse; if(!cTauAValues.UpdateInputWeights(cUGP.AsObject())) ReturnFalse; if(!cTauAScores.UpdateInputWeights(cUGP.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cFusionInput.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMomAD state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomAD::Save(const int file_handle) { if(!CNeuronUncAD::Save(file_handle)) ReturnFalse; if(!cMPI.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMomAD state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMomAD::Load(const int file_handle) { if(!DetachTauEFromMPI()) ReturnFalse; if(!CNeuronUncAD::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMPI.AsObject())) ReturnFalse; //--- return(LinkTauEToMPI()); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMomAD GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMomAD::SetOpenCL(COpenCLMy *obj) { DetachTauEFromMPI(); CNeuronUncAD::SetOpenCL(obj); cMPI.SetOpenCL(OpenCL); if(cMPI.Neurons() == cTauEWeighted.Neurons() && cMPI.Neurons() > 0) LinkTauEToMPI(); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMomAD layer. | //+------------------------------------------------------------------+ void CNeuronMomAD::TrainMode(bool flag) { CNeuronUncAD::TrainMode(flag); cMPI.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronMomAD weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronMomAD::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; if(!CNeuronUncAD::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMomAD *Source = source; dWeightsUpdate(cMPI, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronMomAD execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMomAD::Clear(void) { if(!CNeuronUncAD::Clear()) ReturnFalse; if(!cMPI.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CScenarioCodebook layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CScenarioCodebook::Init(uint scenarios, uint dimension, COpenCLMy *open_cl) { if(scenarios == 0 || dimension == 0) ReturnFalse; iScenarios = scenarios; iDimension = dimension; OpenCL = open_cl; const int total = int(iScenarios * iDimension); if(!cPrototypes.BufferInit(total, 0) || !cEMASums.BufferInit(total, 0) || !cEMACounts.BufferInit(iScenarios, 0) || !cUsage.BufferInit(iScenarios, 0) || !cInactive.BufferInit(iScenarios, 1) || !cInactivityAge.BufferInit(iScenarios, 0)) ReturnFalse; //--- independent deterministic prototypes without changing the terminal RNG for(int i = 0; i < total; i++) { xor128; const float value = float((double(rnd_w) / UINT_MAX) * 2.0 - 1.0); if(!cPrototypes.Update(i, value) || !cEMASums.Update(i, value)) ReturnFalse; } for(uint k = 0; k < iScenarios; k++) if(!cEMACounts.Update(k, 1.0f)) ReturnFalse; SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CScenarioCodebook state to the file handle. | //+------------------------------------------------------------------+ bool CScenarioCodebook::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; dFileWriteUInt(file_handle, iScenarios); dFileWriteUInt(file_handle, iDimension); return(cPrototypes.Save(file_handle) && cEMASums.Save(file_handle) && cEMACounts.Save(file_handle) && cUsage.Save(file_handle) && cInactive.Save(file_handle) && cInactivityAge.Save(file_handle)); } //+------------------------------------------------------------------+ //| Append codebook parameters and their EMA optimizer state. | //+------------------------------------------------------------------+ bool CScenarioCodebook::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronBaseOCL::AppendParameterBufferFingerprint(GetPointer(cPrototypes), fingerprint, 41) || !CNeuronBaseOCL::AppendParameterBufferFingerprint(GetPointer(cEMASums), fingerprint, 42) || !CNeuronBaseOCL::AppendParameterBufferFingerprint(GetPointer(cEMACounts), fingerprint, 43)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CScenarioCodebook state from the file handle. | //+------------------------------------------------------------------+ bool CScenarioCodebook::Load(const int file_handle, COpenCLMy *open_cl, const bool strict) { if(file_handle == INVALID_HANDLE) ReturnFalse; dFileReadUInt(file_handle, iScenarios); dFileReadUInt(file_handle, iDimension); if(iScenarios == 0 || iDimension == 0) ReturnFalse; OpenCL = NULL; if(!(strict ? cPrototypes.LoadInferenceFinite(file_handle, open_cl) : cPrototypes.Load(file_handle)) || !(strict ? cEMASums.LoadInferenceFinite(file_handle, open_cl) : cEMASums.Load(file_handle)) || !(strict ? cEMACounts.LoadInferenceFinite(file_handle, open_cl) : cEMACounts.Load(file_handle)) || !(strict ? cUsage.LoadInferenceFinite(file_handle, open_cl) : cUsage.Load(file_handle)) || !(strict ? cInactive.LoadInferenceFinite(file_handle, open_cl) : cInactive.Load(file_handle))) ReturnFalse; if(!(strict ? cInactivityAge.LoadInferenceFinite(file_handle, open_cl) : cInactivityAge.Load(file_handle))) ReturnFalse; if(!ValidateContract(iScenarios, iDimension)) ReturnFalse; SetOpenCL(open_cl); return(true); } //+------------------------------------------------------------------+ //| Validates the CScenarioCodebook layer contract before execution. | //+------------------------------------------------------------------+ bool CScenarioCodebook::ValidateContract(uint scenarios, uint dimension) { if(iScenarios != scenarios || iDimension != dimension || cPrototypes.Total() != int(iScenarios * iDimension) || cEMASums.Total() != int(iScenarios * iDimension) || cEMACounts.Total() != int(iScenarios) || cUsage.Total() != int(iScenarios) || cInactive.Total() != int(iScenarios) || cInactivityAge.Total() != int(iScenarios)) return(false); return(true); } //+------------------------------------------------------------------+ //| Rebinds the CScenarioCodebook GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CScenarioCodebook::SetOpenCL(COpenCLMy *open_cl) { OpenCL = open_cl; if(!OpenCL) return; cPrototypes.BufferCreate(OpenCL); cEMASums.BufferCreate(OpenCL); cEMACounts.BufferCreate(OpenCL); cUsage.BufferCreate(OpenCL); cInactive.BufferCreate(OpenCL); cInactivityAge.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Transfers codebook buffers before publishing its OpenCL context. | //+------------------------------------------------------------------+ bool CScenarioCodebook::SetOpenCLChecked(COpenCLMy *open_cl) { //--- Validate the target and immutable codebook shape before mutation. if(CheckPointer(open_cl) == POINTER_INVALID) ReturnFalseEx("checked codebook target"); if(iScenarios == 0) ReturnFalseEx("checked codebook scenarios"); if(iDimension == 0) ReturnFalseEx("checked codebook dimension"); if(!ValidateContract(iScenarios, iDimension)) ReturnFalseEx("checked codebook contract"); //--- Read every foreign allocated buffer before any target creation. if(cPrototypes.Total() > 0 && cPrototypes.GetIndex() >= 0 && cPrototypes.GetOpenCL() != open_cl && !cPrototypes.BufferRead()) ReturnFalseEx("codebook prototypes read"); if(cEMASums.Total() > 0 && cEMASums.GetIndex() >= 0 && cEMASums.GetOpenCL() != open_cl && !cEMASums.BufferRead()) ReturnFalseEx("codebook EMA sums read"); if(cEMACounts.Total() > 0 && cEMACounts.GetIndex() >= 0 && cEMACounts.GetOpenCL() != open_cl && !cEMACounts.BufferRead()) ReturnFalseEx("codebook EMA counts read"); if(cUsage.Total() > 0 && cUsage.GetIndex() >= 0 && cUsage.GetOpenCL() != open_cl && !cUsage.BufferRead()) ReturnFalseEx("codebook usage read"); if(cInactive.Total() > 0 && cInactive.GetIndex() >= 0 && cInactive.GetOpenCL() != open_cl && !cInactive.BufferRead()) ReturnFalseEx("codebook inactive read"); if(cInactivityAge.Total() > 0 && cInactivityAge.GetIndex() >= 0 && cInactivityAge.GetOpenCL() != open_cl && !cInactivityAge.BufferRead()) ReturnFalseEx("codebook inactivity age read"); //--- Create only buffers absent from the target context in declaration order. if(cPrototypes.Total() > 0 && (cPrototypes.GetOpenCL() != open_cl || cPrototypes.GetIndex() < 0) && !cPrototypes.BufferCreate(open_cl)) ReturnFalseEx("codebook prototypes create"); if(cEMASums.Total() > 0 && (cEMASums.GetOpenCL() != open_cl || cEMASums.GetIndex() < 0) && !cEMASums.BufferCreate(open_cl)) ReturnFalseEx("codebook EMA sums create"); if(cEMACounts.Total() > 0 && (cEMACounts.GetOpenCL() != open_cl || cEMACounts.GetIndex() < 0) && !cEMACounts.BufferCreate(open_cl)) ReturnFalseEx("codebook EMA counts create"); if(cUsage.Total() > 0 && (cUsage.GetOpenCL() != open_cl || cUsage.GetIndex() < 0) && !cUsage.BufferCreate(open_cl)) ReturnFalseEx("codebook usage create"); if(cInactive.Total() > 0 && (cInactive.GetOpenCL() != open_cl || cInactive.GetIndex() < 0) && !cInactive.BufferCreate(open_cl)) ReturnFalseEx("codebook inactive create"); if(cInactivityAge.Total() > 0 && (cInactivityAge.GetOpenCL() != open_cl || cInactivityAge.GetIndex() < 0) && !cInactivityAge.BufferCreate(open_cl)) ReturnFalseEx("codebook inactivity age create"); //--- Publish the outer owner only after all direct buffers transferred. OpenCL = open_cl; //--- Finalize the bounded checked context transfer. return(true); } //+------------------------------------------------------------------+ //| Validates the GPU launch configuration of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ValidateLaunchConfiguration(COpenCLMy *open_cl, uint scenarios, uint horizon, uint latent, uint variables) { if(!open_cl || variables == 0 || scenarios == 0 || horizon == 0 || latent == 0) return(false); const long max_axis0 = open_cl.GetMaxLocalSize(0); const long max_axis1 = open_cl.GetMaxLocalSize(1); const long max_work_group = open_cl.GetMaxWorkGroupSize(); //--- Reject each axis before forming products. This also makes every long //--- product below bounded well inside LONG_MAX for any uint horizon. if(max_axis0 <= 0 || max_axis1 <= 0 || max_work_group <= 0 || latent > 256 || variables > 256 || scenarios > 256 || latent > (uint)max_axis0 || variables > (uint)max_axis1 || scenarios > (uint)max_axis0) return(false); const long variable_horizon = (long)variables * horizon; const long scenario_horizon = (long)scenarios * horizon; const long target_elements = variable_horizon * latent; const long confidence_elements = (long)scenarios * variable_horizon; const long forecast_elements = confidence_elements * latent; const long generator_condition_elements = 2 * forecast_elements; const long router_condition_elements = 2 * (long)scenarios * variables * latent; const long max_buffer_elements = 2147483647; //--- All subsequent BufferInit/Neuron::Init sizes and OpenCL int index spaces. if(variable_horizon > max_buffer_elements || scenario_horizon > max_buffer_elements || target_elements > max_buffer_elements || confidence_elements > max_buffer_elements || forecast_elements > max_buffer_elements || generator_condition_elements > max_buffer_elements || router_condition_elements > max_buffer_elements) return(false); const long joint_group = (long)latent * variables; const long scenario_group = (long)scenarios; const long pairs = (long)scenarios * (scenarios - 1) / 2; const long diagnostic_group = (pairs > scenario_group ? pairs : scenario_group); const long responsibility_group = (joint_group > scenario_group ? joint_group : scenario_group); const long normalize_lanes = (scenario_horizon < 64 ? scenario_horizon : 64); const long normalize_group = (joint_group > normalize_lanes ? joint_group : normalize_lanes); const long forward_limit = open_cl.GetKernelInfoInteger(def_k_ScenarioForecastForward, CL_KERNEL_WORK_GROUP_SIZE); const long responsibility_limit = open_cl.GetKernelInfoInteger(def_k_ScenarioResponsibilities, CL_KERNEL_WORK_GROUP_SIZE); const long codebook_limit = open_cl.GetKernelInfoInteger(def_k_ScenarioCodebookEMA, CL_KERNEL_WORK_GROUP_SIZE); const long normalize_limit = open_cl.GetKernelInfoInteger(def_k_ScenarioGradientNormalize, CL_KERNEL_WORK_GROUP_SIZE); return(normalize_lanes <= max_axis0 && diagnostic_group <= max_axis0 && joint_group <= 256 && diagnostic_group <= 256 && joint_group <= max_work_group && diagnostic_group <= max_work_group && forward_limit >= joint_group && responsibility_limit >= responsibility_group && responsibility_limit >= diagnostic_group && codebook_limit >= joint_group && normalize_limit >= normalize_group); } //+------------------------------------------------------------------+ //| Initializes the CNeuronScenarioForecast layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint scenarios, uint top_k, uint horizon, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch, uint variables) { #define ORION_FORECAST_INIT_STAGE(EXPR,STAGE) if(!(EXPR)) { PrintFormat("%s line %d: %s", __FUNCTION__, __LINE__, STAGE); return(false); } if(top_k == 0 || top_k > scenarios || !ValidateLaunchConfiguration(open_cl, scenarios, horizon, dimension, variables)) { PrintFormat("%s: invalid launch contract S=%u TopK=%u H=%u D=%u V=%u", __FUNCTION__, scenarios, top_k, horizon, dimension, variables); return(false); } iVariables = variables; iScenarios = scenarios; iTopK = top_k; iHorizon = horizon; iDimension = dimension; bLastTargetAccepted = false; bLastControlValid = false; ORION_FORECAST_INIT_STAGE(CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, iScenarios * iVariables * iHorizon * iDimension, optimization_type, batch), "base output"); ORION_FORECAST_INIT_STAGE(cConfidence.BufferInit(iScenarios * iVariables * iHorizon, 1.0f), "confidence BufferInit"); ORION_FORECAST_INIT_STAGE(cConfidence.BufferCreate(OpenCL), "confidence BufferCreate"); ORION_FORECAST_INIT_STAGE(cLogConfidence.BufferInit(iScenarios * iVariables * iHorizon, MathLog(1.000001f)), "log-confidence BufferInit"); ORION_FORECAST_INIT_STAGE(cLogConfidence.BufferCreate(OpenCL), "log-confidence BufferCreate"); ORION_FORECAST_INIT_STAGE(cLogProbabilities.BufferInit(iScenarios, MathLog(1.0f / iScenarios)), "log-probabilities BufferInit"); ORION_FORECAST_INIT_STAGE(cLogProbabilities.BufferCreate(OpenCL), "log-probabilities BufferCreate"); ORION_FORECAST_INIT_STAGE(cResponsibilities.BufferInit(iScenarios, 0), "responsibilities BufferInit"); ORION_FORECAST_INIT_STAGE(cResponsibilities.BufferCreate(OpenCL), "responsibilities BufferCreate"); ORION_FORECAST_INIT_STAGE(cTarget.BufferInit(iVariables * iHorizon * iDimension, 0), "target BufferInit"); ORION_FORECAST_INIT_STAGE(cTarget.BufferCreate(OpenCL), "target BufferCreate"); ORION_FORECAST_INIT_STAGE(cBatchControl.BufferInit(1, 0) && cBatchControl.BufferCreate(OpenCL), "batch-control buffer"); ORION_FORECAST_INIT_STAGE(cRecovery.BufferInit(iScenarios, 0) && cRecovery.BufferCreate(OpenCL), "recovery buffer"); ORION_FORECAST_INIT_STAGE(cRecoveryEvents.BufferInit(iScenarios, 0) && cRecoveryEvents.BufferCreate(OpenCL), "recovery-events buffer"); ORION_FORECAST_INIT_STAGE(cAgeZeroEvents.BufferInit(iScenarios, 0) && cAgeZeroEvents.BufferCreate(OpenCL), "age-zero-events buffer"); ORION_FORECAST_INIT_STAGE(cForwardValidity.BufferInit(iScenarios, 0) && cForwardValidity.BufferCreate(OpenCL), "forward-validity buffer"); ORION_FORECAST_INIT_STAGE(cDiagnosticPartials.BufferInit(3 * iScenarios, 0) && cDiagnosticPartials.BufferCreate(OpenCL), "diagnostic-partials buffer"); ORION_FORECAST_INIT_STAGE(cEpochDiagnostics.BufferInit(12, 0) && cEpochDiagnostics.BufferCreate(OpenCL), "epoch-diagnostics buffer"); ORION_FORECAST_INIT_STAGE(cDiagnosticScratch.BufferInit((iScenarios * (iScenarios - 1) / 2 > iScenarios ? iScenarios * (iScenarios - 1) / 2 : iScenarios), 0) && cDiagnosticScratch.BufferCreate(OpenCL), "diagnostic-scratch buffer"); ORION_FORECAST_INIT_STAGE(cCodebook.Init(iScenarios, iVariables * iHorizon * iDimension, OpenCL), "codebook"); ORION_FORECAST_INIT_STAGE(cGeneratorInput.Init(0, myIndex + 1, OpenCL, iScenarios * iVariables * iHorizon * 2 * iDimension, optimization, iBatch), "generator input"); ORION_FORECAST_INIT_STAGE(cRouterInput.Init(0, myIndex + 2, OpenCL, iScenarios * 2 * iVariables * iDimension, optimization, iBatch), "router input"); ORION_FORECAST_INIT_STAGE(cConfidenceInput.Init(0, myIndex + 3, OpenCL, iScenarios * iVariables * iHorizon * 2 * iDimension, optimization, iBatch), "confidence input"); ORION_FORECAST_INIT_STAGE(cGenerator.Init(0, myIndex + 4, OpenCL, 2 * iDimension, 2 * iDimension, iDimension, iScenarios * iVariables * iHorizon, optimization, iBatch), "generator"); ORION_FORECAST_INIT_STAGE(cRouter.Init(0, myIndex + 5, OpenCL, 2 * iVariables * iDimension, 2 * iVariables * iDimension, 1, iScenarios, optimization, iBatch), "router"); ORION_FORECAST_INIT_STAGE(cConfidenceHead.Init(0, myIndex + 6, OpenCL, 2 * iDimension, 2 * iDimension, 1, iScenarios * iVariables * iHorizon, optimization, iBatch), "confidence head"); ORION_FORECAST_INIT_STAGE(cRouterSoftmax.Init(0, myIndex + 7, OpenCL, iScenarios, optimization, iBatch), "router softmax"); ORION_FORECAST_INIT_STAGE(cResponsibilityScores.Init(0, myIndex + 8, OpenCL, iScenarios, optimization, iBatch), "responsibility scores"); ORION_FORECAST_INIT_STAGE(cTopKSelector.Init(0, myIndex + 9, OpenCL, 1, iScenarios, iTopK, optimization, iBatch), "TopK selector"); SetActivationFunction(None); cGeneratorInput.SetActivationFunction(None); cRouterInput.SetActivationFunction(None); cConfidenceInput.SetActivationFunction(None); cGenerator.SetActivationFunction(None); cRouter.SetActivationFunction(None); cConfidenceHead.SetActivationFunction(SoftPlus); #undef ORION_FORECAST_INIT_STAGE return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronScenarioForecast feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() != int(iVariables * iDimension) || !NeuronOCL.getOutput() || NeuronOCL.getOutputIndex() < 0) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iScenarios}; uint local_work_size[3] = {iDimension, iVariables, 1}; const int kernel = def_k_ScenarioForecastForward; setBuffer(kernel, def_k_sff_input, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_sff_prototypes, cCodebook.GetPrototypes().GetIndex()) setBuffer(kernel, def_k_sff_generator_input, cGeneratorInput.getOutputIndex()) setBuffer(kernel, def_k_sff_router_input, cRouterInput.getOutputIndex()) setBuffer(kernel, def_k_sff_confidence_input, cConfidenceInput.getOutputIndex()) setBuffer(kernel, def_k_sff_generator_output, cGenerator.getOutputIndex()) setBuffer(kernel, def_k_sff_router_output, cRouter.getOutputIndex()) setBuffer(kernel, def_k_sff_confidence_output, cConfidenceHead.getOutputIndex()) setBuffer(kernel, def_k_sff_z, Output.GetIndex()) setBuffer(kernel, def_k_sff_u, cConfidence.GetIndex()) setBuffer(kernel, def_k_sff_pi, cRouterSoftmax.getOutputIndex()) setArgument(kernel, def_k_sff_scenarios, (int)iScenarios) setArgument(kernel, def_k_sff_horizon, (int)iHorizon) setArgument(kernel, def_k_sff_dimension, (int)iDimension) setArgument(kernel, def_k_sff_phase, (int)0) setBuffer(kernel, def_k_sff_log_pi, cLogProbabilities.GetIndex()) setBuffer(kernel, def_k_sff_log_u, cLogConfidence.GetIndex()) setArgument(kernel, def_k_sff_variables, (int)iVariables) setBuffer(kernel, def_k_sff_forward_validity, cForwardValidity.GetIndex()) CBufferFloat *generator_weights = cGenerator.GetWeightsConv(); CBufferFloat *router_weights = cRouter.GetWeightsConv(); CBufferFloat *confidence_weights = cConfidenceHead.GetWeightsConv(); if(!generator_weights || !router_weights || !confidence_weights || generator_weights.GetIndex() < 0 || router_weights.GetIndex() < 0 || confidence_weights.GetIndex() < 0 || generator_weights.Total() <= 0 || router_weights.Total() <= 0 || confidence_weights.Total() <= 0) ReturnFalse; setBuffer(kernel, def_k_sff_generator_weights, generator_weights.GetIndex()) setBuffer(kernel, def_k_sff_router_weights, router_weights.GetIndex()) setBuffer(kernel, def_k_sff_confidence_weights, confidence_weights.GetIndex()) setArgument(kernel, def_k_sff_generator_weights_count, (int)generator_weights.Total()) setArgument(kernel, def_k_sff_router_weights_count, (int)router_weights.Total()) setArgument(kernel, def_k_sff_confidence_weights_count, (int)confidence_weights.Total()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) if(!cGenerator.FeedForward(cGeneratorInput.AsObject()) || !cRouter.FeedForward(cRouterInput.AsObject()) || !cRouterSoftmax.FeedForward(cRouter.AsObject()) || !cConfidenceHead.FeedForward(cConfidenceInput.AsObject())) ReturnFalse; setArgument(kernel, def_k_sff_phase, (int)1) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) return(true); } //+------------------------------------------------------------------+ //| Executes the bind responsibilities kernel step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::BindResponsibilitiesKernel(CBufferFloat *target_source, CBufferFloat *observation_by_scenario) { if(!OpenCL || !target_source || !observation_by_scenario || target_source.GetIndex() < 0 || observation_by_scenario.GetIndex() < 0 || observation_by_scenario.Total() < int(iScenarios)) return(false); const int kernel = def_k_ScenarioResponsibilities; return(OpenCL.SetArgumentBuffer(kernel, def_k_sr_target, cTarget.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_z, Output.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_u, cConfidence.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_pi, cRouterSoftmax.getOutputIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_responsibilities, cResponsibilities.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_generator_gradient, cGenerator.getGradientIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_router_gradient, cRouter.getGradientIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_confidence_gradient, cConfidenceHead.getGradientIndex()) && OpenCL.SetArgument(kernel, def_k_sr_scenarios, (int)iScenarios) && OpenCL.SetArgument(kernel, def_k_sr_topk, (int)iTopK) && OpenCL.SetArgument(kernel, def_k_sr_horizon, (int)iHorizon) && OpenCL.SetArgument(kernel, def_k_sr_dimension, (int)iDimension) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_scores, cResponsibilityScores.getOutputIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_sparse, cTopKSelector.getOutputIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_indexes, cTopKSelector.GetIndexes().GetIndex()) && OpenCL.SetArgument(kernel, def_k_sr_variables, (int)iVariables) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_prototypes, cCodebook.GetPrototypes().GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_batch_control, cBatchControl.GetIndex()) && OpenCL.SetArgument(kernel, def_k_sr_huber_delta, (float)1.0f) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_target_source, target_source.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_forward_validity, cForwardValidity.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_diagnostic_partials, cDiagnosticPartials.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_epoch_diagnostics, cEpochDiagnostics.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_observation_by_scenario, observation_by_scenario.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_diagnostic_scratch, cDiagnosticScratch.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_usage, cCodebook.GetUsage().GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_inactive, cCodebook.GetInactive().GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_recovery_events, cRecoveryEvents.GetIndex()) && OpenCL.SetArgumentBuffer(kernel, def_k_sr_age_zero_events, cAgeZeroEvents.GetIndex())); } //+------------------------------------------------------------------+ //| Executes the build responsibilities step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::BuildResponsibilities(CBufferFloat *target) { return(BuildResponsibilities(target, target)); } //+------------------------------------------------------------------+ //| Full future latent trains Z/U/Pi; delta latent updates Codebook | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::BuildResponsibilities(CBufferFloat *target, CBufferFloat *delta_target) { bLastTargetAccepted = false; bLastControlValid = false; const int target_total = int(iVariables * iHorizon * iDimension); if(!OpenCL || !target || !delta_target || target.Total() < target_total || delta_target.Total() < target_total || target.GetIndex() < 0 || delta_target.GetIndex() < 0) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, 1}; uint local_work_size[3] = {iDimension, iVariables, 1}; const int kernel = def_k_ScenarioResponsibilities; if(!BindResponsibilitiesKernel(target, GetPointer(cDiagnosticPartials))) ReturnFalse; setArgument(kernel, def_k_sr_phase, (int)0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- one complete (D,V) group per scenario global_work_size[2] = iScenarios; setArgument(kernel, def_k_sr_phase, (int)1) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) if(!cTopKSelector.FeedForward(cResponsibilityScores.AsObject())) ReturnFalse; //--- fused device-only TopK responsibility, router and trajectory gradients setArgument(kernel, def_k_sr_phase, (int)2) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- normalize active latent/confidence branches without host tensor reads const int normalize = def_k_ScenarioGradientNormalize; setBuffer(normalize, def_k_sgn_generator_gradient, cGenerator.getGradientIndex()) setBuffer(normalize, def_k_sgn_generator_input_gradient, cGeneratorInput.getGradientIndex()) setBuffer(normalize, def_k_sgn_router_input_gradient, cRouterInput.getGradientIndex()) setBuffer(normalize, def_k_sgn_confidence_input_gradient, cConfidenceInput.getGradientIndex()) setBuffer(normalize, def_k_sgn_input_gradient, Gradient.GetIndex()) setBuffer(normalize, def_k_sgn_responsibilities, cResponsibilities.GetIndex()) setArgument(normalize, def_k_sgn_scenarios, (int)iScenarios) setArgument(normalize, def_k_sgn_horizon, (int)iHorizon) setArgument(normalize, def_k_sgn_dimension, (int)iDimension) setArgument(normalize, def_k_sgn_phase, (int)0) setBuffer(normalize, def_k_sgn_confidence_gradient, cConfidenceHead.getGradientIndex()) setArgument(normalize, def_k_sgn_variables, (int)iVariables) setBuffer(normalize, def_k_sgn_batch_control, cBatchControl.GetIndex()) setArgument(normalize, def_k_sgn_normalize_gradients, (bGradientNormalization ? 1 : 0)) kernelExecuteLoc(normalize, global_work_offset, global_work_size, local_work_size) if(bCodebookUpdate && !UpdateCodebook(delta_target)) ReturnFalse; //--- only compact control crosses the device boundary; tensors remain resident if(!cBatchControl.BufferRead()) ReturnFalse; bLastControlValid = true; bLastTargetAccepted = (cBatchControl[0] > 0.5f); return(bLastTargetAccepted); } //+------------------------------------------------------------------+ //| Executes the reset epoch diagnostics step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ResetEpochDiagnostics(void) { if(!ValidateLaunchConfiguration(OpenCL, iScenarios, iHorizon, iDimension, iVariables) || !BindResponsibilitiesKernel(GetPointer(cTarget), GetPointer(cDiagnosticPartials))) return(false); const int kernel = def_k_ScenarioResponsibilities; setArgument(kernel, def_k_sr_phase, (int)4) uint offset[1] = {0}; uint global[1] = {iScenarios}; kernelExecute(kernel, offset, global) return(true); } //+------------------------------------------------------------------+ //| Executes the commit batch diagnostics step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::CommitBatchDiagnostics(void) { if(!ValidateLaunchConfiguration(OpenCL, iScenarios, iHorizon, iDimension, iVariables) || !BindResponsibilitiesKernel(GetPointer(cTarget), GetPointer(cDiagnosticPartials))) ReturnFalse; const int kernel = def_k_ScenarioResponsibilities; setArgument(kernel, def_k_sr_phase, (int)3) uint offset[1] = {0}; uint global[1] = {iScenarios}; uint local[1] = {iScenarios}; kernelExecuteLoc(kernel, offset, global, local) return(true); } //+------------------------------------------------------------------+ //| Executes the build epoch codebook diagnostics step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::BuildEpochCodebookDiagnostics(void) { if(!ValidateLaunchConfiguration(OpenCL, iScenarios, iHorizon, iDimension, iVariables) || !BindResponsibilitiesKernel(GetPointer(cTarget), GetPointer(cDiagnosticPartials))) ReturnFalse; const int kernel = def_k_ScenarioResponsibilities; const uint pairs = iScenarios * (iScenarios - 1) / 2; if(pairs > 0) { setArgument(kernel, def_k_sr_phase, (int)5) uint offset[3] = {0, 0, 0}; uint global[3] = {iDimension, iVariables, pairs}; uint local[3] = {iDimension, iVariables, 1}; kernelExecuteLoc(kernel, offset, global, local) } setArgument(kernel, def_k_sr_phase, (int)6) uint reduce_offset[1] = {0}; uint reduce_size[1] = {(pairs > iScenarios ? pairs : iScenarios)}; kernelExecuteLoc(kernel, reduce_offset, reduce_size, reduce_size) return(true); } //+------------------------------------------------------------------+ //| Executes the read epoch diagnostics step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ReadEpochDiagnostics(double &mix_loss, double &router_loss, double &trajectory_loss, double &confidence_loss, double &latent_error, double &observation_error, double &valid_batches, double &invalid_batches, double &usage_entropy, double &pairwise_codebook, double &inactive_count, double &recovered) { if(cEpochDiagnostics.Total() != 12 || !cEpochDiagnostics.BufferRead()) ReturnFalse; for(int i = 0; i < 12; i++) if(!MathIsValidNumber(cEpochDiagnostics[i])) ReturnFalse; mix_loss = cEpochDiagnostics[0]; router_loss = cEpochDiagnostics[1]; trajectory_loss = cEpochDiagnostics[2]; confidence_loss = cEpochDiagnostics[3]; latent_error = cEpochDiagnostics[4]; observation_error = cEpochDiagnostics[5]; valid_batches = cEpochDiagnostics[6]; invalid_batches = cEpochDiagnostics[7]; usage_entropy = cEpochDiagnostics[8]; pairwise_codebook = cEpochDiagnostics[9]; inactive_count = cEpochDiagnostics[10]; recovered = cEpochDiagnostics[11]; return(true); } //+------------------------------------------------------------------+ //| Executes the read epoch recovery events step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ReadEpochRecoveryEvents(uint &events[]) { if(cRecoveryEvents.Total() != int(iScenarios) || !cRecoveryEvents.BufferRead()) ReturnFalse; if(ArrayResize(events, int(iScenarios)) != int(iScenarios)) ReturnFalse; for(uint i = 0; i < iScenarios; i++) { const double value = cRecoveryEvents[i]; if(!MathIsValidNumber(value) || value < 0.0 || value > 4294967295.0) ReturnFalse; events[i] = (uint)MathRound(value); } return(true); } //+------------------------------------------------------------------+ //| Executes the read epoch age zero events step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ReadEpochAgeZeroEvents(uint &events[]) { if(cAgeZeroEvents.Total() != int(iScenarios) || !cAgeZeroEvents.BufferRead()) ReturnFalse; if(ArrayResize(events, int(iScenarios)) != int(iScenarios)) ReturnFalse; for(uint i = 0; i < iScenarios; i++) { const double value = cAgeZeroEvents[i]; if(!MathIsValidNumber(value) || value < 0.0 || value > 4294967295.0) ReturnFalse; events[i] = (uint)MathRound(value); } return(true); } //+------------------------------------------------------------------+ //| Sets the recovery age of the CNeuronScenarioForecast. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::SetRecoveryAge(const uint age) { if(age == 0) ReturnFalse; iRecoveryAge = age; return(true); } //+------------------------------------------------------------------+ //| Executes the update codebook step of the CNeuronScenarioForecast | //| layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::UpdateCodebook(CBufferFloat *delta_target, float decay) { if(!OpenCL || !delta_target || delta_target.Total() < int(iVariables * iHorizon * iDimension) || delta_target.GetIndex() < 0 || decay < 0 || decay >= 1) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iScenarios, 1, 1}; const int kernel = def_k_ScenarioCodebookEMA; setBuffer(kernel, def_k_sce_target, delta_target.GetIndex()) setBuffer(kernel, def_k_sce_prototypes, cCodebook.GetPrototypes().GetIndex()) setBuffer(kernel, def_k_sce_ema_sums, cCodebook.GetEMASums().GetIndex()) setBuffer(kernel, def_k_sce_ema_counts, cCodebook.GetEMACounts().GetIndex()) setBuffer(kernel, def_k_sce_usage, cCodebook.GetUsage().GetIndex()) setBuffer(kernel, def_k_sce_inactive, cCodebook.GetInactive().GetIndex()) setBuffer(kernel, def_k_sce_batch_control, cBatchControl.GetIndex()) setBuffer(kernel, def_k_sce_responsibilities, cResponsibilities.GetIndex()) setArgument(kernel, def_k_sce_scenarios, (int)iScenarios) setArgument(kernel, def_k_sce_dimension, (int)iDimension) setArgument(kernel, def_k_sce_decay, decay) setBuffer(kernel, def_k_sce_inactivity_age, cCodebook.GetInactivityAge().GetIndex()) setArgument(kernel, def_k_sce_seed, (int)MathRand()) setArgument(kernel, def_k_sce_recovery_age, (int)iRecoveryAge) setBuffer(kernel, def_k_sce_recovery, cRecovery.GetIndex()) setArgument(kernel, def_k_sce_phase, (int)0) setArgument(kernel, def_k_sce_horizon, (int)iHorizon) setArgument(kernel, def_k_sce_variables, (int)iVariables) setBuffer(kernel, def_k_sce_recovery_events, cRecoveryEvents.GetIndex()) setBuffer(kernel, def_k_sce_age_zero_events, cAgeZeroEvents.GetIndex()) kernelExecute(kernel, global_work_offset, global_work_size) global_work_size[0] = iDimension; global_work_size[1] = iVariables; global_work_size[2] = iScenarios; uint local_work_size[3] = {iDimension, iVariables, 1}; setArgument(kernel, def_k_sce_phase, (int)1) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() != int(iVariables * iDimension)) ReturnFalse; if(!cGeneratorInput.CalcHiddenGradients(cGenerator.AsObject())) ReturnFalse; if(!cRouterInput.CalcHiddenGradients(cRouter.AsObject())) ReturnFalse; if(!cConfidenceInput.CalcHiddenGradients(cConfidenceHead.AsObject())) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; const uint lanes = (uint)MathMin((long)64, MathMin(OpenCL.GetMaxLocalSize(0), (long)(iScenarios * iHorizon))); if(lanes == 0) ReturnFalse; uint global_work_size[3] = {lanes, iDimension, iVariables}; uint local_work_size[3] = {lanes, 1, 1}; const int kernel = def_k_ScenarioGradientNormalize; setBuffer(kernel, def_k_sgn_generator_gradient, cGenerator.getGradientIndex()) setBuffer(kernel, def_k_sgn_generator_input_gradient, cGeneratorInput.getGradientIndex()) setBuffer(kernel, def_k_sgn_router_input_gradient, cRouterInput.getGradientIndex()) setBuffer(kernel, def_k_sgn_confidence_input_gradient, cConfidenceInput.getGradientIndex()) setBuffer(kernel, def_k_sgn_input_gradient, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_sgn_responsibilities, cResponsibilities.GetIndex()) setArgument(kernel, def_k_sgn_scenarios, (int)iScenarios) setArgument(kernel, def_k_sgn_horizon, (int)iHorizon) setArgument(kernel, def_k_sgn_dimension, (int)iDimension) setArgument(kernel, def_k_sgn_phase, (int)1) setBuffer(kernel, def_k_sgn_confidence_gradient, cConfidenceHead.getGradientIndex()) setArgument(kernel, def_k_sgn_variables, (int)iVariables) setBuffer(kernel, def_k_sgn_batch_control, cBatchControl.GetIndex()) setArgument(kernel, def_k_sgn_normalize_gradients, (bGradientNormalization ? 1 : 0)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronScenarioForecast trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !bLastTargetAccepted) ReturnFalse; if(!cGenerator.UpdateInputWeights(cGeneratorInput.AsObject())) ReturnFalse; if(!cRouter.UpdateInputWeights(cRouterInput.AsObject())) ReturnFalse; if(!cConfidenceHead.UpdateInputWeights(cConfidenceInput.AsObject())) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronScenarioForecast state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iScenarios); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, iHorizon); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, 0); dFileWriteUInt(file_handle, (uint)bLastTargetAccepted); return(cCodebook.Save(file_handle) && cConfidence.Save(file_handle) && cLogConfidence.Save(file_handle) && cLogProbabilities.Save(file_handle) && cResponsibilities.Save(file_handle) && cTarget.Save(file_handle) && cGeneratorInput.Save(file_handle) && cRouterInput.Save(file_handle) && cConfidenceInput.Save(file_handle) && cGenerator.Save(file_handle) && cRouter.Save(file_handle) && cConfidenceHead.Save(file_handle) && cRouterSoftmax.Save(file_handle) && cResponsibilityScores.Save(file_handle) && cTopKSelector.Save(file_handle)); } //+------------------------------------------------------------------+ //| Append scenario forecast parameters, not batch | //| diagnostics/state. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronBaseOCL::AppendParameterFingerprint(fingerprint) || !cCodebook.AppendParameterFingerprint(fingerprint) || !cGenerator.AppendParameterFingerprint(fingerprint) || !cRouter.AppendParameterFingerprint(fingerprint) || !cConfidenceHead.AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronScenarioForecast state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iScenarios); dFileReadUInt(file_handle, iTopK); dFileReadUInt(file_handle, iHorizon); dFileReadUInt(file_handle, iDimension); uint legacy_assignment = 0; dFileReadUInt(file_handle, legacy_assignment); uint accepted = 0; dFileReadUInt(file_handle, accepted); bLastTargetAccepted = (accepted != 0); bLastControlValid = false; if(iVariables == 0 || iScenarios == 0 || iTopK == 0 || iTopK > iScenarios || iHorizon == 0 || iDimension == 0 || !ValidateLaunchConfiguration(OpenCL, iScenarios, iHorizon, iDimension, iVariables)) ReturnFalse; //--- ValidateLaunchConfiguration proved this product fits an int. const long output_total = (long)iScenarios * iVariables * iHorizon * iDimension; if(Output.Total() != (int)output_total) ReturnFalse; if(!(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cCodebook.Load(file_handle, OpenCL, true) : cCodebook.Load(file_handle, NULL)) || !cCodebook.ValidateContract(iScenarios, iVariables * iHorizon * iDimension)) ReturnFalse; if(!(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cConfidence.LoadInferenceFinite(file_handle, OpenCL) : cConfidence.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cLogConfidence.LoadInferenceFinite(file_handle, OpenCL) : cLogConfidence.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cLogProbabilities.LoadInferenceFinite(file_handle, OpenCL) : cLogProbabilities.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cResponsibilities.LoadInferenceFinite(file_handle, OpenCL) : cResponsibilities.Load(file_handle)) || !(OpenCL && OpenCL.InferenceRuntimeLoadActive() ? cTarget.LoadInferenceFinite(file_handle, OpenCL) : cTarget.Load(file_handle)) || !LoadInsideLayer(file_handle, cGeneratorInput.AsObject()) || !LoadInsideLayer(file_handle, cRouterInput.AsObject()) || !LoadInsideLayer(file_handle, cConfidenceInput.AsObject()) || !LoadInsideLayer(file_handle, cGenerator.AsObject()) || !LoadInsideLayer(file_handle, cRouter.AsObject()) || !LoadInsideLayer(file_handle, cConfidenceHead.AsObject()) || !LoadInsideLayer(file_handle, cRouterSoftmax.AsObject()) || !LoadInsideLayer(file_handle, cResponsibilityScores.AsObject()) || !LoadInsideLayer(file_handle, cTopKSelector.AsObject())) ReturnFalse; if(!cBatchControl.BufferInit(1, 0) || !cRecovery.BufferInit(iScenarios, 0) || !cRecoveryEvents.BufferInit(iScenarios, 0) || !cAgeZeroEvents.BufferInit(iScenarios, 0) || !cForwardValidity.BufferInit(iScenarios, 0) || !cDiagnosticPartials.BufferInit(3 * iScenarios, 0) || !cEpochDiagnostics.BufferInit(12, 0) || !cDiagnosticScratch.BufferInit((iScenarios * (iScenarios - 1) / 2 > iScenarios ? iScenarios * (iScenarios - 1) / 2 : iScenarios), 0)) ReturnFalse; SetOpenCL(OpenCL); return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronScenarioForecast state (OpenCL-aware | //| overload). | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::Load(const int file_handle, COpenCLMy *open_cl) { if(file_handle == INVALID_HANDLE || FileReadInteger(file_handle) != Type()) ReturnFalse; SetOpenCL(open_cl); return(Load(file_handle)); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronScenarioForecast GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronScenarioForecast::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cConfidence.BufferCreate(OpenCL); cLogConfidence.BufferCreate(OpenCL); cLogProbabilities.BufferCreate(OpenCL); cResponsibilities.BufferCreate(OpenCL); cTarget.BufferCreate(OpenCL); cBatchControl.BufferCreate(OpenCL); cRecovery.BufferCreate(OpenCL); cRecoveryEvents.BufferCreate(OpenCL); cAgeZeroEvents.BufferCreate(OpenCL); cForwardValidity.BufferCreate(OpenCL); cDiagnosticPartials.BufferCreate(OpenCL); cEpochDiagnostics.BufferCreate(OpenCL); cDiagnosticScratch.BufferCreate(OpenCL); cCodebook.SetOpenCL(OpenCL); cGeneratorInput.SetOpenCL(OpenCL); cRouterInput.SetOpenCL(OpenCL); cConfidenceInput.SetOpenCL(OpenCL); cGenerator.SetOpenCL(OpenCL); cRouter.SetOpenCL(OpenCL); cConfidenceHead.SetOpenCL(OpenCL); cRouterSoftmax.SetOpenCL(OpenCL); cResponsibilityScores.SetOpenCL(OpenCL); cTopKSelector.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Checks checked-transfer support for ScenarioForecast children. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::SupportsOpenCLChecked(void) const { //--- Accept only the exact owner and every embedded neuron capability. return(Type() == defNeuronScenarioForecast && cGeneratorInput.SupportsOpenCLChecked() && cRouterInput.SupportsOpenCLChecked() && cConfidenceInput.SupportsOpenCLChecked() && cGenerator.SupportsOpenCLChecked() && cRouter.SupportsOpenCLChecked() && cConfidenceHead.SupportsOpenCLChecked() && cRouterSoftmax.SupportsOpenCLChecked() && cResponsibilityScores.SupportsOpenCLChecked() && cTopKSelector.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers ScenarioForecast owners to a checked OpenCL context. | //+------------------------------------------------------------------+ //| Queries the same actual Forecast tree as SetOpenCLChecked | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { if(CNeuronBaseOCL::OwnsBufferOrDeviceSlot(candidate)) return(true); CBufferFloat *owned[] = {GetPointer(cConfidence), GetPointer(cLogConfidence), GetPointer(cLogProbabilities), GetPointer(cResponsibilities), GetPointer(cTarget), GetPointer(cBatchControl), GetPointer(cRecovery), GetPointer(cRecoveryEvents), GetPointer(cAgeZeroEvents), GetPointer(cForwardValidity), GetPointer(cDiagnosticPartials), GetPointer(cEpochDiagnostics), GetPointer(cDiagnosticScratch), cCodebook.GetPrototypes(), cCodebook.GetEMASums(), cCodebook.GetEMACounts(), cCodebook.GetUsage(), cCodebook.GetInactive(), cCodebook.GetInactivityAge() }; for(uint i = 0; i < (uint)ArraySize(owned); i++) if(BufferAliases(candidate, owned[i])) return(true); CNeuronBaseOCL *children[] = {GetPointer(cGeneratorInput), GetPointer(cRouterInput), GetPointer(cConfidenceInput), GetPointer(cGenerator), GetPointer(cRouter), GetPointer(cConfidenceHead), GetPointer(cRouterSoftmax), GetPointer(cResponsibilityScores), GetPointer(cTopKSelector) }; for(uint i = 0; i < (uint)ArraySize(children); i++) if(children[i].OwnsBufferOrDeviceSlot(candidate)) return(true); return(false); } //+------------------------------------------------------------------+ //| Function CNeuronScenarioForecast::SetOpenCLChecked | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and the complete embedded neuron tree first. if(CheckPointer(obj) == POINTER_INVALID) ReturnFalseEx("checked forecast target"); if(!SupportsOpenCLChecked()) ReturnFalseEx("checked forecast capability"); //--- Validate the existing codebook contract before any owner changes. if(cCodebook.Scenarios() == 0) ReturnFalseEx("checked forecast codebook scenarios"); if(cCodebook.Dimension() == 0) ReturnFalseEx("checked forecast codebook dimension"); if(cCodebook.Scenarios() != iScenarios) ReturnFalseEx("checked forecast codebook scenario contract"); if(cCodebook.Dimension() != iVariables * iHorizon * iDimension) ReturnFalseEx("checked forecast codebook dimension contract"); if(!cCodebook.ValidateContract(iScenarios, iVariables * iHorizon * iDimension)) ReturnFalseEx("checked forecast codebook contract"); //--- Read all foreign direct owners before any context recreation. if(cConfidence.Total() > 0 && cConfidence.GetIndex() >= 0 && cConfidence.GetOpenCL() != obj && !cConfidence.BufferRead()) ReturnFalseEx("forecast confidence read"); if(cLogConfidence.Total() > 0 && cLogConfidence.GetIndex() >= 0 && cLogConfidence.GetOpenCL() != obj && !cLogConfidence.BufferRead()) ReturnFalseEx("forecast log confidence read"); if(cLogProbabilities.Total() > 0 && cLogProbabilities.GetIndex() >= 0 && cLogProbabilities.GetOpenCL() != obj && !cLogProbabilities.BufferRead()) ReturnFalseEx("forecast log probabilities read"); if(cResponsibilities.Total() > 0 && cResponsibilities.GetIndex() >= 0 && cResponsibilities.GetOpenCL() != obj && !cResponsibilities.BufferRead()) ReturnFalseEx("forecast responsibilities read"); if(cTarget.Total() > 0 && cTarget.GetIndex() >= 0 && cTarget.GetOpenCL() != obj && !cTarget.BufferRead()) ReturnFalseEx("forecast target read"); if(cBatchControl.Total() > 0 && cBatchControl.GetIndex() >= 0 && cBatchControl.GetOpenCL() != obj && !cBatchControl.BufferRead()) ReturnFalseEx("forecast batch control read"); if(cRecovery.Total() > 0 && cRecovery.GetIndex() >= 0 && cRecovery.GetOpenCL() != obj && !cRecovery.BufferRead()) ReturnFalseEx("forecast recovery read"); if(cRecoveryEvents.Total() > 0 && cRecoveryEvents.GetIndex() >= 0 && cRecoveryEvents.GetOpenCL() != obj && !cRecoveryEvents.BufferRead()) ReturnFalseEx("forecast recovery events read"); if(cAgeZeroEvents.Total() > 0 && cAgeZeroEvents.GetIndex() >= 0 && cAgeZeroEvents.GetOpenCL() != obj && !cAgeZeroEvents.BufferRead()) ReturnFalseEx("forecast age zero events read"); if(cForwardValidity.Total() > 0 && cForwardValidity.GetIndex() >= 0 && cForwardValidity.GetOpenCL() != obj && !cForwardValidity.BufferRead()) ReturnFalseEx("forecast forward validity read"); if(cDiagnosticPartials.Total() > 0 && cDiagnosticPartials.GetIndex() >= 0 && cDiagnosticPartials.GetOpenCL() != obj && !cDiagnosticPartials.BufferRead()) ReturnFalseEx("forecast diagnostic partials read"); if(cEpochDiagnostics.Total() > 0 && cEpochDiagnostics.GetIndex() >= 0 && cEpochDiagnostics.GetOpenCL() != obj && !cEpochDiagnostics.BufferRead()) ReturnFalseEx("forecast epoch diagnostics read"); if(cDiagnosticScratch.Total() > 0 && cDiagnosticScratch.GetIndex() >= 0 && cDiagnosticScratch.GetOpenCL() != obj && !cDiagnosticScratch.BufferRead()) ReturnFalseEx("forecast diagnostic scratch read"); //--- Transfer the codebook and embedded neurons in their legacy order. if(!cCodebook.SetOpenCLChecked(obj)) ReturnFalseEx("forecast codebook context"); if(!cGeneratorInput.SetOpenCLChecked(obj)) ReturnFalseEx("forecast generator input context"); if(!cRouterInput.SetOpenCLChecked(obj)) ReturnFalseEx("forecast router input context"); if(!cConfidenceInput.SetOpenCLChecked(obj)) ReturnFalseEx("forecast confidence input context"); if(!cGenerator.SetOpenCLChecked(obj)) ReturnFalseEx("forecast generator context"); if(!cRouter.SetOpenCLChecked(obj)) ReturnFalseEx("forecast router context"); if(!cConfidenceHead.SetOpenCLChecked(obj)) ReturnFalseEx("forecast confidence head context"); if(!cRouterSoftmax.SetOpenCLChecked(obj)) ReturnFalseEx("forecast router softmax context"); if(!cResponsibilityScores.SetOpenCLChecked(obj)) ReturnFalseEx("forecast responsibility scores context"); if(!cTopKSelector.SetOpenCLChecked(obj)) ReturnFalseEx("forecast top K selector context"); //--- Recreate only nonempty direct buffers missing from the target context. if(cConfidence.Total() > 0 && (cConfidence.GetOpenCL() != obj || cConfidence.GetIndex() < 0) && !cConfidence.BufferCreate(obj)) ReturnFalseEx("forecast confidence create"); if(cLogConfidence.Total() > 0 && (cLogConfidence.GetOpenCL() != obj || cLogConfidence.GetIndex() < 0) && !cLogConfidence.BufferCreate(obj)) ReturnFalseEx("forecast log confidence create"); if(cLogProbabilities.Total() > 0 && (cLogProbabilities.GetOpenCL() != obj || cLogProbabilities.GetIndex() < 0) && !cLogProbabilities.BufferCreate(obj)) ReturnFalseEx("forecast log probabilities create"); if(cResponsibilities.Total() > 0 && (cResponsibilities.GetOpenCL() != obj || cResponsibilities.GetIndex() < 0) && !cResponsibilities.BufferCreate(obj)) ReturnFalseEx("forecast responsibilities create"); if(cTarget.Total() > 0 && (cTarget.GetOpenCL() != obj || cTarget.GetIndex() < 0) && !cTarget.BufferCreate(obj)) ReturnFalseEx("forecast target create"); if(cBatchControl.Total() > 0 && (cBatchControl.GetOpenCL() != obj || cBatchControl.GetIndex() < 0) && !cBatchControl.BufferCreate(obj)) ReturnFalseEx("forecast batch control create"); if(cRecovery.Total() > 0 && (cRecovery.GetOpenCL() != obj || cRecovery.GetIndex() < 0) && !cRecovery.BufferCreate(obj)) ReturnFalseEx("forecast recovery create"); if(cRecoveryEvents.Total() > 0 && (cRecoveryEvents.GetOpenCL() != obj || cRecoveryEvents.GetIndex() < 0) && !cRecoveryEvents.BufferCreate(obj)) ReturnFalseEx("forecast recovery events create"); if(cAgeZeroEvents.Total() > 0 && (cAgeZeroEvents.GetOpenCL() != obj || cAgeZeroEvents.GetIndex() < 0) && !cAgeZeroEvents.BufferCreate(obj)) ReturnFalseEx("forecast age zero events create"); if(cForwardValidity.Total() > 0 && (cForwardValidity.GetOpenCL() != obj || cForwardValidity.GetIndex() < 0) && !cForwardValidity.BufferCreate(obj)) ReturnFalseEx("forecast forward validity create"); if(cDiagnosticPartials.Total() > 0 && (cDiagnosticPartials.GetOpenCL() != obj || cDiagnosticPartials.GetIndex() < 0) && !cDiagnosticPartials.BufferCreate(obj)) ReturnFalseEx("forecast diagnostic partials create"); if(cEpochDiagnostics.Total() > 0 && (cEpochDiagnostics.GetOpenCL() != obj || cEpochDiagnostics.GetIndex() < 0) && !cEpochDiagnostics.BufferCreate(obj)) ReturnFalseEx("forecast epoch diagnostics create"); if(cDiagnosticScratch.Total() > 0 && (cDiagnosticScratch.GetOpenCL() != obj || cDiagnosticScratch.GetIndex() < 0) && !cDiagnosticScratch.BufferCreate(obj)) ReturnFalseEx("forecast diagnostic scratch create"); //--- Transfer inherited owners and publish the Forecast context last. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("forecast inherited base context"); //--- Finalize the checked ScenarioForecast context transfer. return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ void CNeuronScenarioForecast::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cGenerator.TrainMode(flag); cRouter.TrainMode(flag); cConfidenceHead.TrainMode(flag); cRouterSoftmax.TrainMode(flag); cTopKSelector.TrainMode(flag); } //+------------------------------------------------------------------+ //| Averages the CNeuronScenarioForecast weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronScenarioForecast *Source = source; return(CNeuronBaseOCL::WeightsUpdate(source, tau) && cGenerator.WeightsUpdate(Source.cGenerator.AsObject(), tau) && cRouter.WeightsUpdate(Source.cRouter.AsObject(), tau) && cConfidenceHead.WeightsUpdate(Source.cConfidenceHead.AsObject(), tau)); } //+------------------------------------------------------------------+ //| Returns the trainable weight parameters of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ uint CNeuronScenarioForecast::TrainableWeights(void) { CBufferFloat *generator = cGenerator.GetWeightsConv(); CBufferFloat *router = cRouter.GetWeightsConv(); CBufferFloat *confidence = cConfidenceHead.GetWeightsConv(); return(uint((generator ? generator.Total() : 0) + (router ? router.Total() : 0) + (confidence ? confidence.Total() : 0))); } //+------------------------------------------------------------------+ //| Executes the active trajectories step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ uint CNeuronScenarioForecast::ActiveTrajectories(void) { if(!!OpenCL && !cResponsibilities.BufferRead()) return(0); uint result = 0; for(uint k = 0; k < iScenarios; k++) if(cResponsibilities[k] > 0) result++; return(result); } //+------------------------------------------------------------------+ //| Executes the dense router gradient count step of the | //| CNeuronScenarioForecast layer. | //+------------------------------------------------------------------+ uint CNeuronScenarioForecast::DenseRouterGradientCount(void) { CBufferFloat *gradient = cRouter.getGradient(); if(!gradient || (!!OpenCL && !gradient.BufferRead())) return(0); uint result = 0; for(uint k = 0; k < iScenarios; k++) if(MathIsValidNumber(gradient[k]) && MathAbs(gradient[k]) > 1.0e-12f) result++; return(result); } //+------------------------------------------------------------------+ //| CPU export for smoke/serialization only; never use in hot path | //+------------------------------------------------------------------+ ulong CNeuronScenarioForecast::ContractSignature(void) { ulong hash = 1469598103934665603; uint values[6] = {uint(Type()), iVariables, iScenarios, iTopK, iHorizon, iDimension}; for(uint i = 0; i < 6; i++) { hash ^= ulong(values[i]); hash *= ulong(1099511628211); } return(hash); } //+------------------------------------------------------------------+ //| Executes the export detached step of the CNeuronScenarioForecast | //| layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::ExportDetached(CBufferFloat *z, CBufferFloat *u, CBufferFloat *log_u, CBufferFloat *pi, CBufferFloat *log_pi) { if(!z || !u || !log_u || !pi || !log_pi || !Output.BufferRead() || !cConfidence.BufferRead() || !cLogConfidence.BufferRead() || !cRouterSoftmax.getOutput().BufferRead() || !cLogProbabilities.BufferRead()) ReturnFalse; if(!z.BufferInit(Output.Total(), 0) || !u.BufferInit(cConfidence.Total(), 0) || !log_u.BufferInit(cLogConfidence.Total(), 0) || !pi.BufferInit(cRouterSoftmax.Neurons(), 0) || !log_pi.BufferInit(cLogProbabilities.Total(), 0)) ReturnFalse; return(z.Fill(GetPointer(Output)) && u.Fill(GetPointer(cConfidence)) && log_u.Fill(GetPointer(cLogConfidence)) && pi.Fill(cRouterSoftmax.getOutput()) && log_pi.Fill(GetPointer(cLogProbabilities))); } //+------------------------------------------------------------------+ //| Clears the CNeuronScenarioForecast execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronScenarioForecast::Clear(void) { bLastTargetAccepted = false; bLastControlValid = false; return(CNeuronBaseOCL::Clear() && cGeneratorInput.Clear() && cRouterInput.Clear() && cConfidenceInput.Clear() && cGenerator.Clear() && cRouter.Clear() && cConfidenceHead.Clear() && cConfidence.Fill(1.0f) && cLogConfidence.Fill(MathLog(1.000001f)) && cLogProbabilities.Fill(MathLog(1.0f / MathMax(iScenarios, 1))) && cResponsibilities.Fill(0) && cTarget.Fill(0) && cBatchControl.Fill(0) && cRecovery.Fill(0) && cRecoveryEvents.Fill(0) && cAgeZeroEvents.Fill(0) && cForwardValidity.Fill(0) && cDiagnosticPartials.Fill(0) && cEpochDiagnostics.Fill(0) && cDiagnosticScratch.Fill(0)); } //+------------------------------------------------------------------+ //| Initializes the ForecastPrice lifecycle | //+------------------------------------------------------------------+ CNeuronScenarioForecastPriceOCL::CNeuronScenarioForecastPriceOCL(void) : m_pricePhase(defForecastPriceVerify), m_frameState(defForecastPriceFrameFree), m_generation(0), m_supervisionGeneration(0), m_nativePrepared(false), m_backwardPrepared(false), m_priceActive(false) { CNeuronScenarioForecast::TrainMode(false); } //+------------------------------------------------------------------+ //| Validates the inherited ForecastPrice tensor contract | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::ValidatePriceContract(void) const { if(iScenarios == 0 || iVariables == 0 || iHorizon == 0 || iDimension == 0 || iTopK == 0 || iTopK > iScenarios || iScenarios > 256 || iVariables > 256 || iDimension > 256) return(false); const ulong vh = (ulong)iVariables * iHorizon; if(vh > (ulong)INT_MAX / iDimension) return(false); const ulong target = vh * iDimension; if(target > (ulong)INT_MAX / (2 * (ulong)iScenarios)) return(false); const int total = (int)(target * iScenarios); const int confidence = (int)(vh * iScenarios); if(CheckPointer(Output) == POINTER_INVALID || CheckPointer(Gradient) == POINTER_INVALID || Output.Total() != total || Gradient.Total() != total || activation != None || (CheckPointer(Weights) != POINTER_INVALID && Weights.Total() != 0)) return(false); return(cCodebook.Scenarios() == iScenarios && cCodebook.Dimension() == (uint)target && cConfidence.Total() == confidence && cLogConfidence.Total() == confidence && cLogProbabilities.Total() == (int)iScenarios && cResponsibilities.Total() == (int)iScenarios && cTarget.Total() == (int)target && cGenerator.Neurons() == total && cConfidenceHead.Neurons() == confidence && cRouter.Neurons() == (int)iScenarios && cGeneratorInput.Neurons() == 2 * total && cConfidenceInput.Neurons() == 2 * total && cRouterInput.Neurons() == (int)(2 * (ulong)iScenarios * iVariables * iDimension) && cRouterSoftmax.Neurons() == (int)iScenarios && cResponsibilityScores.Neurons() == (int)iScenarios && cTopKSelector.Neurons() == (int)iTopK); } //+------------------------------------------------------------------+ //| Allocates transient bridge scratch outside the checkpoint ABI | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::InitPriceBridge(void) { if(!OpenCL || !ValidatePriceContract()) return(false); const uint state = iVariables * iDimension; const uint target = state * iHorizon; const uint total = target * iScenarios; CBufferFloat *buffers[] = {GetPointer(m_priceGradient), GetPointer(m_generatorCombined), GetPointer(m_priceInputGradient), GetPointer(m_deltaTarget), GetPointer(m_bridgeScratch), GetPointer(m_bridgeControl) }; const uint sizes[] = {total, total, state, target, 2 * state, 2}; for(uint i = 0; i < 6; i++) { if(buffers[i].GetIndex() >= 0 && !buffers[i].BufferFree()) return(false); if(!buffers[i].BufferInit(sizes[i], 0.0f) || !buffers[i].BufferCreate(OpenCL)) return(false); } return(ValidatePriceBridge()); } //+------------------------------------------------------------------+ //| Requires exact native Conv geometry and separately owned scratch | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::ValidatePriceBridge(void) { if(!ValidatePriceContract() || !OpenCL || cGenerator.GetOpenCL() != OpenCL || cGenerator.GetWindow() != 2 * iDimension || cGenerator.GetInputStride() != 2 * iDimension || cGenerator.GetFilters() != iDimension || cGenerator.GetVariables() != 1 || cGenerator.GetUnits() != iScenarios * iVariables * iHorizon || cGenerator.Activation() != None || cGeneratorInput.Activation() != None) return(false); CBufferFloat *weights = cGenerator.GetWeightsConv(); const uint count = (2 * iDimension + 1) * iDimension; if(!weights || weights.Total() != (int)count || weights.GetOpenCL() != OpenCL || weights.GetIndex() < 0 || OpenCL.GetBufferSize(weights.GetIndex()) != (ulong)count * sizeof(float)) return(false); const uint state = iVariables * iDimension; const uint target = state * iHorizon; const uint total = target * iScenarios; CBufferFloat *buffers[] = {GetPointer(m_priceGradient), GetPointer(m_generatorCombined), GetPointer(m_priceInputGradient), GetPointer(m_deltaTarget), GetPointer(m_bridgeScratch), GetPointer(m_bridgeControl) }; const uint sizes[] = {total, total, state, target, 2 * state, 2}; for(uint i = 0; i < 6; i++) { if(buffers[i].Total() != (int)sizes[i] || buffers[i].GetOpenCL() != OpenCL || buffers[i].GetIndex() < 0 || OpenCL.GetBufferSize(buffers[i].GetIndex()) != (ulong)sizes[i] * sizeof(float) || CNeuronScenarioForecast::OwnsBufferOrDeviceSlot(buffers[i])) return(false); for(uint j = 0; j < i; j++) if(BufferAliases(buffers[i], buffers[j])) return(false); } return(true); } //+------------------------------------------------------------------+ //| Stages exact E/dGenerator sums and reads only compact admission | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::DispatchPriceBridge(CNeuronBaseOCL *source, const uint phase) { if(!ValidatePriceBridge() || phase > 3 || (phase != 3 && (!source || source.GetOpenCL() != OpenCL || source.Neurons() != (int)(iVariables * iDimension)))) return(false); const uint kernel = def_k_DPCCForecastPriceBridge; setBuffer(kernel, 0, m_priceGradient.GetIndex()); setBuffer(kernel, 1, cGenerator.GetWeightsConv().GetIndex()); setBuffer(kernel, 2, cGenerator.getGradientIndex()); setBuffer(kernel, 3, (source ? source.getGradientIndex() : Gradient.GetIndex())); setBuffer(kernel, 4, m_generatorCombined.GetIndex()); setBuffer(kernel, 5, m_priceInputGradient.GetIndex()); setBuffer(kernel, 6, m_bridgeScratch.GetIndex()); setBuffer(kernel, 7, m_bridgeControl.GetIndex()); setArgument(kernel, 8, iScenarios); setArgument(kernel, 9, iVariables); setArgument(kernel, 10, iHorizon); setArgument(kernel, 11, iDimension); setArgument(kernel, 12, phase); setBuffer(kernel, 13, m_deltaTarget.GetIndex()); uint offset[1] = {0}; uint size[1] = {iVariables * iDimension}; kernelExecute(kernel, offset, size) setArgument(kernel, 12, (uint)1); size[0] = 1; kernelExecute(kernel, offset, size) if(!m_bridgeControl.BufferRead() || !MathIsValidNumber(m_bridgeControl[0]) || !MathIsValidNumber(m_bridgeControl[1]) || m_bridgeControl[0] != 0.0f) return(false); if(phase == 0) m_priceActive = (m_bridgeControl[1] > 0.0f); return(true); } //+------------------------------------------------------------------+ //| Creates ForecastPrice without a generic fully connected edge | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint scenarios, uint top_k, uint horizon, uint latent, ENUM_OPTIMIZATION optimization_type, uint batch, uint variables) { if(numOutputs != 0 || m_frameState != defForecastPriceFrameFree) ReturnFalseEx("ForecastPrice Init contract"); if(!CNeuronScenarioForecast::Init(0, myIndex, open_cl, scenarios, top_k, horizon, latent, optimization_type, batch, variables)) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice native Init"); } m_generation = 0; m_supervisionGeneration = 0; m_nativePrepared = false; m_pricePhase = defForecastPriceVerify; CNeuronScenarioForecast::TrainMode(false); m_backwardPrepared = false; m_priceActive = false; if(!ValidatePriceContract() || !InitPriceBridge()) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice initialized shape/bridge"); } return(true); } //+------------------------------------------------------------------+ //| Changes price phase only between complete frames | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::SetPricePhase(const uint phase) { if(m_frameState != defForecastPriceFrameFree || phase < defForecastPriceLatentPrepare || phase > defForecastPriceVerify) return(false); m_pricePhase = phase; TrainMode(phase == defForecastPriceLatentPrepare || phase == defForecastPriceJoint); return(true); } //+------------------------------------------------------------------+ //| Publishes one native forecast generation | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::feedForward(CNeuronBaseOCL *source) { if(m_frameState != defForecastPriceFrameFree || m_generation == (ulong) - 1 || !ValidatePriceContract()) ReturnFalseEx("ForecastPrice forward frame"); m_frameState = defForecastPriceFrameLive; m_backwardPrepared = false; m_priceActive = false; m_nativePrepared = false; m_supervisionGeneration = 0; bLastTargetAccepted = false; bLastControlValid = false; if(!CNeuronScenarioForecast::feedForward(source)) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice native forward"); } m_generation++; return(true); } //+------------------------------------------------------------------+ //| Prepares native latent signals for the current price frame | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::PrepareLatentSupervision(CBufferFloat *full_future, CBufferFloat *delta_future) { const bool trainable_phase = (m_pricePhase == defForecastPriceLatentPrepare || m_pricePhase == defForecastPriceJoint); const bool frozen_phase = (m_pricePhase == defForecastPriceOMPBCalibrate || m_pricePhase == defForecastPriceAdapt || m_pricePhase == defForecastPriceVerify); if(m_frameState != defForecastPriceFrameLive || m_nativePrepared || (!trainable_phase && !frozen_phase) || (trainable_phase && !bTrain) || (frozen_phase && (bTrain || bCodebookUpdate)) || !ValidatePriceContract() || CheckPointer(full_future) == POINTER_INVALID || CheckPointer(delta_future) == POINTER_INVALID) ReturnFalseEx("ForecastPrice latent phase/frame"); const int target = (int)((ulong)iVariables * iHorizon * iDimension); if(full_future.Total() != target || delta_future.Total() != target || full_future.GetOpenCL() != OpenCL || delta_future.GetOpenCL() != OpenCL || full_future.GetIndex() < 0 || delta_future.GetIndex() < 0 || OwnsBufferOrDeviceSlot(full_future) || OwnsBufferOrDeviceSlot(delta_future) || BufferAliases(full_future, delta_future) || OpenCL.GetBufferSize(full_future.GetIndex()) != (ulong)target * sizeof(float) || OpenCL.GetBufferSize(delta_future.GetIndex()) != (ulong)target * sizeof(float)) ReturnFalseEx("ForecastPrice future buffer contract"); //--- Preserve the latent delta for the phase-authorized single EMA commit. if(!ValidatePriceBridge() || !CopyBufferRaw(delta_future, GetPointer(m_deltaTarget), target) || !DispatchPriceBridge(NULL, 3)) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice saved latent delta"); } //--- Native signal buffers remain owned by this unchanged Forecast tree. const bool codebook_enabled = bCodebookUpdate; //--- T04 must commit EMA only together with the complete gradient/update transaction. bCodebookUpdate = false; const bool accepted = CNeuronScenarioForecast::BuildResponsibilities(full_future, delta_future); bCodebookUpdate = codebook_enabled; if(!accepted) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice native supervision"); } m_nativePrepared = true; m_supervisionGeneration = m_generation; return(true); } //+------------------------------------------------------------------+ //| Releases a frame without an optimizer write | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::CancelTrainingFrame(void) { if(m_frameState == defForecastPriceFrameFailed) return(false); m_backwardPrepared = false; m_priceActive = false; m_nativePrepared = false; m_supervisionGeneration = 0; bLastTargetAccepted = false; bLastControlValid = false; m_frameState = defForecastPriceFrameFree; return(true); } //+------------------------------------------------------------------+ //| Adds exact detached price contributions after native backward | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::calcInputGradients(CNeuronBaseOCL *source) { //--- Decoder-only adaptation stops the signal at the frozen Forecast boundary. if(m_pricePhase == defForecastPriceAdapt) { if(bTrain || bCodebookUpdate || !PriceSupervisionPrepared() || m_backwardPrepared || !ValidatePriceBridge() || !source || source.GetOpenCL() != OpenCL || source.Neurons() != (int)(iVariables * iDimension) || !source.getGradient() || source.getGradient().GetOpenCL() != OpenCL || source.getGradient().Total() != source.Neurons() || !source.getGradient().Fill(0.0f)) ReturnFalseEx("ForecastPrice frozen backward phase/frame"); m_backwardPrepared = true; return(true); } //--- OMPB calibration traverses the frozen native Forecast without a price signal. const bool calibration_phase = (m_pricePhase == defForecastPriceOMPBCalibrate); if((calibration_phase ? (bTrain || bCodebookUpdate) : !bTrain) || !PriceSupervisionPrepared() || m_backwardPrepared || !ValidatePriceBridge() || !source || source.GetOpenCL() != OpenCL || source.Neurons() != (int)(iVariables * iDimension) || (!calibration_phase && m_pricePhase != defForecastPriceLatentPrepare && m_pricePhase != defForecastPriceJoint)) ReturnFalseEx("ForecastPrice backward phase/frame"); m_priceActive = false; if(m_pricePhase == defForecastPriceJoint && (!CopyBufferRaw(Gradient, GetPointer(m_priceGradient), Gradient.Total()) || !DispatchPriceBridge(source, 0))) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice external dZ admission"); } //--- The parent consumes the already prepared native signals exactly once. if(!CNeuronScenarioForecast::calcInputGradients(source)) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice native backward"); } if(m_priceActive) { //--- Stage before publishing; price never traverses native RMS/rho normalization. if(!DispatchPriceBridge(source, 2) || !CopyBufferRaw(GetPointer(m_generatorCombined), cGenerator.getGradient(), cGenerator.Neurons()) || !CopyBufferRaw(GetPointer(m_priceInputGradient), source.getGradient(), source.Neurons())) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice combined gradient publication"); } } //--- Zero external dZ leaves native buffers byte-identical, including signed zero. m_backwardPrepared = true; return(true); } //+------------------------------------------------------------------+ //| Commits one inherited optimizer update and one permitted EMA | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::updateInputWeights(CNeuronBaseOCL *source) { if(!bTrain || !PriceSupervisionPrepared() || !m_backwardPrepared || !ValidatePriceBridge() || !source || source.GetOpenCL() != OpenCL || source.Neurons() != (int)(iVariables * iDimension) || (m_pricePhase != defForecastPriceLatentPrepare && m_pricePhase != defForecastPriceJoint)) ReturnFalseEx("ForecastPrice update phase/frame"); //--- The summed output signal is linear in native Conv dW accumulation. //--- Its one native update applies clipping once and advances Adam once. m_backwardPrepared = false; if(!CNeuronScenarioForecast::updateInputWeights(source) || (bCodebookUpdate && !UpdateCodebook(GetPointer(m_deltaTarget)))) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice combined update/EMA"); } return(CancelTrainingFrame()); } //+------------------------------------------------------------------+ //| Preserves the phase gate when the previous layer remains | //| trainable | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::UpdateInputWeights(CObject *source, CBufferFloat *second) { if(second || CheckPointer(source) == POINTER_INVALID) return(false); if(m_pricePhase == defForecastPriceLatentPrepare || m_pricePhase == defForecastPriceJoint) { CNeuronBaseOCL *previous = source; return(updateInputWeights(previous)); } return(UpdateInputState(source, second)); } //+------------------------------------------------------------------+ //| Applies the same optimizer gate to object second inputs | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::UpdateInputWeightsSecond(CObject *source, CNeuronBaseOCL *second) { return(UpdateInputWeights(source, second ? second.getOutput() : NULL)); } //+------------------------------------------------------------------+ //| Releases frozen frames without changing weights or counters | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::UpdateInputState(CObject *source, CBufferFloat *second) { if(m_pricePhase == defForecastPriceLatentPrepare || m_pricePhase == defForecastPriceJoint) return(false); return(CancelTrainingFrame()); } //+------------------------------------------------------------------+ //| Releases frozen frames with an object second input | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::UpdateInputStateSecond(CObject *source, CNeuronBaseOCL *second) { return(UpdateInputState(source, second ? second.getOutput() : NULL)); } //+------------------------------------------------------------------+ //| Returns the original Forecast descriptor axes with the new type | //+------------------------------------------------------------------+ CLayerDescription *CNeuronScenarioForecastPriceOCL::GetLayerInfo(void) { if(!ValidatePriceContract()) return(NULL); CLayerDescription *result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return(NULL); result.count = (int)iScenarios; result.variables = (int)iTopK; result.window_out = (int)iHorizon; result.window = (int)iDimension; result.layers = (int)iVariables; result.activation = None; result.batch = (int)iBatch; return(result); } //+------------------------------------------------------------------+ //| Saves an explicit price header followed by the native base state | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE || m_frameState != defForecastPriceFrameFree || !ValidatePriceContract()) ReturnFalseEx("ForecastPrice Save frame"); dFileWriteUInt(file_handle, defNeuronScenarioForecastPriceOCL); dFileWriteUInt(file_handle, defScenarioForecastPriceABI); dFileWriteUInt(file_handle, iScenarios); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, iHorizon); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iVariables); const ulong native_start = FileTell(file_handle); if(!CNeuronScenarioForecast::Save(file_handle)) ReturnFalseEx("ForecastPrice native Save"); const ulong native_end = FileTell(file_handle); //--- Parent Save uses virtual Type; make its nested base header explicit. if(!FileSeek(file_handle, (long)native_start, SEEK_SET)) ReturnFalseEx("ForecastPrice nested header seek"); const bool header_written = (FileWriteInteger(file_handle, defNeuronScenarioForecast) == INT_VALUE); const bool cursor_restored = FileSeek(file_handle, (long)native_end, SEEK_SET); if(!header_written || !cursor_restored) ReturnFalseEx("ForecastPrice nested base header"); return(true); } //+------------------------------------------------------------------+ //| Loads ABI and shape before the native Forecast parameters | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE || m_frameState != defForecastPriceFrameFree) ReturnFalseEx("ForecastPrice Load frame"); const ulong position = FileTell(file_handle); const ulong size = FileSize(file_handle); if(position > size || size - position < 7 * sizeof(uint)) ReturnFalseEx("ForecastPrice header length"); uint abi, scenarios, top_k, horizon, dimension, variables, base_type; dFileReadUInt(file_handle, abi); dFileReadUInt(file_handle, scenarios); dFileReadUInt(file_handle, top_k); dFileReadUInt(file_handle, horizon); dFileReadUInt(file_handle, dimension); dFileReadUInt(file_handle, variables); dFileReadUInt(file_handle, base_type); if(abi != defScenarioForecastPriceABI || base_type != defNeuronScenarioForecast || top_k == 0 || top_k > scenarios || !ValidateLaunchConfiguration(OpenCL, scenarios, horizon, dimension, variables)) ReturnFalseEx("ForecastPrice ABI/shape header"); if(!CNeuronScenarioForecast::Load(file_handle) || m_frameState != defForecastPriceFrameFree) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice native Load"); } if(iScenarios != scenarios || iTopK != top_k || iHorizon != horizon || iDimension != dimension || iVariables != variables || !ValidatePriceContract() || !InitPriceBridge()) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice loaded shape/bridge"); } m_generation = 0; m_supervisionGeneration = 0; m_nativePrepared = false; m_pricePhase = defForecastPriceVerify; m_backwardPrepared = false; m_priceActive = false; bLastTargetAccepted = false; bLastControlValid = false; CNeuronScenarioForecast::TrainMode(false); return(true); } //+------------------------------------------------------------------+ //| Loads an explicit outer ForecastPrice type in a supplied context | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::Load(const int file_handle, COpenCLMy *open_cl) { if(file_handle == INVALID_HANDLE || CheckPointer(open_cl) == POINTER_INVALID || m_frameState != defForecastPriceFrameFree) return(false); const ulong position = FileTell(file_handle); const ulong size = FileSize(file_handle); if(position > size || size - position < sizeof(uint) || FileReadInteger(file_handle) != defNeuronScenarioForecastPriceOCL) return(false); AttachOpenCL(open_cl); return(Load(file_handle)); } //+------------------------------------------------------------------+ //| Attaches an unloaded ForecastPrice owner | //+------------------------------------------------------------------+ void CNeuronScenarioForecastPriceOCL::AttachOpenCL(COpenCLMy *obj) { if(m_frameState != defForecastPriceFrameFree || iScenarios != 0) { Print("ForecastPrice AttachOpenCL requires an unloaded owner"); return; } CNeuronBaseOCL::AttachOpenCL(obj); } //+------------------------------------------------------------------+ //| Rebinds native owners only outside a live training frame | //+------------------------------------------------------------------+ void CNeuronScenarioForecastPriceOCL::SetOpenCL(COpenCLMy *obj) { if(m_frameState != defForecastPriceFrameFree) { Print("ForecastPrice SetOpenCL requires FREE"); return; } if(!SetOpenCLChecked(obj)) m_frameState = defForecastPriceFrameFailed; } //+------------------------------------------------------------------+ //| Accepts the exact new owner and every inherited checked child | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::SupportsOpenCLChecked(void) const { return(Type() == defNeuronScenarioForecastPriceOCL && cGeneratorInput.SupportsOpenCLChecked() && cRouterInput.SupportsOpenCLChecked() && cConfidenceInput.SupportsOpenCLChecked() && cGenerator.SupportsOpenCLChecked() && cRouter.SupportsOpenCLChecked() && cConfidenceHead.SupportsOpenCLChecked() && cRouterSoftmax.SupportsOpenCLChecked() && cResponsibilityScores.SupportsOpenCLChecked() && cTopKSelector.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers the unchanged native ownership tree to a checked | //| context | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::SetOpenCLChecked(COpenCLMy *obj) { if(m_frameState != defForecastPriceFrameFree || !ValidatePriceContract() || !ValidateLaunchConfiguration(obj, iScenarios, iHorizon, iDimension, iVariables)) ReturnFalseEx("ForecastPrice checked transfer frame/shape"); if(!CNeuronScenarioForecast::SetOpenCLChecked(obj) || !InitPriceBridge()) { m_frameState = defForecastPriceFrameFailed; ReturnFalseEx("ForecastPrice checked native/bridge transfer"); } return(true); } //+------------------------------------------------------------------+ //| Uses exactly the native Forecast ownership tree | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { CBufferFloat *buffers[] = {GetPointer(m_priceGradient), GetPointer(m_generatorCombined), GetPointer(m_priceInputGradient), GetPointer(m_deltaTarget), GetPointer(m_bridgeScratch), GetPointer(m_bridgeControl) }; for(uint i = 0; i < 6; i++) if(BufferAliases(candidate, buffers[i])) return(true); return(CNeuronScenarioForecast::OwnsBufferOrDeviceSlot(candidate)); } //+------------------------------------------------------------------+ //| Freezes every inherited child in the nontraining price phases | //+------------------------------------------------------------------+ void CNeuronScenarioForecastPriceOCL::TrainMode(bool flag) { CNeuronScenarioForecast::TrainMode(flag && (m_pricePhase == defForecastPriceLatentPrepare || m_pricePhase == defForecastPriceJoint)); } //+------------------------------------------------------------------+ //| Transfers inherited weights only between matching free owners | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != defNeuronScenarioForecastPriceOCL || m_frameState != defForecastPriceFrameFree || !ValidatePriceContract() || source.GetOpenCL() != OpenCL || !MathIsValidNumber(tau) || tau < 0 || tau > 1) return(false); CNeuronScenarioForecastPriceOCL *price_source = source; if(price_source.m_frameState != defForecastPriceFrameFree || !price_source.ValidatePriceContract() || price_source.iScenarios != iScenarios || price_source.iTopK != iTopK || price_source.iHorizon != iHorizon || price_source.iDimension != iDimension || price_source.iVariables != iVariables) return(false); if(!CNeuronScenarioForecast::WeightsUpdate(source, tau)) { m_frameState = defForecastPriceFrameFailed; return(false); } return(true); } //+------------------------------------------------------------------+ //| Clears native transient state without recovering a failed update | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::Clear(void) { if(m_frameState != defForecastPriceFrameFree || !ValidatePriceContract()) return(false); if(!CNeuronScenarioForecast::Clear()) { m_frameState = defForecastPriceFrameFailed; return(false); } return(CancelTrainingFrame()); } //+------------------------------------------------------------------+ //| Reuses native frozen inference state only at a free frame | //| boundary | //+------------------------------------------------------------------+ bool CNeuronScenarioForecastPriceOCL::InferenceRuntimeSidecar(const int handle, const bool restore) { if(m_frameState != defForecastPriceFrameFree || bTrain || !ValidatePriceContract()) return(false); const bool codebook_enabled = bCodebookUpdate; bCodebookUpdate = false; const bool accepted = CNeuronScenarioForecast::InferenceRuntimeSidecar(handle, restore); bCodebookUpdate = codebook_enabled; if(!accepted && restore) m_frameState = defForecastPriceFrameFailed; return(accepted); } //+------------------------------------------------------------------+ //| Initializes the CNeuronScenarioCrossAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint scenarios, uint horizon, uint latent, bool critic, ENUM_OPTIMIZATION optimization_type, uint batch, uint variables, uint stack_size, uint top_k) { if(!open_cl || variables == 0 || scenarios == 0 || horizon == 0 || latent == 0 || stack_size == 0 || top_k == 0 || top_k > scenarios) ReturnFalse; iVariables = variables; iScenarios = scenarios; iHorizon = horizon; iDimension = latent; iStackSize = stack_size; iTopK = top_k; bCritic = critic; iQueries = (bCritic ? 5 : 3); if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, iQueries, iDimension, 1, optimization_type, batch) || !InitHolders(myIndex, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronScenarioCrossAttention layer and | //| allocates its GPU buffers (InitHolders). | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::InitHolders(uint myIndex, ENUM_OPTIMIZATION optimization_type, uint batch) { uint state_windows[]; ArrayResize(state_windows, bCritic ? 5 : 3); state_windows[0] = 4; state_windows[1] = 5; state_windows[2] = 4; if(bCritic) { state_windows[3] = 3; state_windows[4] = 3; } const uint tokens = iScenarios * iVariables * iHorizon; const uint mpi_dimension = iQueries * iDimension; const uint mpi_heads = 4; const uint mpi_key = mpi_dimension / mpi_heads; uint index = myIndex + 1; if(!cQueryProjection.Init(0, index++, OpenCL, state_windows, iScenarios * iDimension, 1, 1, optimization_type, batch)) ReturnFalse; if(!cQueryGradient.Init(0, index++, OpenCL, iQueries * iScenarios * iDimension, optimization_type, batch)) ReturnFalse; if(!cWz.Init(0, index++, OpenCL, iDimension, iDimension, iDimension, tokens, optimization_type, batch)) ReturnFalse; if(!cWu.Init(0, index++, OpenCL, 1, 1, iDimension, tokens, optimization_type, batch)) ReturnFalse; if(!cTokenSum.Init(0, index++, OpenCL, tokens * iDimension, optimization_type, batch)) ReturnFalse; if(!cTokenPE.Init(0, index++, OpenCL, tokens * iDimension, optimization_type, batch)) ReturnFalse; if(!cTokensTranspose.Init(0, index++, OpenCL, iScenarios, iVariables * iHorizon, iDimension, optimization_type, batch)) ReturnFalse; if(!cKV.Init(0, index++, OpenCL, 2 * tokens * iDimension, optimization_type, batch)) ReturnFalse; if(!cKVGradK.Init(0, index++, OpenCL, tokens * iDimension, optimization_type, batch)) ReturnFalse; if(!cKVGradV.Init(0, index++, OpenCL, tokens * iDimension, optimization_type, batch)) ReturnFalse; if(!cAttention.Init(0, index++, OpenCL, iDimension, iScenarios, false, iQueries, iVariables * iHorizon, optimization_type, batch)) ReturnFalse; if(!cAttentionTranspose.Init(0, index++, OpenCL, iQueries, iScenarios, iDimension, optimization_type, batch)) ReturnFalse; if(!cHeadGate.Init(0, index++, OpenCL, iQueries * iScenarios * iDimension, optimization_type, batch)) ReturnFalse; if(!cHeadGateTranspose.Init(0, index++, OpenCL, iScenarios, iQueries, iDimension, optimization_type, batch)) ReturnFalse; if(!cOutputProjection.Init(0, index++, OpenCL, iScenarios * iDimension, iScenarios * iDimension, iDimension, iQueries, 1, optimization_type, batch)) ReturnFalse; if(!cQueryResidual.Init(0, index++, OpenCL, iScenarios * iDimension, iScenarios * iDimension, iDimension, iQueries, 1, optimization_type, batch)) ReturnFalse; if(!cMHAOutput.Init(0, index++, OpenCL, iQueries * iDimension, optimization_type, batch)) ReturnFalse; //--- CNeuronMultiWindowsConvOCL emits one window_out vector per state //--- window. The Task bank must therefore preserve the complete query //--- projection (Q * scenarios * D), not a single scenario slice. //--- The Actor keeps scenario-major contexts until the final Pi gate. TTM //--- sees K candidates of width Q*D, while the stack holds the previous final //--- Cross output of the same layer. if(!bCritic) { if(!cHistoryStack.Init(0, index++, OpenCL, iStackSize, mpi_dimension, 1, optimization_type, batch)) { PrintFormat("ScenarioCross Init: history_stack S=%u D=%u", iStackSize, mpi_dimension); ReturnFalse; } if(!cHistoryTTM.Init(0, index++, OpenCL, mpi_dimension, iScenarios, iStackSize, iTopK, optimization_type, batch)) { PrintFormat("ScenarioCross Init: history_ttm K=%u S=%u D=%u", iScenarios, iStackSize, mpi_dimension); ReturnFalse; } //--- Flash attention receives preprojected Key/Value pairs. Unlike the //--- generic cross-attention kernel it tiles an arbitrary token sequence //--- through a local group bounded by the device capability. if(!cHistoryTokenProjection.Init(0, index++, OpenCL, iDimension, iDimension, 2 * mpi_dimension, tokens, optimization_type, batch)) { PrintFormat("ScenarioCross Init: history_tokens T=%u D=%u", tokens, mpi_dimension); ReturnFalse; } if(!cHistoryMarket.Init(0, index++, OpenCL, mpi_key, mpi_heads, false, iStackSize, tokens, optimization_type, batch)) { PrintFormat("ScenarioCross Init: history_market Q=%u K=%u D=%u", iStackSize, tokens, mpi_dimension); ReturnFalse; } if(!cHistoryContext.Init(0, index++, OpenCL, mpi_dimension, mpi_key, mpi_heads, 1, mpi_dimension, iStackSize, optimization_type, batch)) { PrintFormat("ScenarioCross Init: history_context S=%u D=%u", iStackSize, mpi_dimension); ReturnFalse; } if(!cScenarioHistory.Init(0, index++, OpenCL, mpi_dimension, mpi_key, mpi_heads, iScenarios, mpi_dimension, 1, optimization_type, batch)) { PrintFormat("ScenarioCross Init: scenario_refine K=%u D=%u", iScenarios, mpi_dimension); ReturnFalse; } } cQueryGradient.SetActivationFunction(None); cWz.SetActivationFunction(None); cWu.SetActivationFunction(None); cTokenSum.SetActivationFunction(None); cKV.SetActivationFunction(None); cKVGradK.SetActivationFunction(None); cKVGradV.SetActivationFunction(None); cHeadGate.SetActivationFunction(None); cMHAOutput.SetActivationFunction(None); cOutputProjection.SetActivationFunction(None); cQueryResidual.SetActivationFunction(None); cHistoryTokenProjection.SetActivationFunction(None); return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronScenarioCrossAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::feedForwardSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!OpenCL || !NeuronOCL || !Forecast || Forecast.Type() != defNeuronScenarioForecast || NeuronOCL.Neurons() != int(bCritic ? 19 : 13)) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(forecast.Variables() != iVariables || forecast.Scenarios() != iScenarios || forecast.Horizon() != iHorizon || forecast.Dimension() != iDimension || Forecast.Neurons() != int(iScenarios * iVariables * iHorizon * iDimension) || !forecast.GetLogU() || forecast.GetLogU().Total() != int(iScenarios * iVariables * iHorizon) || !forecast.GetPi() || forecast.GetPi().Total() != int(iScenarios)) ReturnFalse; if(!cQueryProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cWz.FeedForward(Forecast)) ReturnFalse; if(!cWu.FeedForwardBuffer(forecast.GetLogU())) ReturnFalse; if(!SumAndNormalize(cWz.getOutput(), cWu.getOutput(), cTokenSum.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cTokenPE.FeedForward(cTokenSum.AsObject())) ReturnFalse; if(!cTokensTranspose.FeedForward(cTokenPE.AsObject())) ReturnFalse; if(!Concat(cTokensTranspose.getOutput(), cTokensTranspose.getOutput(), cKV.getOutput(), iScenarios * iDimension, iScenarios * iDimension, iVariables * iHorizon)) ReturnFalse; if(!cAttention.FeedForward(cQueryProjection.AsObject(), cKV.getOutput())) ReturnFalse; if(!cAttentionTranspose.FeedForward(cAttention.AsObject())) ReturnFalse; //--- Critic is stateless. Actor first derives a shared history context from //--- all ungated K scenario contexts, then refines each scenario separately. if(!bCritic) { if(!cHistoryStack.FeedForward(GetPointer(this))) ReturnFalse; if(!cHistoryTTM.FeedForward(cAttentionTranspose.AsObject(), cHistoryStack.getOutput())) ReturnFalse; if(!cHistoryTokenProjection.FeedForward(cTokenPE.AsObject())) ReturnFalse; if(!cHistoryMarket.FeedForward(cHistoryStack.AsObject(), cHistoryTokenProjection.getOutput())) ReturnFalse; if(!cHistoryContext.FeedForward(cHistoryTTM.AsObject(), cHistoryMarket.getOutput())) ReturnFalse; if(!cScenarioHistory.FeedForward(cAttentionTranspose.AsObject(), cHistoryContext.getOutput())) ReturnFalse; if(!ScalarToVector(forecast.GetPi(), cScenarioHistory.getOutput(), cHeadGate.getOutput(), iQueries * iDimension)) ReturnFalse; } else if(!ScalarToVector(forecast.GetPi(), cAttentionTranspose.getOutput(), cHeadGate.getOutput(), iQueries * iDimension)) ReturnFalse; if(!cHeadGateTranspose.FeedForward(cHeadGate.AsObject())) ReturnFalse; if(!cOutputProjection.FeedForward(cHeadGateTranspose.AsObject())) ReturnFalse; if(!cQueryResidual.FeedForward(cQueryProjection.AsObject())) ReturnFalse; if(!SumAndNormalize(cOutputProjection.getOutput(), cQueryResidual.getOutput(), cMHAOutput.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cMHAOutput.AsObject())) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronScenarioCrossAttention layer. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::calcInputGradientsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!OpenCL || !NeuronOCL || !Forecast || Forecast.Type() != defNeuronScenarioForecast) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(!forecast.GetPi()) ReturnFalse; if(!CNeuronMSRes::calcInputGradients(cMHAOutput.AsObject())) ReturnFalse; if(!DeActivation(cOutputProjection.getOutput(), cOutputProjection.getGradient(), cMHAOutput.getGradient(), cOutputProjection.Activation())) ReturnFalse; if(!DeActivation(cQueryResidual.getOutput(), cQueryResidual.getGradient(), cMHAOutput.getGradient(), cQueryResidual.Activation())) ReturnFalse; if(!cHeadGateTranspose.CalcHiddenGradients(cOutputProjection.AsObject())) ReturnFalse; if(!cHeadGate.CalcHiddenGradients(cHeadGateTranspose.AsObject())) ReturnFalse; if(!bCritic) { if(!ScalarToVector(forecast.GetPi(), cHeadGate.getGradient(), cScenarioHistory.getGradient(), iQueries * iDimension)) ReturnFalse; if(!cAttentionTranspose.CalcHiddenGradients(cScenarioHistory.AsObject(), cHistoryContext.getOutput(), cHistoryContext.getGradient(), None)) ReturnFalse; if(!cHistoryTTM.CalcHiddenGradients(cHistoryContext.AsObject(), cHistoryMarket.getOutput(), cHistoryMarket.getGradient(), None)) ReturnFalse; if(!cAttentionTranspose.CalcHiddenGradients(cHistoryTTM.AsObject(), cHistoryStack.getOutput(), NULL, None)) ReturnFalse; if(!cHistoryStack.CalcHiddenGradients(cHistoryMarket.AsObject(), cHistoryTokenProjection.getOutput(), cHistoryTokenProjection.getGradient(), None)) ReturnFalse; if(!cTokenPE.CalcHiddenGradients(cHistoryTokenProjection.AsObject())) ReturnFalse; } else if(!ScalarToVector(forecast.GetPi(), cHeadGate.getGradient(), cAttentionTranspose.getGradient(), iQueries * iDimension)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cAttentionTranspose.AsObject())) ReturnFalse; if(!cQueryGradient.CalcHiddenGradients(cQueryResidual.AsObject())) ReturnFalse; if(!cQueryProjection.CalcHiddenGradients(cAttention.AsObject(), cKV.getOutput(), cKV.getGradient(), None)) ReturnFalse; if(!SumAndNormalize(cQueryProjection.getGradient(), cQueryGradient.getGradient(), cQueryProjection.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!DeConcat(cKVGradK.getGradient(), cKVGradV.getGradient(), cKV.getGradient(), iScenarios * iDimension, iScenarios * iDimension, iVariables * iHorizon)) ReturnFalse; if(!SumAndNormalize(cKVGradK.getGradient(), cKVGradV.getGradient(), cTokensTranspose.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cTokenPE.CalcHiddenGradients(cTokensTranspose.AsObject())) ReturnFalse; if(!cTokenSum.CalcHiddenGradients(cTokenPE.AsObject())) ReturnFalse; //--- Forecast is the read-only second stream: only Cross trainables receive gradients. if(!Concat(cTokenSum.getGradient(), cTokenSum.getGradient(), cWz.getGradient(), iScenarios * iVariables * iHorizon * iDimension, 0, 1)) ReturnFalse; if(!Concat(cTokenSum.getGradient(), cTokenSum.getGradient(), cWu.getGradient(), iScenarios * iVariables * iHorizon * iDimension, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQueryProjection.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) return(DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())); return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronScenarioCrossAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::updateInputWeightsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!NeuronOCL || !Forecast || Forecast.Type() != defNeuronScenarioForecast) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(!forecast.GetLogU()) ReturnFalse; if(!cQueryProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cWz.UpdateInputWeights(Forecast)) ReturnFalse; if(!cWu.UpdateInputWeightsBuffer(forecast.GetLogU())) ReturnFalse; if(!cTokenPE.UpdateInputWeights(cTokenSum.AsObject())) ReturnFalse; if(!cOutputProjection.UpdateInputWeights(cHeadGateTranspose.AsObject())) ReturnFalse; if(!cQueryResidual.UpdateInputWeights(cQueryProjection.AsObject())) ReturnFalse; if(!bCritic) { if(!cHistoryTokenProjection.UpdateInputWeights(cTokenPE.AsObject())) ReturnFalse; if(!cHistoryMarket.UpdateInputWeights(cHistoryStack.AsObject(), cHistoryTokenProjection.getOutput())) ReturnFalse; if(!cHistoryContext.UpdateInputWeights(cHistoryTTM.AsObject(), cHistoryMarket.getOutput())) ReturnFalse; if(!cScenarioHistory.UpdateInputWeights(cAttentionTranspose.AsObject(), cHistoryContext.getOutput())) ReturnFalse; } return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronScenarioCrossAttention state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iVariables); dFileWriteUInt(file_handle, iScenarios); dFileWriteUInt(file_handle, iHorizon); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, iQueries); dFileWriteUInt(file_handle, iStackSize); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, uint(bCritic)); if(!cWz.Save(file_handle)) ReturnFalse; if(!cWu.Save(file_handle)) ReturnFalse; if(!cTokenPE.Save(file_handle)) ReturnFalse; if(!cQueryProjection.Save(file_handle)) ReturnFalse; if(!cOutputProjection.Save(file_handle)) ReturnFalse; if(!cQueryResidual.Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(!bCritic) { if(!cHistoryTokenProjection.Save(file_handle)) ReturnFalse; if(!cHistoryMarket.Save(file_handle)) ReturnFalse; if(!cHistoryContext.Save(file_handle)) ReturnFalse; if(!cScenarioHistory.Save(file_handle)) ReturnFalse; } return(true); } //+------------------------------------------------------------------+ //| Append scenario cross-attention parameters and nested flow only. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronMSRes::AppendParameterFingerprint(fingerprint) || !cWz.AppendParameterFingerprint(fingerprint) || !cWu.AppendParameterFingerprint(fingerprint) || !cTokenPE.AppendParameterFingerprint(fingerprint) || !cQueryProjection.AppendParameterFingerprint(fingerprint) || !cOutputProjection.AppendParameterFingerprint(fingerprint) || !cQueryResidual.AppendParameterFingerprint(fingerprint) || !cAttention.AppendParameterFingerprint(fingerprint)) ReturnFalse; if(!bCritic && (!cHistoryTokenProjection.AppendParameterFingerprint(fingerprint) || !cHistoryMarket.AppendParameterFingerprint(fingerprint) || !cHistoryContext.AppendParameterFingerprint(fingerprint) || !cScenarioHistory.AppendParameterFingerprint(fingerprint))) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronScenarioCrossAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iVariables); dFileReadUInt(file_handle, iScenarios); dFileReadUInt(file_handle, iHorizon); dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, iQueries); dFileReadUInt(file_handle, iStackSize); dFileReadUInt(file_handle, iTopK); uint critic = 0; dFileReadUInt(file_handle, critic); bCritic = (critic != 0); if(iVariables == 0 || iScenarios == 0 || iHorizon == 0 || iDimension == 0 || iStackSize == 0 || iTopK == 0 || iTopK > iScenarios || iQueries != uint(bCritic ? 5 : 3) || Output.Total() != int(iQueries * iDimension)) ReturnFalse; if(!InitHolders(0, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cWz.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWu.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTokenPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cOutputProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryResidual.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; if(!bCritic) { if(!LoadInsideLayer(file_handle, cHistoryTokenProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistoryMarket.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistoryContext.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarioHistory.AsObject())) ReturnFalse; } SetOpenCL(OpenCL); return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronScenarioCrossAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::Load(const int file_handle, COpenCLMy *open_cl) { if(file_handle == INVALID_HANDLE || FileReadInteger(file_handle) != Type()) ReturnFalse; if(!open_cl) ReturnFalse; //--- Load() reconstructs bCritic and all nested holders. Rebinding before //--- that point would touch default-constructed actor-only holders. OpenCL = open_cl; if(!Load(file_handle)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Checks ScenarioCrossAttention checked-transfer capability. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::SupportsOpenCLChecked(void) const { //--- Preserve the exact public Scenario identity gate. return(Type() == defNeuronScenarioCrossAttention && ValidateScenarioOwnedTreeChecked()); } //+------------------------------------------------------------------+ //| Validates the Scenario-owned checked-transfer tree. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::ValidateScenarioOwnedTreeChecked(void) const { //--- Validate the owner and scalar geometry independently of identity. if(CheckPointer(OpenCL) == POINTER_INVALID || iVariables == 0 || iScenarios == 0 || iHorizon == 0 || iDimension == 0 || iQueries != (bCritic ? 5 : 3) || iStackSize == 0 || iTopK == 0 || iTopK > iScenarios) return(false); //--- Bound every tensor product before it reaches an int buffer index space. const long max_elements = INT_MAX; if((long)iScenarios > max_elements / (long)iVariables) return(false); const long scenario_variables = (long)iScenarios * (long)iVariables; if(scenario_variables > max_elements / (long)iHorizon) return(false); const long tokens = scenario_variables * (long)iHorizon; if(tokens > max_elements / (long)iDimension) return(false); const long token_dimension = tokens * (long)iDimension; if(token_dimension > max_elements / 2 || (long)iQueries > max_elements / (long)iDimension) return(false); const long query_dimension = (long)iQueries * (long)iDimension; if((long)iScenarios > max_elements / query_dimension) return(false); const long query_scenario_dimension = (long)iScenarios * query_dimension; //--- Match the projected query family to the declared Scenario geometry. if(Neurons() != (int)query_dimension || cQueryProjection.Neurons() != (int)query_scenario_dimension || cQueryGradient.Neurons() != (int)query_scenario_dimension || cMHAOutput.Neurons() != (int)query_dimension) return(false); //--- The Actor-only continuity path owns stack and projected token products. if(!bCritic && ((long)iStackSize > max_elements / query_dimension || tokens > max_elements / 2 / query_dimension)) return(false); //--- Require the complete common ownership tree without demanding one owner. if(!cFlow.SupportsOpenCLChecked() || !cQueryProjection.SupportsOpenCLChecked() || !cQueryGradient.SupportsOpenCLChecked() || !cWz.SupportsOpenCLChecked() || !cWu.SupportsOpenCLChecked() || !cTokenSum.SupportsOpenCLChecked() || !cTokenPE.SupportsOpenCLChecked() || !cTokensTranspose.SupportsOpenCLChecked() || !cKV.SupportsOpenCLChecked() || !cKVGradK.SupportsOpenCLChecked() || !cKVGradV.SupportsOpenCLChecked() || !cAttention.SupportsOpenCLChecked() || !cAttentionTranspose.SupportsOpenCLChecked() || !cHeadGate.SupportsOpenCLChecked() || !cHeadGateTranspose.SupportsOpenCLChecked() || !cOutputProjection.SupportsOpenCLChecked() || !cQueryResidual.SupportsOpenCLChecked() || !cMHAOutput.SupportsOpenCLChecked()) return(false); //--- Critic leaves the Actor-only history holders uninitialized by contract. if(!bCritic && (!cHistoryStack.SupportsOpenCLChecked() || !cHistoryTTM.SupportsOpenCLChecked() || !cHistoryTokenProjection.SupportsOpenCLChecked() || !cHistoryMarket.SupportsOpenCLChecked() || !cHistoryContext.SupportsOpenCLChecked() || !cScenarioHistory.SupportsOpenCLChecked())) return(false); //--- Finalize the complete capability check without mutating any owner. return(true); } //+------------------------------------------------------------------+ //| Transfers ScenarioCrossAttention owners before MSRes publishing. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::SetOpenCLChecked(COpenCLMy *obj) { //--- Reject an invalid target or incomplete tree before any child moves. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or Scenario tree"); //--- Move the common holders in legacy ownership order with local diagnostics. if(!cQueryProjection.SetOpenCLChecked(obj)) ReturnFalseEx("query projection context"); if(!cQueryGradient.SetOpenCLChecked(obj)) ReturnFalseEx("query gradient context"); if(!cWz.SetOpenCLChecked(obj)) ReturnFalseEx("Wz context"); if(!cWu.SetOpenCLChecked(obj)) ReturnFalseEx("Wu context"); if(!cTokenSum.SetOpenCLChecked(obj)) ReturnFalseEx("token sum context"); if(!cTokenPE.SetOpenCLChecked(obj)) ReturnFalseEx("token positional encoding context"); if(!cTokensTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("tokens transpose context"); if(!cKV.SetOpenCLChecked(obj)) ReturnFalseEx("key-value context"); if(!cKVGradK.SetOpenCLChecked(obj)) ReturnFalseEx("key gradient context"); if(!cKVGradV.SetOpenCLChecked(obj)) ReturnFalseEx("value gradient context"); if(!cAttention.SetOpenCLChecked(obj)) ReturnFalseEx("flash attention context"); if(!cAttentionTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("attention transpose context"); if(!cHeadGate.SetOpenCLChecked(obj)) ReturnFalseEx("head gate context"); if(!cHeadGateTranspose.SetOpenCLChecked(obj)) ReturnFalseEx("head gate transpose context"); if(!cOutputProjection.SetOpenCLChecked(obj)) ReturnFalseEx("output projection context"); if(!cQueryResidual.SetOpenCLChecked(obj)) ReturnFalseEx("query residual context"); if(!cMHAOutput.SetOpenCLChecked(obj)) ReturnFalseEx("MHA output context"); //--- Transfer Actor continuity only when those holders were initialized. if(!bCritic) { if(!cHistoryStack.SetOpenCLChecked(obj)) ReturnFalseEx("history stack context"); if(!cHistoryTTM.SetOpenCLChecked(obj)) ReturnFalseEx("history TTM context"); if(!cHistoryTokenProjection.SetOpenCLChecked(obj)) ReturnFalseEx("history token projection context"); if(!cHistoryMarket.SetOpenCLChecked(obj)) ReturnFalseEx("history market context"); if(!cHistoryContext.SetOpenCLChecked(obj)) ReturnFalseEx("history context context"); if(!cScenarioHistory.SetOpenCLChecked(obj)) ReturnFalseEx("scenario history context"); } //--- MSRes transfers cFlow and publishes inherited Base owners last. if(!CNeuronMSRes::SetOpenCLChecked(obj)) ReturnFalseEx("inherited MSRes context"); //--- Finalize only after every Scenario and inherited owner succeeded. return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronScenarioCrossAttention GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronScenarioCrossAttention::SetOpenCL(COpenCLMy *obj) { CNeuronMSRes::SetOpenCL(obj); cQueryProjection.SetOpenCL(OpenCL); cQueryGradient.SetOpenCL(OpenCL); cWz.SetOpenCL(OpenCL); cWu.SetOpenCL(OpenCL); cTokenPE.SetOpenCL(OpenCL); cTokenSum.SetOpenCL(OpenCL); cTokensTranspose.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cKVGradK.SetOpenCL(OpenCL); cKVGradV.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cAttentionTranspose.SetOpenCL(OpenCL); cHeadGate.SetOpenCL(OpenCL); cHeadGateTranspose.SetOpenCL(OpenCL); cOutputProjection.SetOpenCL(OpenCL); cQueryResidual.SetOpenCL(OpenCL); cMHAOutput.SetOpenCL(OpenCL); if(!bCritic) { cHistoryStack.SetOpenCL(OpenCL); cHistoryTTM.SetOpenCL(OpenCL); cHistoryTokenProjection.SetOpenCL(OpenCL); cHistoryMarket.SetOpenCL(OpenCL); cHistoryContext.SetOpenCL(OpenCL); cScenarioHistory.SetOpenCL(OpenCL); } } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronScenarioCrossAttention | //| layer. | //+------------------------------------------------------------------+ void CNeuronScenarioCrossAttention::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); cQueryProjection.TrainMode(flag); cWz.TrainMode(flag); cWu.TrainMode(flag); cTokenPE.TrainMode(flag); cOutputProjection.TrainMode(flag); cQueryResidual.TrainMode(flag); if(!bCritic) { cHistoryStack.TrainMode(flag); cHistoryTTM.TrainMode(flag); cHistoryTokenProjection.TrainMode(flag); cHistoryMarket.TrainMode(flag); cHistoryContext.TrainMode(flag); cScenarioHistory.TrainMode(flag); } } //+------------------------------------------------------------------+ //| Averages the CNeuronScenarioCrossAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronScenarioCrossAttention *Source = source; if(Source.iVariables != iVariables || Source.iScenarios != iScenarios || Source.iHorizon != iHorizon || Source.iDimension != iDimension || Source.bCritic != bCritic) ReturnFalse; if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; if(!cWz.WeightsUpdate(Source.cWz.AsObject(), tau)) ReturnFalse; if(!cWu.WeightsUpdate(Source.cWu.AsObject(), tau)) ReturnFalse; if(!cTokenPE.WeightsUpdate(Source.cTokenPE.AsObject(), tau)) ReturnFalse; if(!cOutputProjection.WeightsUpdate(Source.cOutputProjection.AsObject(), tau)) ReturnFalse; if(!cQueryResidual.WeightsUpdate(Source.cQueryResidual.AsObject(), tau)) ReturnFalse; if(!bCritic) { if(!cHistoryTokenProjection.WeightsUpdate(Source.cHistoryTokenProjection.AsObject(), tau)) ReturnFalse; if(!cHistoryMarket.WeightsUpdate(Source.cHistoryMarket.AsObject(), tau)) ReturnFalse; if(!cHistoryContext.WeightsUpdate(Source.cHistoryContext.AsObject(), tau)) ReturnFalse; if(!cScenarioHistory.WeightsUpdate(Source.cScenarioHistory.AsObject(), tau)) ReturnFalse; } if(!cQueryProjection.WeightsUpdate(Source.cQueryProjection.AsObject(), tau)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronScenarioCrossAttention execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::Clear(void) { if(!CNeuronMSRes::Clear()) ReturnFalse; if(!cWz.Clear()) ReturnFalse; if(!cWu.Clear()) ReturnFalse; if(!cTokenPE.Clear()) ReturnFalse; if(!cQueryProjection.Clear()) ReturnFalse; if(!cOutputProjection.Clear()) ReturnFalse; if(!cQueryResidual.Clear()) ReturnFalse; if(!bCritic) { if(!cHistoryStack.Clear()) ReturnFalse; if(!cHistoryTTM.Clear()) ReturnFalse; if(!cHistoryTokenProjection.Clear()) ReturnFalse; if(!cHistoryMarket.Clear()) ReturnFalse; if(!cHistoryContext.Clear()) ReturnFalse; if(!cScenarioHistory.Clear()) ReturnFalse; } return(true); } //+------------------------------------------------------------------+ //| Checks the exact Market latent object and source geometry | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::PriceSource(CNeuronBaseOCL *source) const { if(!source || source.Type() != defNeuronScenarioForecastPriceOCL) return(false); CNeuronScenarioForecastPriceOCL *forecast = source; return(CheckPointer(forecast) != POINTER_INVALID && forecast.ABI() == defScenarioForecastPriceABI && forecast.Scenarios() == m_shape.K && forecast.Variables() == m_shape.V && forecast.Horizon() == m_shape.H && forecast.Dimension() == m_shape.D && forecast.PricePhase() == m_price_phase); } //+------------------------------------------------------------------+ //| Sets the new price phase only outside a published generation | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::SetPricePhase(const uint phase) { if(!m_price_mode || m_failed || !m_configured || m_phase != 0 || m_public_valid || phase < defForecastPriceLatentPrepare || phase > defForecastPriceVerify) return(Fail(DPCC_CONTRACT_MISMATCH)); m_price_phase = phase; m_price_error = ""; return(true); } //+------------------------------------------------------------------+ //| Copies the causal observation without changing a live decision | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BindPriceAnchor(const double anchor, const double scale, const ulong decision_id) { if(!m_price_mode || m_failed || !m_configured || m_phase != 0 || m_public_valid) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!MathIsValidNumber(scale) || scale <= 0) { m_price_anchor_bound = false; m_price_error = "INVALID_PRICE_SCALE"; return(Fail(DPCC_INVALID_INPUT)); } if(!MathIsValidNumber(anchor)) { m_price_anchor_bound = false; m_price_error = "INVALID_PRICE_ANCHOR"; return(Fail(DPCC_INVALID_INPUT)); } if(m_price_anchor_bound && (m_price_decision != decision_id || m_price_anchor != anchor || m_price_scale != scale)) return(Fail(DPCC_CONTRACT_MISMATCH)); m_price_anchor = anchor; m_price_scale = scale; m_price_decision = decision_id; m_price_anchor_bound = true; m_price_error = ""; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Returns observation provenance only for the published path | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::GetPriceAnchor(double &anchor, double &scale, ulong &decision_id) const { if(!m_price_mode || m_failed || !m_public_valid || !m_price_anchor_bound) return(false); anchor = m_price_anchor; scale = m_price_scale; decision_id = m_price_decision; return(true); } //+------------------------------------------------------------------+ //| Reconstructs one causal OHLC path per k, preserving the raw tape | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::DecodePricePath(void) { const uint kernel = def_k_DPCCDecoderPricePath; setBuffer(kernel, 0, m_last.getOutputIndex()); setBuffer(kernel, 1, Output.GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_shape.K); setArgument(kernel, 4, m_shape.H); uint offset[1] = {0}; uint size[1] = {m_shape.K}; kernelExecute(kernel, offset, size) return(Reduce(m_shape.K)); } //+------------------------------------------------------------------+ //| Reverses the causal normalized path into its saved raw | //| parameters | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ReversePricePath(void) { const uint kernel = def_k_DPCCDecoderPriceGradient; setBuffer(kernel, 0, m_last.getOutputIndex()); setBuffer(kernel, 1, Gradient.GetIndex()); setBuffer(kernel, 2, m_last.getGradientIndex()); setBuffer(kernel, 3, m_work[4].GetIndex()); setArgument(kernel, 4, m_shape.K); setArgument(kernel, 5, m_shape.H); uint offset[1] = {0}; uint size[1] = {m_shape.K}; kernelExecute(kernel, offset, size) return(Reduce(m_shape.K)); } //+------------------------------------------------------------------+ //| Prepares detached Huber loss from the current Joint latent | //| target | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::PreparePriceLoss(CNeuronScenarioForecastPriceOCL *provider, CBufferFloat *target_normalized, const float lambda_price, const float huber_delta, const float rho_sum_tolerance, const bool prepare_gradient, double &loss) { loss = 0.0; //--- Bind loss to this price forward and its already prepared latent responsibility. if(!m_price_mode || m_failed || !m_configured || !m_public_valid || !m_price_anchor_bound || (m_price_phase != defForecastPriceJoint && m_price_phase != defForecastPriceAdapt && m_price_phase != defForecastPriceVerify) || !ValidateOwned() || (m_price_phase == defForecastPriceVerify && prepare_gradient) || !provider || !SameSource(provider) || !PriceSource(provider) || provider.PricePhase() != m_price_phase || !provider.PriceSupervisionPrepared() || (prepare_gradient ? (!bTrain || m_phase != 1) : (bTrain || m_phase != 0))) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!MathIsValidNumber(lambda_price) || lambda_price <= 0 || !MathIsValidNumber(huber_delta) || huber_delta <= 0 || !MathIsValidNumber(rho_sum_tolerance) || rho_sum_tolerance < 0 || rho_sum_tolerance >= 1) return(Fail(DPCC_INVALID_INPUT)); //--- Targets remain external; only the provider's detached rho is read by the loss kernel. if(!ValidForecastLossTarget(provider, target_normalized, m_shape.H * 4) || OpenCL.GetBufferSize(target_normalized.GetIndex()) != (ulong)m_shape.H * 4 * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *rho = provider.GetResponsibilities(); if(!ValidBuffer(rho, m_shape.K) || OpenCL.GetBufferSize(rho.GetIndex()) != (ulong)m_shape.K * sizeof(float) || OpenCL.GetBufferSize(Output.GetIndex()) != (ulong)m_tokens * 4 * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(provider)) return(false); //--- Publish only the normalized price gradient after full rho and numeric admission. if(!DispatchRhoLoss(target_normalized, rho, rho_sum_tolerance, prepare_gradient, loss, lambda_price, huber_delta)) return(false); m_price_error = ""; return(true); } //+------------------------------------------------------------------+ //| Builds an explicit physical prefix with checked fp32 precision | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BuildNativeScenarios(CBufferFloat *destination, const uint plan_horizon, const double tick_size) { const ulong total = (ulong)m_shape.K * plan_horizon * 4; if(!m_price_mode || m_failed || !m_public_valid || !m_price_anchor_bound || !ValidateOwned() || plan_horizon == 0 || plan_horizon > m_shape.H || total > INT_MAX || !ValidBuffer(destination, (uint)total) || OwnsBufferOrDeviceSlot(destination) || !MathIsValidNumber(tick_size) || tick_size <= 0) return(Fail(DPCC_CONTRACT_MISMATCH)); if(CheckPointer(m_source) == POINTER_INVALID || m_source.OwnsBufferOrDeviceSlot(destination)) return(Fail(DPCC_CONTRACT_MISMATCH)); const float anchor = (float)m_price_anchor; const float scale = (float)m_price_scale; const float half_tick = (float)(0.5 * tick_size); const float anchor_error = (float)MathAbs(m_price_anchor - (double)anchor); const float scale_error = (float)MathAbs(m_price_scale - (double)scale); if(!MathIsValidNumber(anchor) || !MathIsValidNumber(scale) || scale <= 0 || !MathIsValidNumber(half_tick) || half_tick <= 0 || !MathIsValidNumber(anchor_error) || !MathIsValidNumber(scale_error)) { m_price_error = "INVALID_PRICE_PRECISION"; return(Fail(DPCC_NUMERIC_ERROR)); } const uint kernel = def_k_DPCCDecoderPriceNative; setBuffer(kernel, 0, Output.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_shape.K); setArgument(kernel, 4, m_shape.H); setArgument(kernel, 5, plan_horizon); setArgument(kernel, 6, anchor); setArgument(kernel, 7, scale); setArgument(kernel, 8, half_tick); setArgument(kernel, 9, anchor_error); setArgument(kernel, 10, scale_error); uint offset[1] = {0}; uint size[1] = {m_shape.K * plan_horizon}; kernelExecute(kernel, offset, size) if(!Reduce(size[0])) { m_price_error = "INVALID_PRICE_PRECISION"; return(false); } m_price_error = ""; return(true); } //+------------------------------------------------------------------+ //| Saves the ABI3 price geometry and native shared affine state | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::SavePrice(const int file_handle) { if(file_handle == INVALID_HANDLE || m_failed || !m_configured || m_phase != 0 || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint header[] = {(uint)Type(), defScenarioDecoderPriceABI, m_shape.K, m_shape.V, m_shape.H, m_shape.D, m_hidden, m_update_count }; if(FileWriteArray(file_handle, header, 0, ArraySize(header)) != ArraySize(header)) return(Fail(DPCC_INVALID_INPUT)); if(!CNeuronBaseOCL::Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); if(!m_first.Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); if(!m_last.Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); return(true); } //+------------------------------------------------------------------+ //| Restores ABI3 parameters while revoking every volatile decision | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::LoadPrice(const int file_handle) { uint header[7]; if(FileReadArray(file_handle, header, 0, 7) != 7 || header[0] != defScenarioDecoderPriceABI || header[6] >= UINT_MAX - 1) return(Fail(DPCC_CONTRACT_MISMATCH, true)); CLayerDescription desc; desc.type = Type(); desc.step = 1; desc.variables = 1; desc.layers = 2; desc.batch = 1; desc.activation = None; desc.optimization = ADAM; const ulong count = (ulong)header[1] * header[3] * 4; const ulong width = (ulong)header[2] * header[4]; if(count > INT_MAX || width > INT_MAX || ArrayResize(desc.units, 6) != 6) return(Fail(DPCC_CONTRACT_MISMATCH, true)); desc.count = (uint)count; desc.window = (uint)width; desc.window_out = header[5]; for(uint i = 0; i < 5; i++) desc.units[i] = header[i + 1]; desc.units[5] = defScenarioDecoderPriceABI; SDPCCShape candidate; if(!DescribeAdmission(GetPointer(desc), candidate) || (m_shape_admitted && (!m_price_mode || !DPCCSameShape(candidate, m_admitted_shape)))) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!InitOwned(0, 0, OpenCL, GetPointer(desc), false)) return(false); m_failed = true; if(FileReadInteger(file_handle, INT_VALUE) != Type() || !CNeuronBaseOCL::Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !m_first.Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !m_last.Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_update_count = header[6]; if(activation != None || optimization != ADAM || iBatch != 1 || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!CheckParameters(m_first)) return(Fail(DPCC_NUMERIC_ERROR, true)); if(!CheckParameters(m_last)) return(Fail(DPCC_NUMERIC_ERROR, true)); m_failed = false; if(!Clear()) return(false); m_admitted_shape = m_shape; m_shape_admitted = true; m_config_admitted = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Initializes shared decoder ownership | //+------------------------------------------------------------------+ CNeuronScenarioDecoderOCL::CNeuronScenarioDecoderOCL(void) : m_price_mode(false), m_price_phase(defForecastPriceVerify), m_price_anchor_bound(false), m_price_anchor(0), m_price_scale(0), m_price_decision(0), m_price_error(""), m_shape_admitted(false), m_config_admitted(false), m_first(NULL), m_last(NULL), m_source(NULL), m_source_output(NULL), m_source_index(INVALID_HANDLE), m_tokens(0), m_width(0), m_hidden(0), m_phase(0), m_update_count(0), m_generation(0), m_configured(false), m_public_valid(false), m_failed(false), m_status(DPCC_CONTRACT_MISMATCH) { ZeroMemory(m_shape); ZeroMemory(m_admitted_shape); ZeroMemory(m_admitted_config); ZeroMemory(m_admitted_composition); for(uint i = 0; i < 8; i++) m_work[i] = NULL; for(uint i = 0; i < 4; i++) m_loss_work[i] = NULL; } //+------------------------------------------------------------------+ //| Releases exclusively owned decoder components | //+------------------------------------------------------------------+ CNeuronScenarioDecoderOCL::~CNeuronScenarioDecoderOCL(void) { DeleteObj(m_first); DeleteObj(m_last); for(uint i = 0; i < 8; i++) DeleteObj(m_work[i]); for(uint i = 0; i < 4; i++) DeleteObj(m_loss_work[i]); } //+------------------------------------------------------------------+ //| Records decoder failure and invalidates publication | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Fail(const ENUM_DPCC_STATUS status, const bool fatal) { m_status = status; m_public_valid = false; if(fatal) m_failed = true; return(false); } //+------------------------------------------------------------------+ //| Checks a decoder buffer's exact device owner and length | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ValidBuffer(CBufferFloat *buffer, const uint total) const { return(CheckPointer(buffer) != POINTER_INVALID && buffer.Total() == (int)total && buffer.GetIndex() >= 0 && buffer.GetOpenCL() == OpenCL); } //+------------------------------------------------------------------+ //| Checks owned graph geometry and native optimizer steps | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ValidateOwned(void) const { if(!m_first || !m_last || !OpenCL || m_tokens == 0 || m_width == 0 || m_hidden == 0) return(false); for(uint i = 0; i < 8; i++) if(!m_work[i]) return(false); for(uint i = 0; i < 4; i++) if(!m_loss_work[i]) return(false); if(m_first.GetWindow() != m_width || m_first.GetInputStride() != m_width || m_first.GetFilters() != m_hidden || m_first.GetVariables() != 1 || m_first.GetUnits() != m_tokens || m_last.GetWindow() != m_hidden || m_last.GetInputStride() != m_hidden || m_last.GetFilters() != 4 || m_last.GetVariables() != 1 || m_last.GetUnits() != m_tokens || m_first.Activation() != None || m_last.Activation() != None || m_first.Optimization() != ADAM || m_last.Optimization() != ADAM || m_first.NativeOptimizerStep() != m_update_count + 1 || m_last.NativeOptimizerStep() != m_update_count + 1) return(false); if(!ValidBuffer(Output, m_tokens * 4) || !ValidBuffer(Gradient, m_tokens * 4) || !ValidBuffer(PrevOutput, m_tokens * 4)) return(false); if(!ValidBuffer(m_first.getOutput(), m_tokens * m_hidden) || !ValidBuffer(m_first.getGradient(), m_tokens * m_hidden) || !ValidBuffer(m_last.getOutput(), m_tokens * 4) || !ValidBuffer(m_last.getGradient(), m_tokens * 4)) return(false); if(!ValidBuffer(m_first.GetWeightsConv(), m_hidden * (m_width + 1)) || !ValidBuffer(m_first.GetFirstMomentumConv(), m_hidden * (m_width + 1)) || !ValidBuffer(m_first.GetSecondMomentumConv(), m_hidden * (m_width + 1)) || !ValidBuffer(m_last.GetWeightsConv(), 4 * (m_hidden + 1)) || !ValidBuffer(m_last.GetFirstMomentumConv(), 4 * (m_hidden + 1)) || !ValidBuffer(m_last.GetSecondMomentumConv(), 4 * (m_hidden + 1))) return(false); const uint sizes[] = {m_tokens * m_width, m_tokens * m_hidden, m_tokens * m_hidden, m_tokens * m_width, (uint)m_work[4].Total(), 1, (uint)m_work[4].Total(), 8 }; for(uint i = 0; i < 8; i++) if(!ValidBuffer(m_work[i], sizes[i])) return(false); const uint loss_sizes[] = {m_tokens * 4, m_shape.K, m_shape.K, 3}; for(uint i = 0; i < 4; i++) if(!ValidBuffer(m_loss_work[i], loss_sizes[i])) return(false); return(true); } //+------------------------------------------------------------------+ //| Allocates shared affines and stable saved tensor buffers | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Allocate(const uint hidden) { const ulong sizes64[] = {(ulong)m_tokens * m_width, (ulong)m_tokens * hidden, (ulong)m_tokens * 4, (ulong)hidden * (m_width + 1), (ulong)4 * (hidden + 1) }; uint capacity = 1; for(uint i = 0; i < 5; i++) { if(sizes64[i] == 0 || sizes64[i] > INT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH, true)); capacity = MathMax(capacity, (uint)sizes64[i]); } DeleteObj(m_first); DeleteObj(m_last); m_first = new CNeuronConvOCL; m_last = new CNeuronConvOCL; if(!m_first || !m_last) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_first.Init(0, 0, OpenCL, m_width, m_width, hidden, m_tokens, 1, ADAM, iBatch)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_last.Init(0, 1, OpenCL, hidden, hidden, 4, m_tokens, 1, ADAM, iBatch)) return(Fail(DPCC_OPENCL_ERROR, true)); m_first.SetActivationFunction(None); m_last.SetActivationFunction(None); m_hidden = hidden; const uint sizes[] = {m_tokens * m_width, m_tokens * hidden, m_tokens * hidden, m_tokens * m_width, capacity, 1, capacity, 8 }; for(uint i = 0; i < 8; i++) { DeleteObj(m_work[i]); m_work[i] = new CBufferFloat; if(!m_work[i]) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_work[i].BufferInit(sizes[i], 0)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_work[i].BufferCreate(OpenCL)) return(Fail(DPCC_OPENCL_ERROR, true)); } const uint loss_sizes[] = {m_tokens * 4, m_shape.K, m_shape.K, 3}; for(uint i = 0; i < 4; i++) { DeleteObj(m_loss_work[i]); m_loss_work[i] = new CBufferFloat; if(!m_loss_work[i]) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_loss_work[i].BufferInit(loss_sizes[i], 0)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_loss_work[i].BufferCreate(OpenCL)) return(Fail(DPCC_OPENCL_ERROR, true)); } return(ValidateOwned()); } //+------------------------------------------------------------------+ //| Validates descriptor into a local shape without owned mutations | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::DescribeAdmission(CLayerDescription *desc, SDPCCShape &shape) const { ZeroMemory(shape); if(desc && desc.type == Type() && ArraySize(desc.units) == 6 && desc.units[5] == defScenarioDecoderPriceABI) { if(ArraySize(desc.windows) != 0 || ArraySize(desc.heads) != 0 || desc.layers != 2 || desc.step != 1 || desc.variables != 1 || desc.batch != 1 || desc.activation != None || desc.optimization != ADAM) return(false); shape.K = desc.units[0]; shape.V = desc.units[1]; shape.H = desc.units[2]; shape.D = desc.units[3]; shape.Embedding = desc.units[4]; const ulong tokens = (ulong)shape.K * shape.H; const ulong width = (ulong)shape.V * shape.D; if(shape.K == 0 || shape.K > 64 || shape.V == 0 || shape.H == 0 || shape.D == 0 || shape.Embedding == 0 || tokens > INT_MAX || width > INT_MAX || tokens * width > INT_MAX || tokens * 4 > INT_MAX || tokens * shape.Embedding > INT_MAX || (ulong)shape.Embedding * (width + 1) > INT_MAX || (ulong)4 * (shape.Embedding + 1) > INT_MAX) return(false); return(desc.count == tokens * 4 && desc.window == width && desc.window_out == shape.Embedding); } if(!desc || desc.type != Type() || ArraySize(desc.windows) != 9 || ArraySize(desc.units) != 5 || ArraySize(desc.heads) != 1 || desc.windows[0] != 2 || desc.window != 6 || desc.window_out != 13 || desc.layers != 0 || desc.activation != None || desc.optimization != ADAM || desc.batch == 0) return(false); shape.K = desc.windows[1]; shape.V = desc.windows[2]; shape.H = desc.windows[3]; shape.D = desc.windows[4]; shape.M = desc.windows[5]; shape.J = desc.windows[6]; shape.E = desc.windows[7]; shape.PlanBatch = desc.windows[8]; shape.Embedding = desc.units[0]; shape.Heads = desc.units[1]; shape.KeyDimension = desc.units[2]; shape.NActions = desc.units[3]; shape.StateSize = desc.units[4]; shape.Depth = desc.layers; SDPCCLayout layout; return(DPCCBuildRolloutLayout(shape, layout) && desc.variables == shape.V && desc.step == shape.H && desc.count == (ulong)shape.K * shape.H * 4 && desc.heads[0] == shape.Heads && (ulong)shape.K * shape.H * 4 <= INT_MAX && (ulong)shape.V * shape.D <= INT_MAX); } //+------------------------------------------------------------------+ //| Validates the exact decoder checkpoint configuration/profile | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ValidComposition(const SDPCCConfig &config, const SDPCCDecoderComposition &composition) const { if(config.contract_version != 2 || config.binary_layer_version != 3 || config.normalization_mode != DPCC_NORMALIZATION_CAUSAL_AFFINE_V2 || config.loss_coordinates != DPCC_LOSS_NORMALIZED || composition.embedding == 0 || composition.architecture != "SHARED_VD_E4" || composition.activation_profile != "GELU" || composition.backward_profile != "DPCC_EXACT_CHAIN_V1" || composition.optimizer_profile != "NN_NATIVE_CONV_ADAM_C6A8_V1" || StringLen(composition.graph_digest) != 64) return(false); for(uint i = 0; i < 64; i++) { const ushort c = StringGetCharacter(composition.graph_digest, i); if(!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return(false); } return(true); } //+------------------------------------------------------------------+ //| Preserves independent admission across operational failure | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::MatchesAdmission(const SDPCCConfig &config, const SDPCCDecoderComposition &composition) const { if(m_shape_admitted && !DPCCSameShape(m_admitted_shape, config.shape)) return(false); if(!m_config_admitted) return(true); return(DPCCSameShape(m_admitted_config.shape, config.shape) && m_admitted_config.contract_version == config.contract_version && m_admitted_config.binary_layer_version == config.binary_layer_version && m_admitted_config.normalization_mode == config.normalization_mode && m_admitted_config.loss_coordinates == config.loss_coordinates && m_admitted_composition.embedding == composition.embedding && m_admitted_composition.architecture == composition.architecture && m_admitted_composition.activation_profile == composition.activation_profile && m_admitted_composition.backward_profile == composition.backward_profile && m_admitted_composition.optimizer_profile == composition.optimizer_profile && m_admitted_composition.graph_digest == composition.graph_digest); } //+------------------------------------------------------------------+ //| Commits admission only after a complete explicit operation | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::AdmitConfiguration(const SDPCCConfig &config, const SDPCCDecoderComposition &composition) { m_admitted_shape = config.shape; m_admitted_config = config; m_admitted_composition = composition; m_shape_admitted = true; m_config_admitted = true; } //+------------------------------------------------------------------+ //| Function CNeuronScenarioDecoderOCL::Init | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Init(uint outputs, uint index, COpenCLMy *open_cl, CLayerDescription *description) { return(InitOwned(outputs, index, open_cl, description, true)); } //+------------------------------------------------------------------+ //| Reconstructs ownership without dropping a prior Load admission | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::InitOwned(uint outputs, uint index, COpenCLMy *open_cl, CLayerDescription *description, const bool admit) { m_public_valid = false; m_configured = false; m_failed = true; m_phase = 0; m_source = NULL; m_source_output = NULL; m_update_count = 0; m_generation = 0; CLayerDescription *desc = description; SDPCCShape candidate_shape; if(!open_cl || !DescribeAdmission(desc, candidate_shape)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_price_mode = (ArraySize(desc.units) == 6 && desc.units[5] == defScenarioDecoderPriceABI); m_price_anchor_bound = false; m_price_anchor = 0; m_price_scale = 0; m_price_decision = 0; m_price_error = ""; m_price_phase = defForecastPriceVerify; m_shape = candidate_shape; m_tokens = m_shape.K * m_shape.H; m_width = m_shape.V * m_shape.D; if(!CNeuronBaseOCL::Init(m_price_mode ? 0 : outputs, index, open_cl, m_tokens * 4, ADAM, desc.batch)) return(Fail(DPCC_OPENCL_ERROR, true)); activation = None; if(!Allocate(m_shape.Embedding)) return(Fail(DPCC_OPENCL_ERROR, true)); m_failed = false; if(m_price_mode) { ZeroMemory(m_composition); m_composition.embedding = m_hidden; m_composition.architecture = "SHARED_VD_CAUSAL_OHLC_V3"; m_composition.activation_profile = "TANH"; m_composition.backward_profile = "DPCC_PRICE_CHAIN_V1"; m_composition.optimizer_profile = "NN_NATIVE_CONV_ADAM_C6A8_V1"; m_composition.graph_digest = ""; m_configured = true; } if(admit) { m_admitted_shape = m_shape; m_shape_admitted = true; m_config_admitted = false; ZeroMemory(m_admitted_config); ZeroMemory(m_admitted_composition); } m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Freezes the exact composition and normalized native profile | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Configure(const SDPCCConfig &config, const SDPCCDecoderComposition &composition) { return(ConfigureOwned(config, composition, true)); } //+------------------------------------------------------------------+ //| Configures provisional Load geometry without admitting it | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ConfigureOwned(const SDPCCConfig &config, const SDPCCDecoderComposition &composition, const bool admit) { if(m_price_mode || m_failed || m_phase != 0 || !DPCCSameShape(m_shape, config.shape) || !ValidComposition(config, composition) || !MatchesAdmission(config, composition) || lr != 0.0003f || b1 != 0.9f || b2 != 0.999f) return(Fail(DPCC_CONTRACT_MISMATCH)); if(m_configured) { if(m_hidden != composition.embedding || m_composition.graph_digest != composition.graph_digest) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; return(true); } if(composition.embedding != m_hidden && !Allocate(composition.embedding)) return(Fail(DPCC_OPENCL_ERROR, true)); m_composition = composition; m_configured = true; if(admit) AdmitConfiguration(config, composition); m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Reduces device numeric flags before publishing or mutation | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Reduce(const uint count) { const uint kernel = def_k_DPCCOperatorReduce; setBuffer(kernel, 0, m_work[4].GetIndex()); setBuffer(kernel, 1, m_work[5].GetIndex()); setArgument(kernel, 2, count); uint offset[1] = {0}; uint size[1] = {1}; kernelExecute(kernel, offset, size) if(!m_work[5].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); if(m_work[5][0] != 0) return(Fail((ENUM_DPCC_STATUS)(int)m_work[5][0])); return(true); } //+------------------------------------------------------------------+ //| Packs KVHD axes or restores exact source gradient coordinates | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Pack(CBufferFloat *source, CBufferFloat *destination, const uint inverse) { const uint count = m_tokens * m_width; if(!ValidBuffer(source, count) || !ValidBuffer(destination, count)) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDecoderPack; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_shape.K); setArgument(kernel, 4, m_shape.V); setArgument(kernel, 5, m_shape.H); setArgument(kernel, 6, m_shape.D); setArgument(kernel, 7, inverse); uint offset[1] = {0}; uint size[1] = {count}; kernelExecute(kernel, offset, size) return(Reduce(count)); } //+------------------------------------------------------------------+ //| Executes exact activation or numeric identity validation | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Activate(CBufferFloat *source, CBufferFloat *destination, const int function) { if(!source || !destination || source.GetOpenCL() != OpenCL || destination.GetOpenCL() != OpenCL || source.GetIndex() < 0 || destination.GetIndex() < 0 || source.Total() > destination.Total() || source.Total() <= 0 || source.Total() > m_work[4].Total()) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCActivationForward; const uint count = source.Total(); setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, function); setArgument(kernel, 4, count); uint offset[1] = {0}; uint size[1] = {count}; kernelExecute(kernel, offset, size) return(Reduce(count)); } //+------------------------------------------------------------------+ //| Validates exact native affine products and ordered sums | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::AffinePreflight(CNeuronConvOCL *child, CBufferFloat *source, const uint input_width, const uint output_width) { const uint kernel = def_k_DPCCSharedAffineForwardPreflight; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, source.GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_tokens); setArgument(kernel, 4, input_width); setArgument(kernel, 5, output_width); uint offset[2] = {0, 0}; uint size[2] = {output_width, m_tokens}; kernelExecute(kernel, offset, size) return(Reduce(output_width * m_tokens)); } //+------------------------------------------------------------------+ //| Computes unclipped old-matrix source_neuron gradients | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::AffineGradient(CNeuronConvOCL *child, CBufferFloat *upstream, CBufferFloat *destination, const uint input_width, const uint output_width) { const uint kernel = def_k_DPCCSharedAffineInputGradient; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, upstream.GetIndex()); setBuffer(kernel, 2, destination.GetIndex()); setBuffer(kernel, 3, m_work[4].GetIndex()); setArgument(kernel, 4, m_tokens); setArgument(kernel, 5, input_width); setArgument(kernel, 6, output_width); uint offset[2] = {0, 0}; uint size[2] = {input_width, m_tokens}; kernelExecute(kernel, offset, size) return(Reduce(input_width * m_tokens)); } //+------------------------------------------------------------------+ //| Validates every native Adam candidate before either child writes | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::AdamPreflight(CNeuronConvOCL *child, CBufferFloat *source, const uint input_width, const uint output_width) { const uint step = child.NativeOptimizerStep(); if(step == 0 || step == UINT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH)); const float corrected_lr = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)step)) / (1.0 - MathPow((double)b1, (double)step))); const uint kernel = def_k_DPCCSharedAffineAdamPreflight; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, child.getGradientIndex()); setBuffer(kernel, 2, source.GetIndex()); setBuffer(kernel, 3, child.GetFirstMomentumConv().GetIndex()); setBuffer(kernel, 4, child.GetSecondMomentumConv().GetIndex()); setBuffer(kernel, 5, m_work[4].GetIndex()); setArgument(kernel, 6, m_tokens); setArgument(kernel, 7, input_width); setArgument(kernel, 8, output_width); setArgument(kernel, 9, corrected_lr); setArgument(kernel, 10, b1); setArgument(kernel, 11, b2); uint offset[1] = {0}; uint size[1] = {output_width * (input_width + 1)}; kernelExecute(kernel, offset, size) return(Reduce(size[0])); } //+------------------------------------------------------------------+ //| Validates actual post-update weights and native moments | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CheckParameters(CNeuronConvOCL *child) { if(!Activate(child.GetWeightsConv(), m_work[6], None)) return(false); if(!Activate(child.GetFirstMomentumConv(), m_work[6], None)) return(false); if(!Activate(child.GetSecondMomentumConv(), m_work[6], None)) return(false); return(true); } //+------------------------------------------------------------------+ //| Binds backward and completion to the live source identity | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ExternalSource(CNeuronBaseOCL *source) const { if(!source || source == GetPointer(this) || !source.getOutput() || !source.getGradient()) return(false); CBufferFloat *source_neuron = source.getOutput(); CBufferFloat *input_gradient = source.getGradient(); if(source_neuron == input_gradient || source_neuron.GetIndex() == input_gradient.GetIndex()) return(false); CBufferFloat *owned[] = {Output, PrevOutput, Gradient, m_first.getOutput(), m_first.getGradient(), m_last.getOutput(), m_last.getGradient(), m_first.GetWeightsConv(), m_last.GetWeightsConv(), m_first.GetFirstMomentumConv(), m_first.GetSecondMomentumConv(), m_last.GetFirstMomentumConv(), m_last.GetSecondMomentumConv() }; for(uint i = 0; i < (uint)ArraySize(owned); i++) if(source_neuron == owned[i] || input_gradient == owned[i] || source_neuron.GetIndex() == owned[i].GetIndex() || input_gradient.GetIndex() == owned[i].GetIndex()) return(false); for(uint i = 0; i < 8; i++) if(source_neuron == m_work[i] || input_gradient == m_work[i] || source_neuron.GetIndex() == m_work[i].GetIndex() || input_gradient.GetIndex() == m_work[i].GetIndex()) return(false); for(uint i = 0; i < 4; i++) if(source_neuron == m_loss_work[i] || input_gradient == m_loss_work[i] || source_neuron.GetIndex() == m_loss_work[i].GetIndex() || input_gradient.GetIndex() == m_loss_work[i].GetIndex()) return(false); return(true); } //+------------------------------------------------------------------+ //| Checks live source object and retained buffer identity | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::SameSource(CNeuronBaseOCL *source) const { return(!!source && source == m_source && source.getOutput() == m_source_output && source.getOutputIndex() == m_source_index && ValidBuffer(source.getOutput(), m_tokens * m_width) && ExternalSource(source)); } //+------------------------------------------------------------------+ //| Runs normalized decoder forward with a saved exact generation | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::feedForward(CNeuronBaseOCL *source) { if((m_phase != 0 && bTrain) || (m_price_mode && m_public_valid)) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } m_public_valid = false; if(m_failed || !m_configured || !ValidateOwned() || !source || !ValidBuffer(source.getOutput(), m_tokens * m_width) || !ExternalSource(source) || (m_price_mode && (!m_price_anchor_bound || !PriceSource(source)))) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!Pack(source.getOutput(), m_work[0], 0)) return(false); if(!AffinePreflight(m_first, m_work[0], m_width, m_hidden)) return(false); if(!m_first.FeedForwardBuffer(m_work[0])) return(Fail(DPCC_OPENCL_ERROR)); if(!Activate(m_first.getOutput(), m_work[1], m_price_mode ? TANH : GELU)) return(false); if(!AffinePreflight(m_last, m_work[1], m_hidden, 4)) return(false); if(!m_last.FeedForwardBuffer(m_work[1])) return(Fail(DPCC_OPENCL_ERROR)); if(m_price_mode) { if(!DecodePricePath()) return(false); } else { if(!Activate(m_last.getOutput(), Output, None)) return(false); } m_source = source; m_source_output = source.getOutput(); m_source_index = source.getOutputIndex(); m_generation++; m_phase = (bTrain ? 1 : 0); m_public_valid = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Computes all hidden and source_neuron derivatives before matrix | //| updates | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::calcInputGradients(CNeuronBaseOCL *source) { if(m_failed || !m_configured || m_phase != 1 || !m_public_valid || !SameSource(source) || !ValidateOwned() || !ValidBuffer(source.getGradient(), m_tokens * m_width)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(m_price_mode && (!m_price_anchor_bound || !PriceSource(source) || (m_price_phase != defForecastPriceLatentPrepare && m_price_phase != defForecastPriceJoint && m_price_phase != defForecastPriceAdapt))) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(source)) return(false); //--- Reverse the saved output parameterization before the shared affine chain. if(m_price_mode) { if(!ReversePricePath()) return(false); } else { if(!Activate(Gradient, m_last.getGradient(), None)) return(false); } if(!AffineGradient(m_last, m_last.getGradient(), m_work[2], m_hidden, 4)) return(false); const uint kernel = def_k_DPCCActivationGradient; setBuffer(kernel, 0, m_first.getOutputIndex()); setBuffer(kernel, 1, m_work[2].GetIndex()); setBuffer(kernel, 2, m_first.getGradientIndex()); setBuffer(kernel, 3, m_work[4].GetIndex()); setArgument(kernel, 4, (int)(m_price_mode ? TANH : GELU)); setArgument(kernel, 5, m_tokens * m_hidden); uint offset[1] = {0}; uint size[1] = {m_tokens * m_hidden}; kernelExecute(kernel, offset, size) if(!Reduce(size[0])) return(false); if(!AffineGradient(m_first, m_first.getGradient(), m_work[3], m_width, m_hidden)) return(false); if(!Pack(m_work[3], source.getGradient(), 1)) return(false); m_phase = 2; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Updates both native affines once then closes the generation | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::updateInputWeights(CNeuronBaseOCL *source) { if(m_price_mode && m_price_phase != defForecastPriceJoint && m_price_phase != defForecastPriceAdapt) { m_price_error = "PRICE_UPDATE_PHASE"; return(Fail(DPCC_CONTRACT_MISMATCH)); } if(m_failed || !m_configured || !bTrain || m_phase != 2 || !SameSource(source) || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(source)) return(false); if(!AdamPreflight(m_first, m_work[0], m_width, m_hidden)) return(false); if(!AdamPreflight(m_last, m_work[1], m_hidden, 4)) return(false); if(!m_first.UpdateInputWeightsBuffer(m_work[0])) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_last.UpdateInputWeightsBuffer(m_work[1])) return(Fail(DPCC_OPENCL_ERROR, true)); if(!CheckParameters(m_first)) return(Fail(DPCC_NUMERIC_ERROR, true)); if(!CheckParameters(m_last)) return(Fail(DPCC_NUMERIC_ERROR, true)); m_update_count++; if(!ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_phase = 0; m_source = NULL; m_source_output = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Closes frozen backward while retaining computed gradients | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CompleteFrozen(CNeuronBaseOCL *source) { if(m_failed || !m_configured || m_phase != 2 || !SameSource(source) || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(source)) return(false); m_phase = 0; m_source = NULL; m_source_output = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Rejects unsupported second inputs | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::feedForward(CNeuronBaseOCL *source, CBufferFloat *second) { if(second) return(Fail(DPCC_CONTRACT_MISMATCH)); return(feedForward(source)); } //+------------------------------------------------------------------+ //| Dispatches exact source_neuron gradients for the | //| single-source_neuron graph | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::calcInputGradients(CNeuronBaseOCL *source, CBufferFloat *second, CBufferFloat *second_gradient, ENUM_ACTIVATION function) { if(!!second || !!second_gradient) return(Fail(DPCC_CONTRACT_MISMATCH)); return(calcInputGradients(source)); } //+------------------------------------------------------------------+ //| Dispatches native updates for the single-source_neuron graph | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::updateInputWeights(CNeuronBaseOCL *source, CBufferFloat *second) { if(second) return(Fail(DPCC_CONTRACT_MISMATCH)); return(updateInputWeights(source)); } //+------------------------------------------------------------------+ //| Dispatches decoder forward without base training shortcuts | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::FeedForward(CObject *source, CBufferFloat *second) { if(!source) return(Fail(DPCC_INVALID_INPUT)); CNeuronBaseOCL *source_neuron = dynamic_cast(source); if(!source_neuron) return(Fail(DPCC_INVALID_INPUT)); return(feedForward(source_neuron, second)); } //+------------------------------------------------------------------+ //| Dispatches object-aware decoder forward | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::FeedForwardSecond(CObject *source, CNeuronBaseOCL *second) { if(second) return(Fail(DPCC_CONTRACT_MISMATCH)); return(FeedForward(source, NULL)); } //+------------------------------------------------------------------+ //| Dispatches decoder updates without silent base training skips | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::UpdateInputWeights(CObject *source, CBufferFloat *second) { if(!source) return(Fail(DPCC_INVALID_INPUT)); CNeuronBaseOCL *source_neuron = dynamic_cast(source); if(!source_neuron) return(Fail(DPCC_INVALID_INPUT)); //--- Latent preparation traverses the full decoder VJP without an affine/Adam write. if(m_price_mode && m_price_phase == defForecastPriceLatentPrepare) { if(second || !PriceSource(source_neuron)) return(Fail(DPCC_CONTRACT_MISMATCH)); return(CompleteFrozen(source_neuron)); } return(updateInputWeights(source_neuron, second)); } //+------------------------------------------------------------------+ //| Dispatches object-aware decoder updates | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::UpdateInputWeightsSecond(CObject *source, CNeuronBaseOCL *second) { if(second) return(Fail(DPCC_CONTRACT_MISMATCH)); return(UpdateInputWeights(source, NULL)); } //+------------------------------------------------------------------+ //| Completes a CNet frozen-weight transaction | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::UpdateInputState(CObject *source, CBufferFloat *second) { if(!source || !!second) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronBaseOCL *source_neuron = dynamic_cast(source); if(!source_neuron) return(Fail(DPCC_INVALID_INPUT)); return(CompleteFrozen(source_neuron)); } //+------------------------------------------------------------------+ //| Completes an object-aware frozen-weight transaction | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::UpdateInputStateSecond(CObject *source, CNeuronBaseOCL *second) { if(second) return(Fail(DPCC_CONTRACT_MISMATCH)); return(UpdateInputState(source, NULL)); } //+------------------------------------------------------------------+ //| Sets train mode only between completed generations | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::TrainMode(bool flag) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return; } bTrain = flag; if(m_first) m_first.TrainMode(flag); if(m_last) m_last.TrainMode(flag); } //+------------------------------------------------------------------+ //| Clears transient decoder state preserving matrices and moments | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Clear(void) { m_public_valid = false; m_price_anchor_bound = false; m_price_anchor = 0; m_price_scale = 0; m_price_decision = 0; m_price_error = ""; if(m_failed || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CNeuronBaseOCL::Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); if(!PrevOutput.Fill(0)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_first.Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_last.Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 8; i++) if(!m_work[i].Fill(0)) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 4; i++) if(!m_loss_work[i].Fill(0)) return(Fail(DPCC_OPENCL_ERROR, true)); m_phase = 0; m_source = NULL; m_source_output = NULL; m_source_index = INVALID_HANDLE; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Publishes decoder status and generation diagnostics | //+------------------------------------------------------------------+ CBufferFloat *CNeuronScenarioDecoderOCL::GetDiagnostics(void) { if(m_failed || !m_configured || !m_public_valid) return(NULL); const float values[] = {(float)m_status, (float)m_phase, (float)m_update_count, (float)m_generation, (float)m_first.NativeOptimizerStep(), (float)m_last.NativeOptimizerStep(), (float)m_tokens, (float)m_hidden }; for(uint i = 0; i < 8; i++) m_work[7].Update(i, values[i]); if(!m_work[7].BufferWrite()) { Fail(DPCC_OPENCL_ERROR); return(NULL); } return(m_work[7]); } //+------------------------------------------------------------------+ //| Hashes graph and fixed composition without execution capacity | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::HashComposition(ulong &hash) const { DPCCRolloutHashWord(Type(), hash); if(m_price_mode) DPCCRolloutHashWord(defScenarioDecoderPriceABI, hash); DPCCRolloutHashWord(m_width, hash); DPCCRolloutHashWord(m_shape.V, hash); DPCCRolloutHashWord(m_shape.D, hash); DPCCRolloutHashWord(DPCC_NORMALIZATION_CAUSAL_AFFINE_V2, hash); DPCCRolloutHashWord(DPCC_LOSS_NORMALIZED, hash); DPCCRolloutHashWord(m_hidden, hash); const string texts[] = {m_composition.architecture, m_composition.activation_profile, m_composition.backward_profile, m_composition.optimizer_profile, m_composition.graph_digest }; for(uint tindex = 0; tindex < 5; tindex++) { DPCCRolloutHashWord(StringLen(texts[tindex]), hash); for(int c = 0; c < StringLen(texts[tindex]); c++) DPCCRolloutHashWord(StringGetCharacter(texts[tindex], c), hash); } } //+------------------------------------------------------------------+ //| Hashes weight values excluding native moments and token capacity | //+------------------------------------------------------------------+ ulong CNeuronScenarioDecoderOCL::WeightSignature(void) { if(m_failed || !m_configured || !ValidateOwned()) return(0); ulong hash = 1469598103934665603; HashComposition(hash); if(!AppendParameterBufferFingerprint(m_first.GetWeightsConv(), hash, 1)) return(0); if(!AppendParameterBufferFingerprint(m_last.GetWeightsConv(), hash, 2)) return(0); return(hash); } //+------------------------------------------------------------------+ //| Hashes execution capacities and fixed decoder contract | //+------------------------------------------------------------------+ ulong CNeuronScenarioDecoderOCL::ContractSignature(void) const { if(m_failed || !m_configured) return(0); ulong hash = 1469598103934665603; HashComposition(hash); const uint values[] = {2, 3, m_shape.K, m_shape.V, m_shape.H, m_shape.D, m_shape.M, m_shape.J, m_shape.E, m_shape.PlanBatch, m_shape.Embedding, m_shape.Heads, m_shape.KeyDimension, m_shape.NActions, m_shape.StateSize }; for(uint i = 0; i < (uint)ArraySize(values); i++) DPCCRolloutHashWord(values[i], hash); return(hash); } //+------------------------------------------------------------------+ //| Appends both native children including actual optimizer steps | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::AppendParameterFingerprint(ulong &fingerprint) { if(m_failed || !m_configured || !ValidateOwned()) return(false); HashComposition(fingerprint); DPCCRolloutHashWord(m_update_count, fingerprint); DPCCRolloutHashWord(m_first.NativeOptimizerStep(), fingerprint); DPCCRolloutHashWord(m_last.NativeOptimizerStep(), fingerprint); if(!m_first.AppendParameterFingerprint(fingerprint)) return(false); if(!m_last.AppendParameterFingerprint(fingerprint)) return(false); return(true); } //+------------------------------------------------------------------+ //| Returns complete trainable state fingerprint | //+------------------------------------------------------------------+ ulong CNeuronScenarioDecoderOCL::TrainableStateFingerprint(void) { ulong hash = 1469598103934665603; return(AppendParameterFingerprint(hash) ? hash : 0); } //+------------------------------------------------------------------+ //| Hashes runtime decoder source_neuron independently from weights | //+------------------------------------------------------------------+ ulong CNeuronScenarioDecoderOCL::SnapshotFingerprint(void) { if(m_failed || !m_public_valid) return(0); ulong hash = ContractSignature(); if(m_price_mode) { DPCCRolloutHashWord(m_price_decision, hash); const string values[] = {DoubleToString(m_price_anchor, -16), DoubleToString(m_price_scale, -16)}; for(uint i = 0; i < 2; i++) for(int c = 0; c < StringLen(values[i]); c++) DPCCRolloutHashWord(StringGetCharacter(values[i], c), hash); } if(!AppendParameterBufferFingerprint(m_work[0], hash, 1)) return(0); return(hash); } //+------------------------------------------------------------------+ //| Imports parameters only into a fresh compatible receiver | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type() || tau != 1.0f || m_failed || !m_configured || m_phase != 0 || m_update_count != 0 || m_generation != 0 || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronScenarioDecoderOCL *decoder = source; if(decoder == GetPointer(this) || decoder.OpenCL != OpenCL || decoder.m_failed || !decoder.m_configured || decoder.m_phase != 0 || !decoder.ValidateOwned() || m_width != decoder.m_width || m_hidden != decoder.m_hidden || m_price_mode != decoder.m_price_mode || m_shape.V != decoder.m_shape.V || m_shape.D != decoder.m_shape.D || m_composition.graph_digest != decoder.m_composition.graph_digest) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckParameters(decoder.m_first) || !CheckParameters(decoder.m_last)) return(Fail(DPCC_NUMERIC_ERROR)); if(!m_first.WeightsUpdate(decoder.m_first, 1.0f)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_last.WeightsUpdate(decoder.m_last, 1.0f)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!CheckParameters(m_first) || !CheckParameters(m_last)) return(Fail(DPCC_NUMERIC_ERROR, true)); m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Validates the exact decoder owned-tree transfer capability | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::SupportsOpenCLChecked(void) const { return(Type() == defNeuronScenarioDecoderOCL && !m_failed && !!m_first && !!m_last && m_first.SupportsOpenCLChecked() && m_last.SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers decoder buffers while source contexts remain alive | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::SetOpenCLChecked(COpenCLMy *target) { if(!target || !SupportsOpenCLChecked() || m_phase != 0 || (m_price_mode && m_public_valid)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(target == OpenCL && ValidateOwned()) return(true); m_public_valid = false; for(uint i = 0; i < 8; i++) if(m_work[i].GetIndex() >= 0 && m_work[i].GetOpenCL() != target && !m_work[i].BufferRead()) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 4; i++) if(m_loss_work[i].GetIndex() >= 0 && m_loss_work[i].GetOpenCL() != target && !m_loss_work[i].BufferRead()) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_first.SetOpenCLChecked(target)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_last.SetOpenCLChecked(target)) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 8; i++) if((m_work[i].GetOpenCL() != target || m_work[i].GetIndex() < 0) && !m_work[i].BufferCreate(target)) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 4; i++) if((m_loss_work[i].GetOpenCL() != target || m_loss_work[i].GetIndex() < 0) && !m_loss_work[i].BufferCreate(target)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!CNeuronBaseOCL::SetOpenCLChecked(target)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_source = NULL; m_source_output = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Routes legacy context rebinding through checked ownership | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::SetOpenCL(COpenCLMy *target) { SetOpenCLChecked(target); } //+------------------------------------------------------------------+ //| Reconstructs the canonical descriptor | //+------------------------------------------------------------------+ CLayerDescription *CNeuronScenarioDecoderOCL::GetLayerInfo(void) { if(m_failed) return(NULL); CLayerDescription *desc = new CLayerDescription; if(!desc) return(NULL); desc.type = Type(); desc.count = m_tokens * 4; if(m_price_mode) { desc.window = m_width; desc.window_out = m_hidden; desc.step = 1; desc.variables = 1; desc.layers = 2; desc.batch = 1; desc.activation = None; desc.optimization = ADAM; if(ArrayResize(desc.units, 6) != 6) { DeleteObj(desc); return(NULL); } const uint units[] = {m_shape.K, m_shape.V, m_shape.H, m_shape.D, m_hidden, defScenarioDecoderPriceABI }; for(uint i = 0; i < 6; i++) desc.units[i] = units[i]; return(desc); } desc.window = 6; desc.window_out = 13; desc.variables = m_shape.V; desc.step = m_shape.H; desc.layers = 0; desc.batch = iBatch; desc.activation = None; desc.optimization = ADAM; if(ArrayResize(desc.windows, 9) != 9 || ArrayResize(desc.units, 5) != 5 || ArrayResize(desc.heads, 1) != 1) { DeleteObj(desc); return(NULL); } const uint windows[] = {2, m_shape.K, m_shape.V, m_shape.H, m_shape.D, m_shape.M, m_shape.J, m_shape.E, m_shape.PlanBatch }; const uint units[] = {m_shape.Embedding, m_shape.Heads, m_shape.KeyDimension, 6, 13}; for(uint i = 0; i < 9; i++) desc.windows[i] = windows[i]; for(uint i = 0; i < 5; i++) desc.units[i] = units[i]; desc.heads[0] = m_shape.Heads; return(desc); } //+------------------------------------------------------------------+ //| Saves completed native checkpoint and exact decoder composition | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Save(const int file_handle) { if(m_price_mode) return(SavePrice(file_handle)); if(file_handle == INVALID_HANDLE || m_failed || !m_configured || m_phase != 0 || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint header[] = {(uint)Type(), 1, 2, 3, m_shape.K, m_shape.V, m_shape.H, m_shape.D, m_shape.M, m_shape.J, m_shape.E, m_shape.PlanBatch, m_shape.Embedding, m_shape.Heads, m_shape.KeyDimension, m_hidden, m_update_count }; for(uint i = 0; i < (uint)ArraySize(header); i++) if(FileWriteInteger(file_handle, (int)header[i], INT_VALUE) != INT_VALUE) return(Fail(DPCC_INVALID_INPUT)); const string texts[] = {m_composition.architecture, m_composition.activation_profile, m_composition.backward_profile, m_composition.optimizer_profile, m_composition.graph_digest }; for(uint i = 0; i < 5; i++) { const uint length = StringLen(texts[i]); if(FileWriteInteger(file_handle, (int)length, INT_VALUE) != INT_VALUE) return(Fail(DPCC_INVALID_INPUT)); if(FileWriteString(file_handle, texts[i], (int)length) == 0) return(Fail(DPCC_INVALID_INPUT)); } if(!CNeuronBaseOCL::Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); if(!m_first.Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); if(!m_last.Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); return(true); } //+------------------------------------------------------------------+ //| Loads native child matrices and moments without live generations | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::Load(const int file_handle) { m_public_valid = false; m_configured = false; m_failed = true; m_phase = 0; m_source = NULL; m_source_output = NULL; m_source_index = INVALID_HANDLE; if(file_handle == INVALID_HANDLE || !OpenCL) return(Fail(DPCC_INVALID_INPUT, true)); const ulong payload_start = FileTell(file_handle); const int candidate_abi = FileReadInteger(file_handle, INT_VALUE); if(!FileSeek(file_handle, (long)payload_start, SEEK_SET)) return(Fail(DPCC_INVALID_INPUT, true)); if(candidate_abi == defScenarioDecoderPriceABI) return(LoadPrice(file_handle)); if(m_price_mode) return(Fail(DPCC_CONTRACT_MISMATCH, true)); uint header[16]; if(FileReadArray(file_handle, header, 0, 16) != 16 || header[0] != 1 || header[1] != 2 || header[2] != 3) return(Fail(DPCC_CONTRACT_MISMATCH, true)); CLayerDescription desc; desc.type = Type(); const ulong output_total = (ulong)header[3] * header[5] * 4; if(output_total > INT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH, true)); desc.count = (uint)output_total; desc.window = 6; desc.window_out = 13; desc.variables = header[4]; desc.step = header[5]; desc.layers = 0; desc.batch = 1; desc.activation = None; desc.optimization = ADAM; ArrayResize(desc.windows, 9); ArrayResize(desc.units, 5); ArrayResize(desc.heads, 1); desc.windows[0] = 2; for(uint i = 1; i < 9; i++) desc.windows[i] = header[i + 2]; desc.units[0] = header[11]; desc.units[1] = header[12]; desc.units[2] = header[13]; desc.units[3] = 6; desc.units[4] = 13; desc.heads[0] = header[12]; const uint restored_count = header[15]; SDPCCShape candidate_shape; if(restored_count >= UINT_MAX - 1 || !DescribeAdmission(GetPointer(desc), candidate_shape)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); string texts[5]; for(uint i = 0; i < 5; i++) { const int length = FileReadInteger(file_handle, INT_VALUE); if(length <= 0 || length > 128) return(Fail(DPCC_CONTRACT_MISMATCH, true)); texts[i] = FileReadString(file_handle, length); if(StringLen(texts[i]) != length) return(Fail(DPCC_CONTRACT_MISMATCH, true)); } SDPCCDecoderComposition composition; composition.embedding = header[14]; composition.architecture = texts[0]; composition.activation_profile = texts[1]; composition.backward_profile = texts[2]; composition.optimizer_profile = texts[3]; composition.graph_digest = texts[4]; SDPCCConfig config; ZeroMemory(config); config.shape = candidate_shape; config.contract_version = 2; config.binary_layer_version = 3; config.normalization_mode = DPCC_NORMALIZATION_CAUSAL_AFFINE_V2; config.loss_coordinates = DPCC_LOSS_NORMALIZED; //--- Compare the complete candidate before Init or any child reallocation. if(!ValidComposition(config, composition) || !MatchesAdmission(config, composition)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!InitOwned(0, 0, OpenCL, GetPointer(desc), false) || !ConfigureOwned(config, composition, false)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_failed = true; if(FileReadInteger(file_handle, INT_VALUE) != Type() || !CNeuronBaseOCL::Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !m_first.Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !m_last.Load(file_handle)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_update_count = restored_count; if(activation != None || optimization != ADAM || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!CheckParameters(m_first) || !CheckParameters(m_last)) return(Fail(DPCC_NUMERIC_ERROR, true)); m_failed = false; if(!Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); if(!AdamPreflight(m_first, m_work[0], m_width, m_hidden)) return(Fail(DPCC_NUMERIC_ERROR, true)); if(!AdamPreflight(m_last, m_work[1], m_hidden, 4)) return(Fail(DPCC_NUMERIC_ERROR, true)); AdmitConfiguration(config, composition); m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Keeps decoder output activation fixed at normalized linear | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::SetActivationFunction(ENUM_ACTIVATION value) { if(value != None) Fail(DPCC_CONTRACT_MISMATCH); } //+------------------------------------------------------------------+ //| Checks exact live source bits against the owned packed tape | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CheckSourceSnapshot(CNeuronBaseOCL *source) { if(!SameSource(source)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); const uint kernel = def_k_DPCCDecoderSnapshotCheck; setBuffer(kernel, 0, source.getOutputIndex()); setBuffer(kernel, 1, m_work[0].GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_shape.K); setArgument(kernel, 4, m_shape.V); setArgument(kernel, 5, m_shape.H); setArgument(kernel, 6, m_shape.D); uint offset[1] = {0}; uint size[1] = {m_tokens * m_width}; kernelExecute(kernel, offset, size) if(!Reduce(size[0])) { m_failed = true; return(false); } return(true); } //+------------------------------------------------------------------+ //| Allows constructor context attachment before allocation only | //+------------------------------------------------------------------+ void CNeuronScenarioDecoderOCL::AttachOpenCL(COpenCLMy *context) { if(m_first || m_last || m_configured || m_tokens != 0) { Fail(DPCC_CONTRACT_MISMATCH, true); return; } OpenCL = context; } //+------------------------------------------------------------------+ //| Checks provider Z against the decoder's saved packed input | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CheckForecastLossSnapshot(CNeuronScenarioForecast *provider) { if(!provider || provider.Type() != defNeuronScenarioForecast || provider.Scenarios() != m_shape.K || provider.Variables() != m_shape.V || provider.Horizon() != m_shape.H || provider.Dimension() != m_shape.D || !ValidBuffer(provider.GetZ(), m_tokens * m_width)) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDecoderSnapshotCheck; setBuffer(kernel, 0, provider.GetZ().GetIndex()); setBuffer(kernel, 1, m_work[0].GetIndex()); setBuffer(kernel, 2, m_work[4].GetIndex()); setArgument(kernel, 3, m_shape.K); setArgument(kernel, 4, m_shape.V); setArgument(kernel, 5, m_shape.H); setArgument(kernel, 6, m_shape.D); uint offset[1] = {0}; uint size[1] = {m_tokens * m_width}; kernelExecute(kernel, offset, size) return(Reduce(size[0])); } //+------------------------------------------------------------------+ //| Rejects supervision aliases into mutable model storage | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::ValidForecastLossTarget(CNeuronScenarioForecast *provider, CBufferFloat *target, const uint total) const { if(!ValidBuffer(target, total) || !provider || !m_source || provider.OwnsBufferOrDeviceSlot(target) || m_source.OwnsBufferOrDeviceSlot(target) || m_first.OwnsBufferOrDeviceSlot(target) || m_last.OwnsBufferOrDeviceSlot(target)) return(false); CBufferFloat *owned[] = {Output, PrevOutput, Weights, DeltaWeights, Gradient, FirstMomentum, SecondMomentum }; for(uint i = 0; i < (uint)ArraySize(owned); i++) if(BufferAliases(target, owned[i])) return(false); for(uint i = 0; i < 8; i++) if(BufferAliases(target, m_work[i])) return(false); for(uint i = 0; i < 4; i++) if(BufferAliases(target, m_loss_work[i])) return(false); return(true); } //+------------------------------------------------------------------+ //| Builds fresh forecast rho loss and its owned negative gradient | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::DispatchRhoLoss(CBufferFloat *target_normalized, CBufferFloat *rho, const float rho_sum_tolerance, const bool prepare_gradient, double &loss, const float lambda_price, const float huber_delta) { //--- Compute into private scratch; no public gradient is written before numeric admission. const uint partial_kernel = m_price_mode ? def_k_DPCCDecoderPriceLoss : def_k_DPCCDecoderRhoLoss; setBuffer(partial_kernel, 0, Output.GetIndex()); setBuffer(partial_kernel, 1, target_normalized.GetIndex()); setBuffer(partial_kernel, 2, rho.GetIndex()); setBuffer(partial_kernel, 3, m_loss_work[0].GetIndex()); setBuffer(partial_kernel, 4, m_loss_work[1].GetIndex()); setBuffer(partial_kernel, 5, m_loss_work[2].GetIndex()); setArgument(partial_kernel, 6, m_shape.K); setArgument(partial_kernel, 7, m_shape.H); setArgument(partial_kernel, 8, (uint)prepare_gradient); if(m_price_mode) { setArgument(partial_kernel, 9, lambda_price); setArgument(partial_kernel, 10, huber_delta); } uint offset[1] = {0}; uint size[1] = {m_shape.K}; kernelExecute(partial_kernel, offset, size) const uint reduce_kernel = def_k_DPCCDecoderRhoLossReduce; setBuffer(reduce_kernel, 0, rho.GetIndex()); setBuffer(reduce_kernel, 1, m_loss_work[1].GetIndex()); setBuffer(reduce_kernel, 2, m_loss_work[2].GetIndex()); setBuffer(reduce_kernel, 3, m_loss_work[3].GetIndex()); setArgument(reduce_kernel, 4, m_shape.K); setArgument(reduce_kernel, 5, rho_sum_tolerance); size[0] = 1; kernelExecute(reduce_kernel, offset, size) if(!m_loss_work[3].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); const float status = m_loss_work[3][0]; const float value = m_loss_work[3][1]; const float mass = m_loss_work[3][2]; if(!MathIsValidNumber(status) || !MathIsValidNumber(value) || !MathIsValidNumber(mass)) return(Fail(DPCC_NUMERIC_ERROR)); if(status != 0) return(Fail((ENUM_DPCC_STATUS)(int)status)); if(value < 0 || MathAbs((double)mass - 1.0) > (double)rho_sum_tolerance) return(Fail(DPCC_INVALID_INPUT)); if(prepare_gradient && !CopyBufferRaw(m_loss_work[0], Gradient, m_tokens * 4)) return(Fail(DPCC_OPENCL_ERROR, true)); //--- Validation leaves Gradient unchanged and never performs a backward pass. loss = (double)value; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Normalizes immutable native supervision with admitted statistics | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BuildNativeBankRhoLoss(CBufferFloat *target_native, CBufferFloat *price_norm, CBufferFloat *responsibilities, const float rho_sum_tolerance, const bool prepare_gradient, double &loss) { if(m_price_mode) { loss = 0.0; m_price_error = "PRICE_LOSS_NOT_IMPLEMENTED"; return(Fail(DPCC_CONTRACT_MISMATCH)); } loss = 0.0; if(!m_public_valid || m_failed || !m_configured || !ValidateOwned() || CheckPointer(m_source) == POINTER_INVALID || !MathIsValidNumber(rho_sum_tolerance) || rho_sum_tolerance < 0 || rho_sum_tolerance >= 1 || (prepare_gradient ? (!bTrain || m_phase != 1) : (bTrain || m_phase != 0))) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *inputs[] = {target_native, price_norm, responsibilities}; const uint sizes[] = {m_shape.H * 4, 16, m_shape.K}; //--- All caller-owned inputs must be separate from the complete model/source tree. for(uint i = 0; i < 3; i++) { if(!ValidBuffer(inputs[i], sizes[i]) || OpenCL.GetBufferSize(inputs[i].GetIndex()) != (ulong)sizes[i] * sizeof(float) || OwnsBufferOrDeviceSlot(inputs[i]) || m_source.OwnsBufferOrDeviceSlot(inputs[i])) return(Fail(DPCC_CONTRACT_MISMATCH)); for(uint j = 0; j < i; j++) if(BufferAliases(inputs[i], inputs[j])) return(Fail(DPCC_CONTRACT_MISMATCH)); } if(!CheckSourceSnapshot(m_source)) return(false); //--- Work6 has capacity >= K*H*4; only its first H4 values are consumed. //--- It is not a retained activation and parameter preflight overwrites it later. const uint kernel = def_k_DPCCDecoderNativeTarget; setBuffer(kernel, 0, target_native.GetIndex()); setBuffer(kernel, 1, price_norm.GetIndex()); setBuffer(kernel, 2, m_work[6].GetIndex()); setBuffer(kernel, 3, m_loss_work[3].GetIndex()); setArgument(kernel, 4, m_shape.H * 4); uint offset[1] = {0}; uint size[1] = {1}; kernelExecute(kernel, offset, size) if(!m_loss_work[3].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); const float status = m_loss_work[3][0]; if(!MathIsValidNumber(status)) return(Fail(DPCC_NUMERIC_ERROR)); if(status != 0) return(Fail((ENUM_DPCC_STATUS)(int)status)); return(DispatchRhoLoss(m_work[6], responsibilities, rho_sum_tolerance, prepare_gradient, loss)); } //+------------------------------------------------------------------+ //| Build the bank rho loss | //+------------------------------------------------------------------+ //--- Caller admits the immutable bank record and rho provenance before this numerical API. //+------------------------------------------------------------------+ //| Builds the bank rho loss and optional gradient. | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BuildBankRhoLoss(CBufferFloat *target_normalized, CBufferFloat *responsibilities, const float rho_sum_tolerance, const bool prepare_gradient, double &loss) { if(m_price_mode) { loss = 0.0; m_price_error = "PRICE_LOSS_NOT_IMPLEMENTED"; return(Fail(DPCC_CONTRACT_MISMATCH)); } loss = 0.0; if(!m_public_valid || m_failed || !m_configured || !ValidateOwned() || CheckPointer(m_source) == POINTER_INVALID || !MathIsValidNumber(rho_sum_tolerance) || rho_sum_tolerance < 0 || rho_sum_tolerance >= 1 || (prepare_gradient ? (!bTrain || m_phase != 1) : (bTrain || m_phase != 0))) return(Fail(DPCC_CONTRACT_MISMATCH)); //--- Every alias is refused before any kernel, scratch write, or source snapshot check. if(!ValidBuffer(target_normalized, m_shape.H * 4) || !ValidBuffer(responsibilities, m_shape.K) || BufferAliases(target_normalized, responsibilities) || OwnsBufferOrDeviceSlot(target_normalized) || OwnsBufferOrDeviceSlot(responsibilities) || m_source.OwnsBufferOrDeviceSlot(target_normalized) || m_source.OwnsBufferOrDeviceSlot(responsibilities)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(m_source)) return(false); return(DispatchRhoLoss(target_normalized, responsibilities, rho_sum_tolerance, prepare_gradient, loss)); } //+------------------------------------------------------------------+ //| Normalizes native OHLC before the existing Forecast rho loss | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BuildForecastNativeRhoLoss(CNeuronScenarioForecast *provider, CBufferFloat *target_native, CBufferFloat *price_norm, CBufferFloat *full_latent, CBufferFloat *delta_latent, const float rho_sum_tolerance, const bool prepare_gradient, double &loss) { if(m_price_mode) { loss = 0.0; m_price_error = "PRICE_LOSS_NOT_IMPLEMENTED"; return(Fail(DPCC_CONTRACT_MISMATCH)); } loss = 0.0; //--- Admit the frozen provider and the decoder's existing forward generation. if(!m_public_valid || m_failed || !m_configured || CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(m_source) == POINTER_INVALID || CheckPointer(provider) == POINTER_INVALID || !ValidateOwned() || provider.Type() != defNeuronScenarioForecast || provider.GetOpenCL() != OpenCL || provider.Scenarios() != m_shape.K || provider.Variables() != m_shape.V || provider.Horizon() != m_shape.H || provider.Dimension() != m_shape.D || !MathIsValidNumber(rho_sum_tolerance) || rho_sum_tolerance < 0 || rho_sum_tolerance >= 1 || (prepare_gradient ? (!bTrain || m_phase != 1) : (bTrain || m_phase != 0))) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronBaseOCL *provider_base = provider; if(provider_base.TrainMode() || provider.CodebookUpdate() || m_source.GetOpenCL() != OpenCL || !ValidBuffer(provider.GetZ(), m_tokens * m_width) || OpenCL.GetBufferSize(provider.GetZ().GetIndex()) != (ulong)m_tokens * m_width * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *rho = provider.GetResponsibilities(); if(!ValidBuffer(rho, m_shape.K) || OpenCL.GetBufferSize(rho.GetIndex()) != (ulong)m_shape.K * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); //--- Refuse every external alias before normalization or any snapshot dispatch. const uint target_total = m_shape.V * m_shape.H * m_shape.D; CBufferFloat *inputs[] = {target_native, price_norm, full_latent, delta_latent}; const uint sizes[] = {m_shape.H * 4, 16, target_total, target_total}; for(uint i = 0; i < 4; i++) { if(!ValidForecastLossTarget(provider, inputs[i], sizes[i]) || OpenCL.GetBufferSize(inputs[i].GetIndex()) != (ulong)sizes[i] * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); for(uint j = 0; j < i; j++) if(BufferAliases(inputs[i], inputs[j])) return(Fail(DPCC_CONTRACT_MISMATCH)); } if(!CheckSourceSnapshot(m_source)) return(false); if(!CheckForecastLossSnapshot(provider)) return(false); //--- The existing loss refuses model-owned scratch targets; keep this H4 candidate detached. //--- Its scoped owner releases the device slot on every return while OpenCL remains alive. CBufferFloat normalized; if(!normalized.BufferInit(m_shape.H * 4, 0.0f)) return(Fail(DPCC_OPENCL_ERROR)); if(!normalized.BufferCreate(OpenCL)) return(Fail(DPCC_OPENCL_ERROR)); CBufferFloat *candidate = GetPointer(normalized); if(!ValidForecastLossTarget(provider, candidate, m_shape.H * 4) || OpenCL.GetBufferSize(candidate.GetIndex()) != (ulong)m_shape.H * 4 * sizeof(float)) return(Fail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 4; i++) if(BufferAliases(candidate, inputs[i])) return(Fail(DPCC_CONTRACT_MISMATCH)); //--- Reuse the single causal gain=1 native-target normalization kernel. const uint kernel = def_k_DPCCDecoderNativeTarget; if(!OpenCL.SetArgumentBuffer(kernel, 0, target_native.GetIndex())) return(Fail(DPCC_OPENCL_ERROR)); if(!OpenCL.SetArgumentBuffer(kernel, 1, price_norm.GetIndex())) return(Fail(DPCC_OPENCL_ERROR)); if(!OpenCL.SetArgumentBuffer(kernel, 2, normalized.GetIndex())) return(Fail(DPCC_OPENCL_ERROR)); if(!OpenCL.SetArgumentBuffer(kernel, 3, m_loss_work[3].GetIndex())) return(Fail(DPCC_OPENCL_ERROR)); if(!OpenCL.SetArgument(kernel, 4, m_shape.H * 4)) return(Fail(DPCC_OPENCL_ERROR)); uint offset[1] = {0}; uint size[1] = {1}; if(!OpenCL.Execute(kernel, 1, offset, size)) return(Fail(DPCC_OPENCL_ERROR)); if(!m_loss_work[3].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); const float status = m_loss_work[3][0]; if(!MathIsValidNumber(status)) return(Fail(DPCC_NUMERIC_ERROR)); if(status != 0) return(Fail((ENUM_DPCC_STATUS)(int)status)); //--- Snapshot checks, responsibilities and gradient publication remain single-sourced. return(BuildForecastRhoLoss(provider, candidate, full_latent, delta_latent, rho_sum_tolerance, prepare_gradient, loss)); } //+------------------------------------------------------------------+ //| Function CNeuronScenarioDecoderOCL::BuildForecastRhoLoss | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::BuildForecastRhoLoss(CNeuronScenarioForecast *provider, CBufferFloat *target_normalized, CBufferFloat *full_latent, CBufferFloat *delta_latent, const float rho_sum_tolerance, const bool prepare_gradient, double &loss) { if(m_price_mode) { loss = 0.0; m_price_error = "PRICE_LOSS_NOT_IMPLEMENTED"; return(Fail(DPCC_CONTRACT_MISMATCH)); } loss = 0.0; //--- Require an existing forward and its own unchanged source generation. if(!m_public_valid || m_failed || !m_configured || !ValidateOwned() || !m_source || !provider || provider.Type() != defNeuronScenarioForecast || !MathIsValidNumber(rho_sum_tolerance) || rho_sum_tolerance < 0 || rho_sum_tolerance >= 1 || (prepare_gradient ? (!bTrain || m_phase != 1) : (bTrain || m_phase != 0))) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronBaseOCL *provider_base = provider; if(provider_base.TrainMode() || provider.CodebookUpdate()) return(Fail(DPCC_CONTRACT_MISMATCH)); //--- Admit every external target before any device operation or provider mutation. const uint target_total = m_shape.V * m_shape.H * m_shape.D; if(full_latent == delta_latent || !ValidForecastLossTarget(provider, target_normalized, m_shape.H * 4) || !ValidForecastLossTarget(provider, full_latent, target_total) || !ValidForecastLossTarget(provider, delta_latent, target_total) || target_normalized.GetIndex() == full_latent.GetIndex() || target_normalized.GetIndex() == delta_latent.GetIndex() || full_latent.GetIndex() == delta_latent.GetIndex()) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *rho = provider.GetResponsibilities(); if(!ValidBuffer(rho, m_shape.K)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CheckSourceSnapshot(m_source)) return(false); if(!CheckForecastLossSnapshot(provider)) return(false); //--- Validate both detached targets even when frozen codebook skips reading delta. if(!Activate(target_normalized, m_work[6], None)) return(false); if(!Activate(full_latent, m_work[6], None)) return(false); if(!Activate(delta_latent, m_work[6], None)) return(false); //--- A fresh two-target call is mandatory; previous LastTargetAccepted never authorizes this loss. if(!provider.BuildResponsibilities(full_latent, delta_latent)) return(Fail(DPCC_INVALID_INPUT)); if(!provider.LastBatchControlValid() || !provider.LastTargetAccepted()) return(Fail(DPCC_INVALID_INPUT)); if(!CheckForecastLossSnapshot(provider)) return(false); if(!CheckSourceSnapshot(m_source)) return(false); return(DispatchRhoLoss(target_normalized, rho, rho_sum_tolerance, prepare_gradient, loss)); } //+------------------------------------------------------------------+ //| Cancels an unfinished generation without updating parameters | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CancelGeneration(void) { if(m_failed || !m_configured || !ValidateOwned() || ((m_phase != 1 && m_phase != 2) && !(m_price_mode && m_phase == 0 && m_price_anchor_bound))) return(false); if(m_price_mode) { m_price_anchor_bound = false; m_price_anchor = 0; m_price_scale = 0; m_price_decision = 0; m_price_error = ""; } //--- Revoke the saved tape and every backward or update authorization. m_public_valid = false; m_phase = 0; m_source = NULL; m_source_output = NULL; m_source_index = INVALID_HANDLE; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Function CNeuronScenarioDecoderOCL::OwnsBufferOrDeviceSlot | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { if(CNeuronBaseOCL::OwnsBufferOrDeviceSlot(candidate)) return(true); for(uint i = 0; i < 8; i++) if(BufferAliases(candidate, m_work[i])) return(true); for(uint i = 0; i < 4; i++) if(BufferAliases(candidate, m_loss_work[i])) return(true); if(CheckPointer(m_first) != POINTER_INVALID && m_first.OwnsBufferOrDeviceSlot(candidate)) return(true); if(CheckPointer(m_last) != POINTER_INVALID && m_last.OwnsBufferOrDeviceSlot(candidate)) return(true); return(false); } //+------------------------------------------------------------------+ //| Function CNeuronScenarioDecoderOCL::CanImportPreparedParameters | //+------------------------------------------------------------------+ bool CNeuronScenarioDecoderOCL::CanImportPreparedParameters(CNeuronScenarioDecoderOCL *donor) { if(!donor || donor == GetPointer(this) || m_failed || !m_configured || m_phase != 0 || m_update_count != 0 || m_generation != 0 || !ValidateOwned() || donor.OpenCL != OpenCL || donor.m_failed || !donor.m_configured || donor.m_phase != 0 || !donor.ValidateOwned()) return(false); ulong receiver = 1469598103934665603, source = 1469598103934665603; HashComposition(receiver); donor.HashComposition(source); if(receiver != source) return(false); CNeuronConvOCL *own[] = {m_first, m_last}; CNeuronConvOCL *other[] = {donor.m_first, donor.m_last}; for(uint i = 0; i < 2; i++) { if(own[i].NativeOptimizerStep() != 1 || !CheckParameters(own[i]) || !CheckParameters(other[i])) return(false); CBufferFloat *moments[] = {own[i].GetFirstMomentumConv(), own[i].GetSecondMomentumConv()}; for(uint j = 0; j < 2; j++) { if(!moments[j] || !moments[j].BufferRead()) return(false); for(int n = 0; n < moments[j].Total(); n++) if(moments[j].At(n) != 0.0f) return(false); } } return(true); } //+------------------------------------------------------------------+ //| Validates the native Forecast provider and its declared axes | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::ValidPolicyForecast(CNeuronBaseOCL *source) const { if(!source || source.GetOpenCL() != OpenCL || source.Type() != defNeuronScenarioForecast) return(false); CNeuronScenarioForecast *forecast = dynamic_cast(source); if(!forecast || forecast.Scenarios() != iScenarios || forecast.Variables() != iVariables || forecast.Horizon() != iHorizon || forecast.Dimension() != iDimension || !NNQValid(OpenCL, forecast.GetZ(), (ulong)iScenarios * iVariables * iHorizon * iDimension) || !NNQValid(OpenCL, forecast.GetLogU(), (ulong)iScenarios * iVariables * iHorizon) || !NNQValid(OpenCL, forecast.GetPi(), iScenarios)) return(false); return(true); } //+------------------------------------------------------------------+ //| Complete native Cross runtime, including nested Spike/MSRes/LSE. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::CaptureEvaluationState(SLayerReadState &state) { if(!bCritic || !CNeuronMSRes::CaptureEvaluationState(state)) return(false); CNeuronBaseOCL *children[] = {cQueryProjection.AsObject(), cQueryGradient.AsObject(), cWz.AsObject(), cWu.AsObject(), cTokenSum.AsObject(), cTokenPE.AsObject(), cTokensTranspose.AsObject(), cKV.AsObject(), cKVGradK.AsObject(), cKVGradV.AsObject(), cAttention.AsObject(), cAttentionTranspose.AsObject(), cHeadGate.AsObject(), cHeadGateTranspose.AsObject(), cOutputProjection.AsObject(), cQueryResidual.AsObject(), cMHAOutput.AsObject() }; for(int i = 0; i < ArraySize(children); i++) if(!children[i].CaptureEvaluationState(state)) return(false); return(true); } //+------------------------------------------------------------------+ //| Function CNeuronScenarioCrossAttention::RestoreEvaluationState | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::RestoreEvaluationState(SLayerReadState &state) { if(!bCritic || !CNeuronMSRes::RestoreEvaluationState(state)) return(false); CNeuronBaseOCL *children[] = {cQueryProjection.AsObject(), cQueryGradient.AsObject(), cWz.AsObject(), cWu.AsObject(), cTokenSum.AsObject(), cTokenPE.AsObject(), cTokensTranspose.AsObject(), cKV.AsObject(), cKVGradK.AsObject(), cKVGradV.AsObject(), cAttention.AsObject(), cAttentionTranspose.AsObject(), cHeadGate.AsObject(), cHeadGateTranspose.AsObject(), cOutputProjection.AsObject(), cQueryResidual.AsObject(), cMHAOutput.AsObject() }; for(int i = 0; i < ArraySize(children); i++) if(!children[i].RestoreEvaluationState(state)) return(false); return(true); } //+------------------------------------------------------------------+ //| Candidate queries share Market keys; no native state is | //| committed. | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttention::ForwardRowsReadOnly(CBufferFloat *source, const uint rows, CBufferFloat *z, CBufferFloat *log_u, CBufferFloat *pi, const int forecast_type, const SLayerReadState &state, SLayerRowsWorkspace &work, CBufferFloat *output, CBufferFloat *flags) { if((ulong)iScenarios * iVariables * iHorizon * iDimension > (ulong)UINT_MAX / sizeof(float) || (ulong)rows * iQueries * iScenarios * iDimension > (ulong)UINT_MAX / sizeof(float)) return(false); const uint steps = iVariables * iHorizon, tokens = iScenarios * steps, width = iDimension; if(!bCritic || iQueries != 5 || rows == 0 || state.Context() != OpenCL || work.Context() != OpenCL || state.State() == 0 || !state.Saved(Output) || forecast_type != defNeuronScenarioForecast || !NNQValid(OpenCL, source, (ulong)rows * 19) || !NNQValid(OpenCL, z, (ulong)tokens * width) || !NNQValid(OpenCL, log_u, tokens) || !NNQValid(OpenCL, pi, iScenarios) || z.Total() != (int)(tokens * width) || log_u.Total() != (int)tokens || pi.Total() != (int)iScenarios || !NNQValid(OpenCL, output, (ulong)rows * iQueries * width) || !NNQValid(OpenCL, flags, rows)) return(false); CBufferFloat *query = work.Take(rows * (uint)cQueryProjection.Neurons()); CBufferFloat *wz = work.Take(tokens * width), *wu = work.Take(tokens * width), *sum = work.Take(tokens * width); CBufferFloat *keys = work.Take(tokens * width), *attention = work.Take(rows * iQueries * iScenarios * width); CBufferFloat *lse = work.Take(rows * iQueries * iScenarios), *projection = work.Take(rows * iQueries * width); CBufferFloat *residual = work.Take(rows * iQueries * width), *combined = work.Take(rows * iQueries * width); if(!query || !wz || !wu || !sum || !keys || !attention || !lse || !projection || !residual || !combined) return(false); CBufferFloat *validate[] = {pi, pi, flags}; uint counts[] = {iScenarios}, one[] = {1}; float tolerance[] = {defFQF_PROBABILITY_TOLERANCE}; if(!NNQLaunch(OpenCL, def_k_DPCCQRowsValidate, validate, counts, tolerance, one) || !cQueryProjection.ForwardRowsReadOnly(source, rows, state, work, query, flags) || !cWz.ForwardRowsReadOnly(z, 1, state, work, wz, flags) || !cWu.ForwardRowsReadOnly(log_u, 1, state, work, wu, flags) || !NNQMap(OpenCL, wz, wu, sum, flags, 1, tokens * width, 0, 0, 0, 0, 1) || !cTokenPE.ForwardRowsReadOnly(sum, 1, state, work, keys, flags)) return(false); CBufferFloat *buffers[] = {query, keys, pi, attention, lse, flags}; uint values[] = {iQueries, iScenarios, steps, width}, global[] = {iQueries*iScenarios * width, rows}; float reals[]; if(!NNQLaunch(OpenCL, def_k_DPCCQRowsAttention, buffers, values, reals, global) || !cOutputProjection.ForwardRowsReadOnly(attention, rows, state, work, projection, flags) || !cQueryResidual.ForwardRowsReadOnly(query, rows, state, work, residual, flags) || !NNQMap(OpenCL, projection, residual, combined, flags, rows, iQueries * width, iQueries * width, iQueries * width, 0, 0, 1)) return(false); return(CNeuronMSRes::ForwardRowsReadOnly(combined, rows, state, work, output, flags)); } //+------------------------------------------------------------------+ //| Native Q compatibility for the phase-gated price forecast | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint scenarios, uint horizon, uint latent, bool critic, ENUM_OPTIMIZATION optimization_type, uint batch, uint variables, uint stack_size, uint top_k) { if(!critic) ReturnFalseEx("ForecastPrice Cross requires Critic queries"); return(CNeuronScenarioCrossAttention::Init(numOutputs, myIndex, open_cl, scenarios, horizon, latent, true, optimization_type, batch, variables, stack_size, top_k)); } //+------------------------------------------------------------------+ //| Validates the frozen Price provider without changing its Type | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::ValidPolicyForecast(CNeuronBaseOCL *source) const { if(!bCritic || !source || source.GetOpenCL() != OpenCL) return(false); if(source.Type() == defNeuronScenarioForecast) return(CNeuronScenarioCrossAttention::ValidPolicyForecast(source)); if(source.Type() != defNeuronScenarioForecastPriceOCL) return(false); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(source); if(!forecast || forecast.ABI() != defScenarioForecastPriceABI || forecast.TrainMode() || forecast.CodebookUpdate() || forecast.Scenarios() != iScenarios || forecast.Variables() != iVariables || forecast.Horizon() != iHorizon || forecast.Dimension() != iDimension || forecast.Neurons() != (int)((ulong)iScenarios * iVariables * iHorizon * iDimension) || !forecast.GetLogU() || forecast.GetLogU().Total() != (int)((ulong)iScenarios * iVariables * iHorizon) || !forecast.GetPi() || forecast.GetPi().Total() != (int)iScenarios || !NNQValid(OpenCL, forecast.GetZ(), (ulong)iScenarios * iVariables * iHorizon * iDimension) || !NNQValid(OpenCL, forecast.GetLogU(), (ulong)iScenarios * iVariables * iHorizon) || !NNQValid(OpenCL, forecast.GetPi(), iScenarios)) return(false); return(true); } //+------------------------------------------------------------------+ //| Describes the native Q query and scenario geometry | //+------------------------------------------------------------------+ CLayerDescription *CNeuronScenarioCrossAttentionPriceOCL::GetLayerInfo(void) { CLayerDescription *result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return(NULL); result.count = iScenarios; result.window = iDimension; result.window_out = iHorizon; result.variables = 5; result.layers = iVariables; result.step = iStackSize; result.probability = (float)iTopK; return(result); } //+------------------------------------------------------------------+ //| Saves a native Cross payload with its native type header | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::SaveNative(const int file_handle) { if(file_handle == INVALID_HANDLE || !bCritic) return(false); const ulong start = FileTell(file_handle); if(!CNeuronScenarioCrossAttention::Save(file_handle)) return(false); const ulong end = FileTell(file_handle); if(!FileSeek(file_handle, (long)start, SEEK_SET)) return(false); const bool written = (FileWriteInteger(file_handle, defNeuronScenarioCrossAttention) == INT_VALUE); const bool restored = FileSeek(file_handle, (long)end, SEEK_SET); return(written && restored); } //+------------------------------------------------------------------+ //| Saves the adapter ABI and the unchanged native Q payload | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE || !bCritic) return(false); if(FileWriteInteger(file_handle, Type()) != INT_VALUE) return(false); if(FileWriteInteger(file_handle, defScenarioCrossAttentionPriceABI) != INT_VALUE) return(false); return(SaveNative(file_handle)); } //+------------------------------------------------------------------+ //| Loads an explicit native layer into the Q compatibility type | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::LoadNative(const int file_handle, COpenCLMy *open_cl) { if(file_handle == INVALID_HANDLE || CheckPointer(open_cl) == POINTER_INVALID || iVariables != 0) return(false); const ulong position = FileTell(file_handle); const ulong size = FileSize(file_handle); if(position > size || size - position < sizeof(uint) || FileReadInteger(file_handle) != defNeuronScenarioCrossAttention) return(false); AttachOpenCL(open_cl); if(!CNeuronScenarioCrossAttention::Load(file_handle) || !bCritic) return(false); return(ValidateScenarioOwnedTreeChecked()); } //+------------------------------------------------------------------+ //| Restores the adapter after the outer type was consumed | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) return(false); const ulong position = FileTell(file_handle); const ulong size = FileSize(file_handle); if(position > size || size - position < sizeof(uint) || FileReadInteger(file_handle) != defScenarioCrossAttentionPriceABI) return(false); return(LoadNative(file_handle, OpenCL)); } //+------------------------------------------------------------------+ //| Loads the exact new type in the supplied OpenCL context | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::Load(const int file_handle, COpenCLMy *open_cl) { if(file_handle == INVALID_HANDLE || CheckPointer(open_cl) == POINTER_INVALID) return(false); const ulong position = FileTell(file_handle); const ulong size = FileSize(file_handle); if(position > size || size - position < sizeof(uint) || FileReadInteger(file_handle) != Type()) return(false); AttachOpenCL(open_cl); return(Load(file_handle)); } //+------------------------------------------------------------------+ //| Reuses the native Q ownership tree for checked transfer | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::SupportsOpenCLChecked(void) const { return(Type() == defNeuronScenarioCrossAttentionPriceOCL && bCritic && ValidateScenarioOwnedTreeChecked()); } //+------------------------------------------------------------------+ //| Native Q feedForwardSecond with Forecast validation | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::feedForwardSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!OpenCL || !NeuronOCL || !Forecast || !ValidPolicyForecast(Forecast) || NeuronOCL.Neurons() != int(bCritic ? 19 : 13)) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(forecast.Variables() != iVariables || forecast.Scenarios() != iScenarios || forecast.Horizon() != iHorizon || forecast.Dimension() != iDimension || Forecast.Neurons() != int(iScenarios * iVariables * iHorizon * iDimension) || !forecast.GetLogU() || forecast.GetLogU().Total() != int(iScenarios * iVariables * iHorizon) || !forecast.GetPi() || forecast.GetPi().Total() != int(iScenarios)) ReturnFalse; if(!cQueryProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cWz.FeedForward(Forecast)) ReturnFalse; if(!cWu.FeedForwardBuffer(forecast.GetLogU())) ReturnFalse; if(!SumAndNormalize(cWz.getOutput(), cWu.getOutput(), cTokenSum.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cTokenPE.FeedForward(cTokenSum.AsObject())) ReturnFalse; if(!cTokensTranspose.FeedForward(cTokenPE.AsObject())) ReturnFalse; if(!Concat(cTokensTranspose.getOutput(), cTokensTranspose.getOutput(), cKV.getOutput(), iScenarios * iDimension, iScenarios * iDimension, iVariables * iHorizon)) ReturnFalse; if(!cAttention.FeedForward(cQueryProjection.AsObject(), cKV.getOutput())) ReturnFalse; if(!cAttentionTranspose.FeedForward(cAttention.AsObject())) ReturnFalse; //--- Critic is stateless. Actor first derives a shared history context from //--- all ungated K scenario contexts, then refines each scenario separately. if(!bCritic) { if(!cHistoryStack.FeedForward(GetPointer(this))) ReturnFalse; if(!cHistoryTTM.FeedForward(cAttentionTranspose.AsObject(), cHistoryStack.getOutput())) ReturnFalse; if(!cHistoryTokenProjection.FeedForward(cTokenPE.AsObject())) ReturnFalse; if(!cHistoryMarket.FeedForward(cHistoryStack.AsObject(), cHistoryTokenProjection.getOutput())) ReturnFalse; if(!cHistoryContext.FeedForward(cHistoryTTM.AsObject(), cHistoryMarket.getOutput())) ReturnFalse; if(!cScenarioHistory.FeedForward(cAttentionTranspose.AsObject(), cHistoryContext.getOutput())) ReturnFalse; if(!ScalarToVector(forecast.GetPi(), cScenarioHistory.getOutput(), cHeadGate.getOutput(), iQueries * iDimension)) ReturnFalse; } else if(!ScalarToVector(forecast.GetPi(), cAttentionTranspose.getOutput(), cHeadGate.getOutput(), iQueries * iDimension)) ReturnFalse; if(!cHeadGateTranspose.FeedForward(cHeadGate.AsObject())) ReturnFalse; if(!cOutputProjection.FeedForward(cHeadGateTranspose.AsObject())) ReturnFalse; if(!cQueryResidual.FeedForward(cQueryProjection.AsObject())) ReturnFalse; if(!SumAndNormalize(cOutputProjection.getOutput(), cQueryResidual.getOutput(), cMHAOutput.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cMHAOutput.AsObject())) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Native Q calcInputGradientsSecond with Forecast validation | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::calcInputGradientsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!OpenCL || !NeuronOCL || !Forecast || !ValidPolicyForecast(Forecast)) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(!forecast.GetPi()) ReturnFalse; if(!CNeuronMSRes::calcInputGradients(cMHAOutput.AsObject())) ReturnFalse; if(!DeActivation(cOutputProjection.getOutput(), cOutputProjection.getGradient(), cMHAOutput.getGradient(), cOutputProjection.Activation())) ReturnFalse; if(!DeActivation(cQueryResidual.getOutput(), cQueryResidual.getGradient(), cMHAOutput.getGradient(), cQueryResidual.Activation())) ReturnFalse; if(!cHeadGateTranspose.CalcHiddenGradients(cOutputProjection.AsObject())) ReturnFalse; if(!cHeadGate.CalcHiddenGradients(cHeadGateTranspose.AsObject())) ReturnFalse; if(!bCritic) { if(!ScalarToVector(forecast.GetPi(), cHeadGate.getGradient(), cScenarioHistory.getGradient(), iQueries * iDimension)) ReturnFalse; if(!cAttentionTranspose.CalcHiddenGradients(cScenarioHistory.AsObject(), cHistoryContext.getOutput(), cHistoryContext.getGradient(), None)) ReturnFalse; if(!cHistoryTTM.CalcHiddenGradients(cHistoryContext.AsObject(), cHistoryMarket.getOutput(), cHistoryMarket.getGradient(), None)) ReturnFalse; if(!cAttentionTranspose.CalcHiddenGradients(cHistoryTTM.AsObject(), cHistoryStack.getOutput(), NULL, None)) ReturnFalse; if(!cHistoryStack.CalcHiddenGradients(cHistoryMarket.AsObject(), cHistoryTokenProjection.getOutput(), cHistoryTokenProjection.getGradient(), None)) ReturnFalse; if(!cTokenPE.CalcHiddenGradients(cHistoryTokenProjection.AsObject())) ReturnFalse; } else if(!ScalarToVector(forecast.GetPi(), cHeadGate.getGradient(), cAttentionTranspose.getGradient(), iQueries * iDimension)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cAttentionTranspose.AsObject())) ReturnFalse; if(!cQueryGradient.CalcHiddenGradients(cQueryResidual.AsObject())) ReturnFalse; if(!cQueryProjection.CalcHiddenGradients(cAttention.AsObject(), cKV.getOutput(), cKV.getGradient(), None)) ReturnFalse; if(!SumAndNormalize(cQueryProjection.getGradient(), cQueryGradient.getGradient(), cQueryProjection.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!DeConcat(cKVGradK.getGradient(), cKVGradV.getGradient(), cKV.getGradient(), iScenarios * iDimension, iScenarios * iDimension, iVariables * iHorizon)) ReturnFalse; if(!SumAndNormalize(cKVGradK.getGradient(), cKVGradV.getGradient(), cTokensTranspose.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cTokenPE.CalcHiddenGradients(cTokensTranspose.AsObject())) ReturnFalse; if(!cTokenSum.CalcHiddenGradients(cTokenPE.AsObject())) ReturnFalse; //--- Forecast is the read-only second stream: only Cross trainables receive gradients. if(!Concat(cTokenSum.getGradient(), cTokenSum.getGradient(), cWz.getGradient(), iScenarios * iVariables * iHorizon * iDimension, 0, 1)) ReturnFalse; if(!Concat(cTokenSum.getGradient(), cTokenSum.getGradient(), cWu.getGradient(), iScenarios * iVariables * iHorizon * iDimension, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQueryProjection.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) return(DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())); return(true); } //+------------------------------------------------------------------+ //| Native Q updateInputWeightsSecond with Forecast validation | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::updateInputWeightsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) { if(!NeuronOCL || !Forecast || !ValidPolicyForecast(Forecast)) ReturnFalse; CNeuronScenarioForecast *forecast = Forecast; if(!forecast.GetLogU()) ReturnFalse; if(!cQueryProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cWz.UpdateInputWeights(Forecast)) ReturnFalse; if(!cWu.UpdateInputWeightsBuffer(forecast.GetLogU())) ReturnFalse; if(!cTokenPE.UpdateInputWeights(cTokenSum.AsObject())) ReturnFalse; if(!cOutputProjection.UpdateInputWeights(cHeadGateTranspose.AsObject())) ReturnFalse; if(!cQueryResidual.UpdateInputWeights(cQueryProjection.AsObject())) ReturnFalse; if(!bCritic) { if(!cHistoryTokenProjection.UpdateInputWeights(cTokenPE.AsObject())) ReturnFalse; if(!cHistoryMarket.UpdateInputWeights(cHistoryStack.AsObject(), cHistoryTokenProjection.getOutput())) ReturnFalse; if(!cHistoryContext.UpdateInputWeights(cHistoryTTM.AsObject(), cHistoryMarket.getOutput())) ReturnFalse; if(!cScenarioHistory.UpdateInputWeights(cAttentionTranspose.AsObject(), cHistoryContext.getOutput())) ReturnFalse; } return(true); } //+------------------------------------------------------------------+ //| Evaluates compatible Q rows using native weights and scratch | //+------------------------------------------------------------------+ bool CNeuronScenarioCrossAttentionPriceOCL::ForwardRowsReadOnly(CBufferFloat *source, const uint rows, CBufferFloat *z, CBufferFloat *log_u, CBufferFloat *pi, const int forecast_type, const SLayerReadState &state, SLayerRowsWorkspace &work, CBufferFloat *output, CBufferFloat *flags) { if((ulong)iScenarios * iVariables * iHorizon * iDimension > (ulong)UINT_MAX / sizeof(float) || (ulong)rows * iQueries * iScenarios * iDimension > (ulong)UINT_MAX / sizeof(float)) return(false); const uint steps = iVariables * iHorizon, tokens = iScenarios * steps, width = iDimension; if(!bCritic || iQueries != 5 || rows == 0 || state.Context() != OpenCL || work.Context() != OpenCL || state.State() == 0 || !state.Saved(Output) || (forecast_type != defNeuronScenarioForecast && forecast_type != defNeuronScenarioForecastPriceOCL) || !NNQValid(OpenCL, source, (ulong)rows * 19) || !NNQValid(OpenCL, z, (ulong)tokens * width) || !NNQValid(OpenCL, log_u, tokens) || !NNQValid(OpenCL, pi, iScenarios) || z.Total() != (int)(tokens * width) || log_u.Total() != (int)tokens || pi.Total() != (int)iScenarios || !NNQValid(OpenCL, output, (ulong)rows * iQueries * width) || !NNQValid(OpenCL, flags, rows)) return(false); CBufferFloat *query = work.Take(rows * (uint)cQueryProjection.Neurons()); CBufferFloat *wz = work.Take(tokens * width), *wu = work.Take(tokens * width), *sum = work.Take(tokens * width); CBufferFloat *keys = work.Take(tokens * width), *attention = work.Take(rows * iQueries * iScenarios * width); CBufferFloat *lse = work.Take(rows * iQueries * iScenarios), *projection = work.Take(rows * iQueries * width); CBufferFloat *residual = work.Take(rows * iQueries * width), *combined = work.Take(rows * iQueries * width); if(!query || !wz || !wu || !sum || !keys || !attention || !lse || !projection || !residual || !combined) return(false); CBufferFloat *validate[] = {pi, pi, flags}; uint counts[] = {iScenarios}, one[] = {1}; float tolerance[] = {defFQF_PROBABILITY_TOLERANCE}; if(!NNQLaunch(OpenCL, def_k_DPCCQRowsValidate, validate, counts, tolerance, one) || !cQueryProjection.ForwardRowsReadOnly(source, rows, state, work, query, flags) || !cWz.ForwardRowsReadOnly(z, 1, state, work, wz, flags) || !cWu.ForwardRowsReadOnly(log_u, 1, state, work, wu, flags) || !NNQMap(OpenCL, wz, wu, sum, flags, 1, tokens * width, 0, 0, 0, 0, 1) || !cTokenPE.ForwardRowsReadOnly(sum, 1, state, work, keys, flags)) return(false); CBufferFloat *buffers[] = {query, keys, pi, attention, lse, flags}; uint values[] = {iQueries, iScenarios, steps, width}, global[] = {iQueries*iScenarios * width, rows}; float reals[]; if(!NNQLaunch(OpenCL, def_k_DPCCQRowsAttention, buffers, values, reals, global) || !cOutputProjection.ForwardRowsReadOnly(attention, rows, state, work, projection, flags) || !cQueryResidual.ForwardRowsReadOnly(query, rows, state, work, residual, flags) || !NNQMap(OpenCL, projection, residual, combined, flags, rows, iQueries * width, iQueries * width, iQueries * width, 0, 0, 1)) return(false); return(CNeuronMSRes::ForwardRowsReadOnly(combined, rows, state, work, output, flags)); } //+------------------------------------------------------------------+