//+------------------------------------------------------------------+ //| NeuroNet.cl| //| OpenCL kernels library for NeuroNet.| //| Copyright 2024, DNG| //| https://www.mql5.com/ru/users/dng| //+------------------------------------------------------------------+ /// \file /// \brief NeuroNet.cl /// Library consist OpenCL kernels /// \author DNG /// \copyright Copyright 2019, DNG //--- //--- by default some GPU doesn't support floats //--- cl_khr_fp64 directive is used to enable work with floats // #pragma OPENCL EXTENSION cl_khr_fp64 : enable #define l1 1.0e-4f #define l2 1.0e-4f #define MAX_GRAD 1.0e-2f #define LOCAL_ARRAY_SIZE 64 #define SCENARIO_LOCAL_CAP 256 #define MAX_VALUE 3.4e37f #define MIN_VALUE -MAX_VALUE #define RAG_MAX_VALUE 3.402823466e+38f #define RAG_INVALID_SCORE (-RAG_MAX_VALUE) #define D2SKILL_FREE 0 #define D2SKILL_CANDIDATE 1 #define D2SKILL_PROTECTED 2 #define D2SKILL_ACTIVE 3 #define D2SKILL_INACTIVE 4 //--- Action6 direct-lot limit shared by rollout validation and projection. #define DPCC_ACTION_VOLUME_MAX 1.0f #define DPCC_ACTION_PROBABILITY_EPS 1.0e-6f //--- Activation Functions #define ActFunc_None -1 #define ActFunc_TANH 0 #define ActFunc_SIGMOID 1 #define ActFunc_LReLU 2 #define ActFunc_SoftPlus 3 #define ActFunc_GELU 4 #define ActFunc_MinusSoftPlus 5 #define ActFunc_ELU 6 //--- Spectral Risk Measure risk types (FQF_SRMForward) #define SRM_MEAN 0 #define SRM_CVAR 1 #define SRM_MEAN_CVAR 2 #define SRM_EXPONENTIAL 3 #define SRM_DUAL_POWER 4 #define SRM_PARAMETER_EPS 1.0e-6f //--- #define BarrierLoc barrier(CLK_LOCAL_MEM_FENCE); //+------------------------------------------------------------------+ //| Sanitizes the input, replacing NaN or infinite values with the | //| fallback value. | //+------------------------------------------------------------------+ inline float IsNaNOrInf(const float value, const float def_value) { if(isnan(value) || isinf(value)) return def_value; return value; } //+------------------------------------------------------------------+ //| Sanitizes the input, replacing NaN or infinite values with the | //| fallback value. | //+------------------------------------------------------------------+ inline float2 IsNaNOrInf2(const float2 value, const float2 def_value) { if(isnan(value.x) || isinf(value.x) || isnan(value.y) || isinf(value.y)) return def_value; return value; } //+------------------------------------------------------------------+ //| Sanitizes the input, replacing NaN or infinite values with the | //| fallback value. | //+------------------------------------------------------------------+ inline float4 IsNaNOrInf4(const float4 value, const float def_value) { float4 result; result.s0 = IsNaNOrInf(value.s0, def_value); result.s1 = IsNaNOrInf(value.s1, def_value); result.s2 = IsNaNOrInf(value.s2, def_value); result.s3 = IsNaNOrInf(value.s3, def_value); return result; } //+------------------------------------------------------------------+ //| Applies the selected activation function to a scalar value. | //+------------------------------------------------------------------+ float fActivation(const float value, const int function) { float result = IsNaNOrInf(value, 0); switch(function) { case ActFunc_TANH: result = tanh(clamp(result, -20.0f, 20.0f)); break; case ActFunc_SIGMOID: //Sigmoid result = 1 / (1 + exp(clamp(-result, -20.0f, 20.0f))); break; case ActFunc_LReLU: //LReLU if(result < 0) result *= 0.01f; break; case ActFunc_SoftPlus: //SoftPlus result = (result >= 20.0f ? result : IsNaNOrInf(log(1 + exp(result)), 0)); break; case ActFunc_GELU: //GELU result = result / (1 + exp(clamp(-1.702f * result, -20.0f, 20.0f))); break; case ActFunc_MinusSoftPlus: // -SoftPlus result = -fActivation(result, 3); break; case ActFunc_ELU: //ELU if(result < 0) result = IsNaNOrInf(exp(result), 0) - 1; break; default: break; } //--- return IsNaNOrInf(result, 0); } //+------------------------------------------------------------------+ //| Applies the derivative of the activation function to an output | //| gradient. | //+------------------------------------------------------------------+ float Deactivation(const float grad, const float inp_value, const int function) { float result = IsNaNOrInf(grad, 0); //--- if(isnan(inp_value) || isinf(inp_value)) result = 0; else switch(function) { case ActFunc_TANH: //TANH result = clamp(grad + inp_value, -1.0f, 1.0f) - inp_value; result *= 1.0f - inp_value * inp_value; break; case ActFunc_SIGMOID: //Sigmoid result = clamp(grad + inp_value, 0.0f, 1.0f) - inp_value; result *= inp_value * (1.0f - inp_value); break; case ActFunc_LReLU: //LReLU if(inp_value < 0) result *= 0.01f; break; case ActFunc_SoftPlus: //SoftPlus result *= (1.0f - exp(-inp_value)); break; case ActFunc_GELU: //GELU if(inp_value < 0.9f) result *= fActivation(5 * inp_value, 1); break; case ActFunc_MinusSoftPlus: // -SoftPlus result = Deactivation(-result, -inp_value, ActFunc_SoftPlus); break; case ActFunc_ELU: //ELU if(inp_value < 0) result *= inp_value + 1; break; default: break; } //--- return clamp(IsNaNOrInf(result, 0), -MAX_GRAD, MAX_GRAD); } //+------------------------------------------------------------------+ //| Maps the row/column indices of a tensor to its flat buffer | //| offset. | //+------------------------------------------------------------------+ inline int RCtoFlat(const int row, const int col, const int total_rows, const int total_cols, const int variable) { return (variable * total_rows + row) * total_cols + col; } //+------------------------------------------------------------------+ //| Implements the vget4 helper on the GPU. | //+------------------------------------------------------------------+ inline float vget4(float4 v, int idx) { float val = 0.0f; switch(idx) { case 0: val = v.s0; break; case 1: val = v.s1; break; case 2: val = v.s2; break; case 3: val = v.s3; break; } return val; } //+------------------------------------------------------------------+ //| Performs a local reduction (local max) over the work-group items.| //+------------------------------------------------------------------+ float LocalMax(const float value, const int loc, __local float* Temp) { const size_t id = get_local_id(loc); const size_t total = get_local_size(loc); //--- const uint ls = min((uint)total, (uint)LOCAL_ARRAY_SIZE); float val = IsNaNOrInf(value, MIN_VALUE); //--- Look Max if(id < ls) Temp[id] = val; BarrierLoc for(int d = ls; d < total; d += ls) { if(id >= d && id < (d + ls) && (Temp[id - d] < val)) Temp[id - d] = val; BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(id < count && (id + count) < ls && Temp[id] < Temp[id + count]) Temp[id] = Temp[id + count]; BarrierLoc } while(count > 1); //--- return Temp[0]; } //+------------------------------------------------------------------+ //| Performs a local reduction (local min) over the work-group items.| //+------------------------------------------------------------------+ float LocalMin(const float value, const int loc, __local float* Temp) { const size_t id = get_local_id(loc); const size_t total = get_local_size(loc); //--- const uint ls = min((uint)total, (uint)LOCAL_ARRAY_SIZE); float val = IsNaNOrInf(value, MAX_VALUE); //--- Look Min if(id < ls) Temp[id] = val; BarrierLoc for(int d = ls; d < total; d += ls) { if(id >= d && id < (d + ls) && (Temp[id - d] > val)) Temp[id - d] = val; BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(id < count && (id + count) < ls && Temp[id] > Temp[id + count]) Temp[id] = Temp[id + count]; BarrierLoc } while(count > 1); //--- return Temp[0]; } //+------------------------------------------------------------------+ //| Performs a local reduction (local sum) over the work-group items.| //+------------------------------------------------------------------+ float LocalSum(const float value, const int loc, __local float* Temp) { const size_t id = get_local_id(loc); const size_t total = get_local_size(loc); //--- if(total <= 1) return IsNaNOrInf(value, 0.0f); //--- const uint ls = min((uint)total, (uint)LOCAL_ARRAY_SIZE); //--- Sum float result = IsNaNOrInf(value, 0); if(id < ls) Temp[id] = result; BarrierLoc for(int d = ls; d < total; d += ls) { if(id >= d && id < (d + ls)) Temp[id - d] = Temp[id - d] + result; BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(id < count && (id + count) < ls) { Temp[id] += Temp[id + count]; Temp[id + count] = 0; } BarrierLoc } while(count > 1); result = IsNaNOrInf(Temp[0], 0); //--- return result; } //+------------------------------------------------------------------+ //| Performs a local reduction (local sum4) over the work-group | //| items. | //+------------------------------------------------------------------+ float4 LocalSum4(const float4 value, const int loc, __local float4* Temp) { const size_t id = get_local_id(loc); const size_t total = get_local_size(loc); //--- if(total <= 1) return IsNaNOrInf4(value, 0.0f); //--- const uint ls = min((uint)total, (uint)LOCAL_ARRAY_SIZE); //--- float4 result = IsNaNOrInf4(value, 0.0f); if(id < ls) Temp[id] = result; BarrierLoc for(int d = ls; d < total; d += ls) { if(id >= d && id < (d + ls)) Temp[id - d] = Temp[id - d] + result; BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(id < count && (id + count) < ls) { Temp[id] += Temp[id + count]; Temp[id + count] = (float4)0.0f; } BarrierLoc } while(count > 1); //--- result = IsNaNOrInf4(Temp[0], 0.0f); //--- return result; } //+------------------------------------------------------------------+ //| Computes the numerically stable softmax over the local work-group| //| items. | //+------------------------------------------------------------------+ float LocalSoftMax(const float value, const int loc, __local float* Temp) { //--- Look Max float max = LocalMax(value, loc, Temp); if(max == MIN_VALUE) return 0.0f; //--- SoftMax float result = (value == MIN_VALUE ? 0.0f : IsNaNOrInf(exp(value - max), 0.0f)); const float sum = LocalSum(result, loc, Temp); if(sum == 0.0f) result = 0; else result = IsNaNOrInf(result / sum, 0.0f); //--- return result; } //+------------------------------------------------------------------+ //| Computes the local softmax gradient contribution for one item. | //+------------------------------------------------------------------+ float LocalSoftMaxGrad(const float value, const float grad, const int loc, __local float* Temp) { const float y = IsNaNOrInf(value, 0.0f); const float g = IsNaNOrInf(grad, 0.0f); const float s = LocalSum(y * g, loc, Temp); //--- d_i = y_i * (g_i - sum_j(y_j * g_j)) return IsNaNOrInf(y * (g - s), 0.0f); } //+------------------------------------------------------------------+ ///\ingroup neuron_base_ff Feed forward process kernel /// Describes the forward path process for the Neuron Base (#CNeuronBaseOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void FeedForward(__global const float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number ///< of neurons in layer and n - number of outputs ///< (neurons in next layer) __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_o, ///<[out] Output tensor const int inputs, ///< Number of inputs const int activation, ///< Activation type (#ENUM_ACTIVATION) __global float *numeric_flags, ///<[out] One status per output const int flags_enabled ) { const int i = get_global_id(0); const int total_out = get_global_size(0); const int loc = get_local_id(1); const int total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; __local float FlagTemp[LOCAL_ARRAY_SIZE]; //--- float sum = 0; float inp; float invalid = 0; int shift = RCtoFlat(i, 0, total_out, (inputs + 1), 0); for(int k = loc; k < inputs; k += total_loc) { if(flags_enabled) { const float raw_inp = matrix_i[k]; const float raw_weight = matrix_w[shift + k]; const float raw_product = raw_inp * raw_weight; invalid += (isnan(raw_inp) || isinf(raw_inp) || isnan(raw_weight) || isinf(raw_weight) || isnan(raw_product) || isinf(raw_product)); } inp = IsNaNOrInf(matrix_i[k], 0.0f); if(inp == 0.0f) continue; sum += IsNaNOrInf(inp * matrix_w[shift + k], 0.0f); } if(loc == 0) { if(flags_enabled) { const float raw_bias = matrix_w[shift + inputs]; invalid += isnan(raw_bias) || isinf(raw_bias); } sum += IsNaNOrInf(matrix_w[shift + inputs], 0.0f); } if(flags_enabled) invalid += isnan(sum) || isinf(sum); if(total_loc > 1) sum = LocalSum(sum, 1, Temp); if(flags_enabled) { if(total_loc > 1) invalid += isnan(Temp[0]) || isinf(Temp[0]); invalid = LocalSum(invalid, 1, FlagTemp); } //--- if(loc == 0) { if(flags_enabled) invalid += isnan(sum) || isinf(sum); //--- Preserve the activation result for the numerical flag before storing it. const float activation_value = fActivation(sum, activation); if(flags_enabled) { invalid += isnan(activation_value) || isinf(activation_value); numeric_flags[i] = (invalid > 0.0f ? 1.0f : 0.0f); } matrix_o[i] = activation_value; } } //+------------------------------------------------------------------+ ///\ingroup neuron_base_gr Neuron Base Output Gradients Calculation kernel /// Describes the process of output gradients calculation for the Neuron Base /// (#CNeuronBaseOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void CalcOutputGradient(__global float *matrix_t, ///<[in] Target tensor __global float *matrix_o, ///<[in] Output tensor __global float *matrix_ig, ///<[out] Tensor of gradients int activation, ///< Activation type (#ENUM_ACTIVATION) float error) { int i = get_global_id(0); float out = matrix_o[i]; float temp = 0; if(!isnan(out) && !isinf(out)) temp = Deactivation(matrix_t[i] - out, out, activation); matrix_ig[i] = temp; } //+------------------------------------------------------------------+ ///\ingroup neuron_base_gr Neuron Base Hidden Gradients Calculation kernel /// Describes the process of hidden gradients calculation for the Neuron Base /// (#CNeuronBaseOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void CalcHiddenGradient(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number ///< of neurons in previous layer and n - number ///< of neurons in current layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_o, ///<[in] Previous layer Output tensor __global float *matrix_ig, ///<[out] Tensor of gradients at previous layer int outputs, ///< Number of outputs int activation ///< Activation type (#ENUM_ACTIVATION) ) { const int i = get_global_id(0); const int inputs = get_global_size(0); const int loc = get_local_id(1); const int total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- float sum = 0; float out = matrix_o[i]; float4 grad, weight; for(int k = 4 * loc; k < outputs; k += 4 * total_loc) { switch(outputs - k) { case 1: weight = (float4)(matrix_w[k * (inputs + 1) + i], 0, 0, 0); grad = (float4)(matrix_g[k], 0, 0, 0); break; case 2: grad = (float4)(matrix_g[k], matrix_g[k + 1], 0, 0); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], 0, 0); break; case 3: grad = (float4)(matrix_g[k], matrix_g[k + 1], matrix_g[k + 2], 0); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], matrix_w[(k + 2) * (inputs + 1) + i], 0); break; default: grad = (float4)(matrix_g[k], matrix_g[k + 1], matrix_g[k + 2], matrix_g[k + 3]); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], matrix_w[(k + 2) * (inputs + 1) + i], matrix_w[(k + 3) * (inputs + 1) + i]); break; } //--- weight = IsNaNOrInf4(weight, 0); grad = IsNaNOrInf4(grad, 0); //--- sum += dot(grad, weight); } if(total_loc > 1) sum = LocalSum(sum, 1, Temp); //--- matrix_ig[i] = Deactivation(sum, out, activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_base_opt Neuron Base SGD Updating Weights Calculation kernel /// Describes the process of SGD optimization weights for the Neuron Base /// (#CNeuronBaseOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void UpdateWeightsMomentum(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_dw, ///<[in,out] Matrix of delta weights in last correction int inputs, ///< Number of inputs float learning_rates, ///< Learning rates float momentum ///< Momentum multiplier ) { int i = get_global_id(0); int j = get_global_id(1); int wi = i * (inputs + 1) + j; float grad = clamp(matrix_g[i], -MAX_GRAD, MAX_GRAD); float delta = IsNaNOrInf(learning_rates * grad * (j < inputs ? matrix_i[j] : 1), 0) + IsNaNOrInf(momentum * matrix_dw[wi], 0); matrix_dw[wi] = delta; if(fabs(delta) > 0) matrix_w[wi] = IsNaNOrInf(matrix_w[wi] + delta, 0); } //+------------------------------------------------------------------+ ///\ingroup neuron_base_opt Neuron Base Adam Updating Weights Calculation /// kernel /// Describes the process of Adam optimization weights for the Neuron Base /// (#CNeuronBaseOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void UpdateWeightsAdam(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int i = get_global_id(0); const int j = get_global_id(1); const int wi = i * (inputs + 1) + j; float m, v, weight, inp; inp = IsNaNOrInf((j == inputs ? 1.0f : matrix_i[j]), 0); weight = IsNaNOrInf(matrix_w[wi], 0); m = IsNaNOrInf(matrix_m[wi], 0); v = IsNaNOrInf(matrix_v[wi], 0); //--- float g = clamp(IsNaNOrInf(matrix_g[i] * inp, 0), -MAX_GRAD, MAX_GRAD); float mt = IsNaNOrInf(b1 * m + (1 - b1) * g, 0); float vt = IsNaNOrInf(b2 * v + (1 - b2) * (g * g), 0); float delta = IsNaNOrInf(l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)), 0); if(fabs(delta) > 0) matrix_w[wi] = IsNaNOrInf(matrix_w[wi] + delta, 0); matrix_m[wi] = mt; matrix_v[wi] = vt; } //+------------------------------------------------------------------+ ///\ingroup neuron_base_opt Neuron Base Least Squares Updating Weights /// Calculation kernel /// Describes the process of Least Squares optimization weights for the Neuron /// Base (#CNeuronBaseOCL). //\details Detailed description on the link. //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights ls step using| //| the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsLS(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_xg, ///<[in,out] Matrix of summ x*g __global float *matrix_xx, ///<[in,out] Matrix of summ x*x const int inputs, ///< Number of inputs const float l, ///< Learning rates const int update ///< Update flag ) { const int i = get_global_id(0); const int j = get_global_id(1); const int wi = i * (inputs + 1) + j * 4; float4 xg, xx, weight, inp; switch(inputs + 1 - j * 4) { case 0: inp = (float4)(1, 0, 0, 0); weight = (float4)(matrix_w[wi], 0, 0, 0); break; case 1: inp = (float4)(matrix_i[j * 4], 1, 0, 0); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], 0, 0); break; case 2: inp = (float4)(matrix_i[j * 4], matrix_i[j * 4 + 1], 1, 0); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], 0); break; case 3: inp = (float4)(matrix_i[j * 4], matrix_i[j * 4 + 1], matrix_i[j * 4 + 2], 1); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], matrix_w[wi + 3]); break; default: inp = (float4)(matrix_i[j * 4], matrix_i[j * 4 + 1], matrix_i[j * 4 + 2], matrix_i[j * 4 + 3]); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], matrix_w[wi + 3]); break; } xg = (float4)(matrix_g[i]) * inp; xx = inp * inp; switch(min(inputs + 1 - j * 4, 3)) { case 3: if(update) { matrix_w[wi + 3] = matrix_w[wi + 3] + l * (matrix_xg[wi + 3] + xg.s3) / (matrix_xx[wi + 3] + xx.s3 + 1.0e-37f); matrix_xg[wi + 3] = 0; matrix_xx[wi + 3] = 0; } else { matrix_xg[wi + 3] += xg.s3; matrix_xx[wi + 3] += xx.s3; } case 2: if(update) { matrix_w[wi + 2] = matrix_w[wi + 2] + l * (matrix_xg[wi + 2] + xg.s2) / (matrix_xx[wi + 2] + xx.s2 + 1.0e-37f); matrix_xg[wi + 2] = 0; matrix_xx[wi + 2] = 0; } else { matrix_xg[wi + 2] += xg.s2; matrix_xx[wi + 2] += xx.s2; } case 1: if(update) { matrix_w[wi + 1] = matrix_w[wi + 1] + l * (matrix_xg[wi + 1] + xg.s1) / (matrix_xx[wi + 1] + xx.s1 + 1.0e-37f); matrix_xg[wi + 1] = 0; matrix_xx[wi + 1] = 0; } else { matrix_xg[wi + 1] += xg.s1; matrix_xx[wi + 1] += xx.s1; } case 0: if(update) { matrix_w[wi] = matrix_w[wi] + l * (matrix_xg[wi] + xg.s0) / (matrix_xx[wi] + xx.s0 + 1.0e-37f); matrix_xg[wi] = 0; matrix_xx[wi] = 0; } else { matrix_xg[wi] += xg.s0; matrix_xx[wi] += xx.s0; } break; } } //+------------------------------------------------------------------+ ///\ingroup neuron_proof_ff /// Kernel of the Pooling neuron for Feed forward process (#CNeuronProofOCL) //+------------------------------------------------------------------+ __kernel void FeedForwardProof(__global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_o, ///<[out] Output tensor int inputs, ///< Number of inputs int window, ///< Size of input window int step ///< Step size ) { int i = get_global_id(0); int pos = i * step; float result = matrix_i[pos]; //--- for(int k = 1; k < window; k++) { int shift = k + pos; if(shift >= inputs) break; result = max(result, matrix_i[shift]); } matrix_o[i] = result; } //+------------------------------------------------------------------+ ///\ingroup neuron_proof_gr /// Kernel of the Pooling neuron to transfer gradient to previous layer /// (#CNeuronProofOCL) //+------------------------------------------------------------------+ __kernel void CalcInputGradientProof(__global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_o, ///<[in] Output tensor __global float *matrix_ig, ///<[out] Tensor of gradients at previous layer int outputs, ///< Number of outputs int window, ///< Size of input window int step ///< Step size ) { int i = get_global_id(0); float prev_gradient = 0; float value = matrix_i[i]; int start = i - window + step; start = (start - start % step) / step; int stop = (i - i % step) / step + 1; for(int out = max(0, start); out < min(outputs, stop); out++) { if(value == matrix_o[out]) prev_gradient += matrix_g[out]; } matrix_ig[i] = prev_gradient; } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_ff /// Kernel of the Convolution neuron for Feed forward process (#CNeuronConvOCL) //+------------------------------------------------------------------+ __kernel void FeedForwardConv(__global const float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_o, ///<[out] Output tensor const int inputs, ///< Number of inputs const int step, ///< Step size const int window_in, ///< Size of input window const int window_out, ///< Size of output window const int activation ///< Activation type (#ENUM_ACTIVATION) ) { const size_t i = get_global_id(0); const int out = get_global_id(1); const size_t v = get_global_id(2); const size_t outputs = get_global_size(0); //--- const int shift_out = window_out * i; const int shift_in = step * i; //--- const int shift_var_in = v * inputs; const int shift_var_out = v * window_out * outputs; const int shift_var_w = v * window_out * (window_in + 1); //--- float sum = 0; float inp; //--- int shift = (window_in + 1) * out; int stop = (window_in <= (inputs - shift_in) ? window_in : (inputs - shift_in)); for(int k = 0; k < stop; k ++) { inp = IsNaNOrInf(matrix_i[shift_var_in + shift_in + k], 0.0f); if(inp == 0.0f) continue; sum += IsNaNOrInf(inp * matrix_w[shift_var_w + shift + k], 0.0f); } sum += IsNaNOrInf(matrix_w[shift_var_w + shift + window_in], 0.0f); //--- matrix_o[shift_var_out + out + shift_out] = fActivation(sum, activation);; } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_gr /// Kernel of the Convolution neuron to transfer gradient /// to previous layer (#CNeuronConvOCL) //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientConv(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_o, ///<[in] Output tensor __global float *matrix_ig, ///<[out] Tensor of gradients at previous layer const int outputs, ///< Number of outputs const int step, ///< Step size const int window_in, ///< Size of input window const int window_out, ///< Size of output window const int activation, ///< Activation type (#ENUM_ACTIVATION) const int shift_out ///< Shift in output and gradient buffer ) { const size_t i = get_global_id(0); const size_t inputs = get_global_size(0); const size_t v = get_global_id(1); //--- Resolve per-variable tensor offsets. const int shift_var_in = v * inputs; const int shift_var_out = v * outputs; const int shift_var_w = v * window_out * (window_in + 1); //--- Derive the exact forward-window interval from the signed input position. float sum = 0; float out = matrix_o[shift_var_in + i]; const int position = (int)i; const int start = position < window_in ? 0 : (position - window_in) / step + 1; const int stop = min(position / step + 1, outputs / window_out); //--- Accumulate each output component across the contributing windows. for(int h = 0; h < window_out; h ++) { for(int k = start; k < stop; k++) { int shift_g = k * window_out + h; int shift_w = h * (window_in + 1) + position - k * step; float grad = IsNaNOrInf(matrix_g[shift_out + shift_g + shift_var_out], 0.0f); if(fabs(grad) > 0.0f) sum += IsNaNOrInf(grad * matrix_w[shift_w + shift_var_w], 0.0f); } } //--- Apply the existing activation derivative to the accumulated gradient. matrix_ig[shift_var_in + i] = Deactivation(sum, out, activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_opt Convolution Neuron SGD optimization Updating Weights /// Calculation kernel /// Describes the process of SGD optimization weights for the Convolution Neuron /// (#CNeuronConvOCL). //+------------------------------------------------------------------+ __kernel void UpdateWeightsConvMomentum(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_dw, ///<[in,out] Matrix of delta weights in last correction int inputs, ///< Number of inputs float learning_rates, ///< Learning rates float momentum, ///< Momentum multiplier int window_in, ///< Size of input window int window_out, ///< Size of output window int step ///< Step size ) { const size_t i = get_global_id(0); //--- const int v = i / ((window_in + 1) * window_out); const int shift = i % (window_in + 1); const int shift_out = i / (window_in + 1) - v; const int total = (inputs - window_in + step - 1) / step; //--- const int shift_var_in = v * inputs; const int shift_var_out = v * total * window_out; //--- float grad = 0; //--- for(int t = 0; t < total; t++) { if(shift != window_in && (shift + t * window_in) >= inputs) break; grad += IsNaNOrInf(matrix_g[t * window_out + shift_out + shift_var_out] * (shift == window_in ? 1 : matrix_i[shift + t * step + shift_var_in]), 0.0f); } float delta = IsNaNOrInf(learning_rates * grad, 0) + momentum * matrix_dw[i]; if(!isnan(delta)) { matrix_dw[i] = delta; if(fabs(delta) > 0) matrix_w[i] = IsNaNOrInf(matrix_w[i] + delta, 0); } } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_opt Convolution Neuron Adam optimization Updating /// Weights Calculation kernel /// Describes the process of Adam optimization weights for the Convolution /// Neuron (#CNeuronConvOCL). //+------------------------------------------------------------------+ __kernel void UpdateWeightsConvAdam(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in] Matrix of first momentum __global float *matrix_v, ///<[in] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2, ///< Second momentum multiplier int window_in, ///< Size of input window int window_out, ///< Size of output window int step ///< Step size ) { const size_t i = get_global_id(0); //--- const int v = i / ((window_in + 1) * window_out); const int shift = i % (window_in + 1); const int shift_out = i / (window_in + 1) - v * window_out; const int total = (inputs - (window_in - step) + (step - 1)) / step; //--- const int shift_var_in = v * inputs; const int shift_var_out = v * total * window_out; //--- float grad = 0; //--- for(int t = 0; t < total; t++) { if(shift != window_in && (shift + t * window_in) >= inputs) break; grad += IsNaNOrInf(matrix_g[t * window_out + shift_out + shift_var_out] * (shift == window_in ? 1 : matrix_i[shift + t * step + shift_var_in]), 0); } grad = clamp(IsNaNOrInf(grad, 0), -MAX_GRAD, MAX_GRAD); float mt = IsNaNOrInf(b1 * matrix_m[i] + (1 - b1) * grad, 0); float vt = IsNaNOrInf(b2 * matrix_v[i] + (1 - b2) * (grad * grad), 1.0f); float weight = IsNaNOrInf(matrix_w[i] + IsNaNOrInf(l * mt / sqrt(vt), 0), 0); matrix_w[i] = weight; matrix_m[i] = mt; matrix_v[i] = vt; } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_opt Convolution Neuron Least Squares optimization /// Updating Weights Calculation kernel /// Describes the process of Least Squares optimization weights for the /// Convolution Neuron (#CNeuronConvOCL). //+------------------------------------------------------------------+ __kernel void UpdateWeightsConvLS(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_xg, ///<[in] Matrix of summ x*g __global float *matrix_xx, ///<[in] Matrix of summ x*x const int inputs, ///< Number of inputs const float l, ///< Learning rates const int update, ///< Update flag int window_in, ///< Size of input window int window_out, ///< Size of output window int step ///< Step size ) { const int i = get_global_id(0); if(i > window_in) return; //--- int total = (inputs - (window_in - step)) % step; total = (inputs - (window_in - step) - total) / step + (total > 0 ? 1 : 0); //--- for(int out = 0; out < window_out; out++) { if((window_out - out) > 4) { float4 xg = {0, 0, 0, 0}; float x2 = 0; int shift_w = i + out * (window_in + 1); for(int t = 0; t < total; t++) { if(i != window_in && (i + t * window_in) >= inputs) break; xg += (float4)(matrix_g[t * window_out + out], matrix_g[t * window_out + out + 1], matrix_g[t * window_out + out + 2], matrix_g[t * window_out + out + 3]) * (i == window_in ? 1 : matrix_i[i + t * step]); x2 += (i == window_in ? 1 : matrix_i[i + t * step] * matrix_i[i + t * step]); } if(update) { xg = (float4)(matrix_xg[shift_w], matrix_xg[shift_w + window_in + 1], matrix_xg[shift_w + 2 * (window_in + 1)], matrix_xg[shift_w + 3 * (window_in + 1)]) + xg; float4 xx = (float4)(matrix_xx[shift_w], matrix_xx[shift_w + window_in + 1], matrix_xx[shift_w + 2 * (window_in + 1)], matrix_xx[shift_w + 3 * (window_in + 1)]) + x2; float4 delta = l * xg / (xx + 1.0e-37f); float4 weight = (float4)(matrix_w[shift_w], matrix_w[shift_w + (window_in + 1)], matrix_w[shift_w + 2 * (window_in + 1)], matrix_w[shift_w + 3 * (window_in + 1)]) + delta; matrix_w[shift_w] = weight.s0; matrix_w[shift_w + (window_in + 1)] = weight.s1; matrix_w[shift_w + 2 * (window_in + 1)] = weight.s2; matrix_w[shift_w + 3 * (window_in + 1)] = weight.s3; matrix_xg[shift_w] = 0; matrix_xg[shift_w + (window_in + 1)] = 0; matrix_xg[shift_w + 2 * (window_in + 1)] = 0; matrix_xg[shift_w + 3 * (window_in + 1)] = 0; matrix_xx[shift_w] = 0; matrix_xx[shift_w + (window_in + 1)] = 0; matrix_xx[shift_w + 2 * (window_in + 1)] = 0; matrix_xx[shift_w + 3 * (window_in + 1)] = 0; } else { matrix_xg[shift_w] += xg.s0; matrix_xg[shift_w + (window_in + 1)] += xg.s1; matrix_xg[shift_w + 2 * (window_in + 1)] += xg.s2; matrix_xg[shift_w + 3 * (window_in + 1)] += xg.s3; matrix_xx[shift_w] = matrix_xx[shift_w + (window_in + 1)] = matrix_xx[shift_w + 2 * (window_in + 1)] = matrix_xx[shift_w + 3 * (window_in + 1)] += x2; } out += 3; } else { float xg = 0; float xx = 0; int shift_w = i + out * (window_in + 1); for(int t = 0; t < total; t++) { if(i != window_in && (i + t * window_in) >= inputs) break; xg += matrix_g[t * window_out + out] * (i == window_in ? 1 : matrix_i[i + t * step]); xx += (i == window_in ? 1 : matrix_i[i + t * step] * matrix_i[i + t * step]); } if(update) { xg = matrix_xg[shift_w] + xg; xx = matrix_xx[shift_w] + xx; float delta = l * xg / (xx + 1.0e-37f); matrix_w[shift_w] = matrix_w[shift_w] + delta; matrix_xg[shift_w] = 0; matrix_xx[shift_w] = 0; } else { matrix_xg[shift_w] += xg; matrix_xx[shift_w] += xx; } } } } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Attention Neuron Score calculation kernel | /// Describes the Score calculation process for the Neuron of attention layer /// (#CNeuronAttentionOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void AttentionScore(__global float *querys, ///<[in] Matrix of Querys __global float *keys, ///<[in] Matrix of Keys __global float *score, ///<[out] Matrix of Scores int dimension, ///< Dimension of Key int mask ///< 1 - calc only previous units, 0 - calc all ) { int q = get_global_id(0); int shift_q = q * dimension; int units = get_global_size(0); int shift_s = q * units; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; float sum = 0; //--- for(int k = 0; k < units; k++) { if(mask > 0 && k > q) { score[shift_s + k] = 0; continue; } float result = 0; int shift_k = k * dimension; for(int i = 0; i < dimension; i++) result += (querys[shift_q + i] * keys[shift_k + i]); result = IsNaNOrInf(exp(result / koef), 0); score[shift_s + k] = result; sum += result; } //--- for(int k = 0; (k < units && sum > 0); k++) score[shift_s + k] /= sum; } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Attention Neuron Out calculation kernel /// Describes the Attention out calculation process for the Neuron of attention /// layer (#CNeuronAttentionOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void AttentionOut(__global float *scores, ///<[in] Matrix of Scores __global float *values, ///<[in] Matrix of Values __global float *inputs, ///<[in] Inputs tensor __global float *out ///<[out] Output tensor ) { int units = get_global_size(0); int u = get_global_id(0); int d = get_global_id(1); int dimension = get_global_size(1); int shift = u * dimension + d; float result = 0; //--- for(int i = 0; i < units; i++) result += IsNaNOrInf(scores[u * units + i], 0) * IsNaNOrInf(values[i * dimension + d], 0); out[shift] = IsNaNOrInf(result, 0) + inputs[shift]; } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Kernel for calculation Sum of 2 matrixs with /// multiplyer. /// Describes the calculation Sum of 2 matrixs. ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void SumMatrix(__global float *matrix1, ///<[in] First matrix __global float *matrix2, ///<[in] Second matrix __global float *matrix_out, ///<[out] Output matrix int dimension, ///< Dimension of matrix float multiplyer, ///< Multiplyer for output int shift_in1, ///< Shift for input 1 int shift_in2, ///< Shift for input 2 int shift_out, ///< Shift for output __global float *numeric_flags, const int flags_enabled ) { const int i = get_global_id(0); const int step = get_global_size(0); //--- for(int k = 0; k < dimension; k++) { int index = i * dimension + k; if(flags_enabled) { const float value1 = matrix1[i * shift_in1 + index]; const float value2 = matrix2[i * shift_in2 + index]; const float raw_sum = value1 + value2; const float raw_result = raw_sum * multiplyer; const int invalid = isnan(value1) || isinf(value1) || isnan(value2) || isinf(value2) || isnan(raw_sum) || isinf(raw_sum) || isnan(raw_result) || isinf(raw_result); numeric_flags[index] = (invalid ? 1.0f : 0.0f); } matrix_out[i * shift_out + index] = IsNaNOrInf((matrix1[i * shift_in1 + index] + matrix2[i * shift_in2 + index]) * multiplyer, 0); } } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Kernel for calculation Sum of 4 matrixs with /// multiplyer. /// Describes the calculation Sum of 4 matrixs. ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void Sum5Matrix(__global float *matrix1, ///<[in] First matrix __global float *matrix2, ///<[in] Second matrix __global float *matrix3, ///<[in] Third matrix __global float *matrix4, ///<[in] Fourth matrix __global float *matrix5, ///<[in] Fifth matrix __global float *matrix_out, ///<[out] Output matrix int dimension, ///< Dimension of matrix float multiplyer ///< Multiplyer for output ) { const int i = get_global_id(0) * dimension; //--- for(int k = 0; k < dimension; k++) matrix_out[i + k] = (matrix1[i + k] + matrix2[i + k] + matrix3[i + k] + matrix4[i + k] + matrix5[i + k]) * multiplyer; } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_gr Attention layer's neuron Gradients Calculation /// kernel /// Describes the gradients calculation process for the Neuron of attention /// layer (#CNeuronAttentionOCL). ///\details Detailed description on the link. /// @param[in] querys Matrix of Querys /// @param[out] querys_g Matrix of Querys' Gradients /// @param[in] keys Matrix of Keys /// @param[out] keys_g Matrix of Keys' Gradients /// @param[in] values Matrix of Values /// @param[out] values_g Matrix of Values' Gradients /// @param[in] scores Matrix of Scores /// @param[in] gradient Matrix of Gradients from previous iteration //+------------------------------------------------------------------+ __kernel void AttentionInsideGradients(__global float *querys, __global float *querys_g, __global float *keys, __global float *keys_g, __global float *values, __global float *values_g, __global float *scores, __global float *gradient) { int u = get_global_id(0); int d = get_global_id(1); int units = get_global_size(0); int dimension = get_global_size(1); float koef = sqrt((float)dimension); if(koef < 1) koef = 1; float vg = 0; float qg = 0; float kg = 0; //--- for(int iu = 0; iu < units; iu++) { float g = gradient[iu * dimension + d]; float sc = scores[iu * units + u]; vg += sc * g; //--- float sqg = 0; float skg = 0; for(int id = 0; id < dimension; id++) { sqg += values[iu * dimension + id] * gradient[u * dimension + id]; skg += values[u * dimension + id] * gradient[iu * dimension + id]; } qg += (scores[u * units + iu] == 0 || scores[u * units + iu] == 1 ? 0.0001f : scores[u * units + iu] * (1 - scores[u * units + iu])) * sqg * keys[iu * dimension + d] / koef; //--- kg += (scores[iu * units + u] == 0 || scores[iu * units + u] == 1 ? 0.0001f : scores[iu * units + u] * (1 - scores[iu * units + u])) * skg * querys[iu * dimension + d] / koef; } int shift = u * dimension + d; values_g[shift] = clamp(IsNaNOrInf(vg, 0.0f), -1.0f, 1.0f); querys_g[shift] = clamp(IsNaNOrInf(qg, 0.0f), -1.0f, 1.0f); keys_g[shift] = clamp(IsNaNOrInf(kg, 0.0f), -1.0f, 1.0f); } //+------------------------------------------------------------------+ ///\ingroup neuron_norm Kernels of matrix normalization process /// Describes the process of matrix normalization. ///\details Detailed description on the link. /// @param[in,out] buffer In/Out Matrix /// @param[in] dimension Dimension of matrix //+------------------------------------------------------------------+ __kernel void Normalize(__global float *buffer, int dimension) { int n = get_global_id(0); int shift = n * dimension; if(dimension < 1) return; //--- float mean = 0; float M2 = 0; float variance = 0; //--- for(int i = 0; i < dimension; i++) { float val = IsNaNOrInf(buffer[shift + i], 0); double delta = val - mean; mean += delta / (i + 1); M2 += delta * (val - mean); } variance = M2 / (dimension - 1); //--- for(int i = 0; i < dimension; i++) if(variance > 1) buffer[shift + i] = IsNaNOrInf((buffer[shift + i] - mean) / variance, 0); else buffer[shift + i] = IsNaNOrInf(buffer[shift + i] - mean, 0); } //+------------------------------------------------------------------+ ///\ingroup neuron_norm Kernels of weights matrix normalization process /// Describes the process of weights matrix normalization. ///\details Detailed description on the link. /// @param[in,out] buffer In/Out Matrix /// @param[in] dimension Dimension of matrix //+------------------------------------------------------------------+ __kernel void NormalizeWeights(__global float *buffer, int dimension) { int n = get_global_id(0); int shift = n * dimension; float sum = 0; float k = 1; //--- do { sum = 0; for(int i = 0; (i < dimension && !(isnan(sum) || isinf(sum))); i++) { float normalized = IsNaNOrInf(buffer[shift + i], 0) / k; sum += normalized * normalized; } if(isnan(sum) || isinf(sum)) k *= 10; } while(isnan(sum) || isinf(sum)); sum = sqrt(sum); if(k * sum > 1) for(int i = 0; i < dimension; i++) buffer[shift + i] = IsNaNOrInf(buffer[shift + i], 0) / (k * sum); } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff /// Describes the process of concatenate 4 matrices. ///\details Detailed description on the link. /// @param[in] input1, input2, input3, input4 Input buffers /// @param[in] window1, window2, window3, window4 Windows for every buffer /// @param[out] output Output buffer //+------------------------------------------------------------------+ __kernel void ConcatenateBuffers(__global float *input1, int window1, __global float *input2, int window2, __global float *input3, int window3, __global float *input4, int window4, __global float *output) { int n = get_global_id(0); int w = get_global_id(1); int total_w = get_global_size(1); int shift = n * (window1 + window2 + window3 + window4); int shift_in = n * window1; for(int i = w; i < window1; i += total_w) output[shift + i] = IsNaNOrInf(input1[shift_in + i], 0); //--- shift += window1; shift_in = n * window2; //--- for(int i = w; i < window2; i += total_w) output[shift + i] = IsNaNOrInf(input2[shift_in + i], 0); //--- shift += window2; shift_in = n * window3; //--- for(int i = w; i < window3; i += total_w) output[shift + i] = IsNaNOrInf(input3[shift_in + i], 0); //--- shift += window3; shift_in = n * window4; //--- for(int i = w; i < window4; i += total_w) output[shift + i] = IsNaNOrInf(input4[shift_in + i], 0); } //+------------------------------------------------------------------+ //| Copies raw 32-bit buffer payloads without float sanitization. | //+------------------------------------------------------------------+ __kernel void CopyBufferRaw(__global const uint *source, __global uint *destination, const int total) { const int index = get_global_id(0); if(index < total) destination[index] = source[index]; } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_gr /// Describes the process of deconcatenate matrix. ///\details Detailed description on the link. /// @param[in] output1, output2, output3, output4 Output buffers /// @param[in] window1, window2, window3, window4 Windows for every buffer /// @param[out] inputs Input buffer //+------------------------------------------------------------------+ __kernel void DeconcatenateBuffers(__global float *output1, int window1, __global float *output2, int window2, __global float *output3, int window3, __global float *output4, int window4, __global float *inputs) { int n = get_global_id(0); int w = get_global_id(1); int total_w = get_global_size(1); //--- Head 1 int shift = n * (window1 + window2 + window3 + window4); int shift_out = n * window1; //--- for(int i = w; i < window1; i += total_w) output1[shift_out + i] = IsNaNOrInf(inputs[shift + i], 0); //--- Head 2 shift += window1; shift_out = n * window2; //--- for(int i = w; i < window2; i += total_w) output2[shift_out + i] = IsNaNOrInf(inputs[shift + i], 0); //--- Head 3 shift += window2; if(window3 > 0) { shift_out = n * window3; //--- for(int i = w; i < window3; i += total_w) output3[shift_out + i] = IsNaNOrInf(inputs[shift + i], 0); } //--- Head 4 shift += window3; if(window4 > 0) { shift_out = n * window4; //--- for(int i = w; i < window4; i += total_w) output4[shift_out + i] = IsNaNOrInf(inputs[shift + i], 0); } } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Multi-Heads Attention Neuron Score calculation /// kernel /// Describes the Score calculation process for the Neuron of multi-heads /// attention layer (#CNeuronMLMHAttentionOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void MHAttentionScore(__global float *qkv, ///<[in] Matrix of Querys, Keys, Values __global float *score, ///<[out] Matrix of Scores int dimension, ///< Dimension of Key int mask ///< 1 - calc only previous units, 0 - calc all ) { int q = get_global_id(0); int h = get_global_id(1); int units = get_global_size(0); int heads = get_global_size(1); //--- int shift_q = dimension * (h + 3 * q * heads); int shift_s = units * (h + q * heads); //--- float koef = sqrt((float)dimension); if(koef < 1) koef = 1; float sum = 0; //--- for(int k = 0; k < units; k++) { if(mask > 0 && k > q) { score[shift_s + k] = 0; continue; } float result = 0; int shift_k = dimension * (h + heads * (3 * k + 1)); for(int i = 0; i < dimension; i++) { if((dimension - i) > 4) { result += dot(IsNaNOrInf4((float4)(qkv[shift_q + i], qkv[shift_q + i + 1], qkv[shift_q + i + 2], qkv[shift_q + i + 3]), 0), IsNaNOrInf4((float4)(qkv[shift_k + i], qkv[shift_k + i + 1], qkv[shift_k + i + 2], qkv[shift_k + i + 3]), 0)); i += 3; } else result += IsNaNOrInf(qkv[shift_q + i] * qkv[shift_k + i], 0); } result = exp(clamp(result / koef, -100.0f, 100.0f)); if(isnan(result)) result = 0; score[shift_s + k] = result; sum += result; } //--- for(int k = 0; (k < units && sum > 1); k++) score[shift_s + k] /= sum; } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_ff Multi-heads Attention Neuron Out calculation kernel /// Describes the Multi-heads Attention out calculation process for the Neuron /// of multi-heads attention layer (#CNeuronMLMHAttentionOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void MHAttentionOut(__global float *scores, ///<[in] Matrix of Scores __global float *qkv, ///<[in] Matrix of Values __global float *out, ///<[out] Output tensor int dimension ///< Dimension of Value ) { int u = get_global_id(0); int units = get_global_size(0); int h = get_global_id(1); int heads = get_global_size(1); //--- int shift_s = units * (h + heads * u); int shift_out = dimension * (h + heads * u); int layer = 3 * dimension * heads; //--- //--- for(int d = 0; d < dimension; d++) { float result = 0; for(int v = 0; v < units; v++) { int shift_v = dimension * (h + heads * (3 * v + 2)) + d; result += scores[shift_s + v] * qkv[shift_v]; } out[shift_out + d] = result; } } //+------------------------------------------------------------------+ ///\ingroup neuron_atten_gr Attention layer's neuron Gradients Calculation /// kernel /// Describes the gradients calculation process for the Neuron of attention /// layer (#CNeuronMLMHAttentionOCL). ///\details Detailed description on the link. /// @param[in] qkv Matrix of Querys, Keys and Values /// @param[out] qkv_g Matrix of Querys', Keys' and Values' Gradients /// @param[in] scores Matrix of Scores /// @param[in] scores_g Matrix of Scores' Gradients /// @param[in] gradient Matrix of Gradients from previous iteration /// @param[in] dimension Dimension of Key vector //+------------------------------------------------------------------+ __kernel void MHAttentionInsideGradients(__global float *qkv, __global float *qkv_g, __global float *scores, __global float *gradient) { size_t u = get_global_id(0); size_t h = get_global_id(1); size_t d = get_global_id(2); size_t units = get_global_size(0); size_t heads = get_global_size(1); size_t dimension = get_global_size(2); //--- float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- init const int shift_q = dimension * (heads * 3 * u + h); const int shift_k = dimension * (heads * (3 * u + 1) + h); const int shift_v = dimension * (heads * (3 * u + 2) + h); const int shift_g = dimension * (heads * u + h); int shift_score = h * units; int step_score = units * heads; //--- Calculating Value's gradients float sum = 0; //--- for(int i = 0; i < units; i++) sum += gradient[(h + i * heads) * dimension + d] * scores[shift_score + u + i * step_score]; qkv_g[shift_v + d] = sum; //--- Calculating Query's gradients shift_score = h * units + u * step_score; float grad = 0; float grad_out = gradient[shift_g + d]; //--- for(int k = 0; k < units; k++) { float sc_g = 0; float sc = scores[shift_score + k]; for(int v = 0; v < units; v++) sc_g += scores[shift_score + v] * qkv[dimension * (heads * (3 * v + 2) + h)] * grad_out * ((k == v) - sc); grad += sc_g / koef * qkv[dimension * (heads * (3 * k + 1) + h) + d]; } qkv_g[shift_q + d] = grad; //--- Calculating Key's gradients grad = 0; //--- for(int q = 0; q < units; q++) { shift_score = h * units + q * step_score; float sc_g = 0; float sc = scores[shift_score + u]; float grad_out = gradient[dimension * (heads * q + h) + d]; for(int v = 0; v < units; v++) sc_g += scores[shift_score + v] * qkv[dimension * (heads * (3 * v + 2) + h)] * grad_out * ((u == v) - sc); grad += sc_g / koef * qkv[dimension * (heads * 3 * q + h) + d]; } qkv_g[shift_k + d] = grad; } //+------------------------------------------------------------------+ ///\ingroup neuron_dropout Kernel for Dropout. /// Describes the dropout method. ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void Dropout(__global const float *inputs, ///<[in] Input matrix __global const float *map, ///<[in] Dropout map matrix __global float *out, ///<[out] Output matrix const int dimension ///< Dimension of matrix ) { const int i = get_global_id(0) * 4; float m = 0, inp = 0; //--- for(int k = i; k < min(dimension, i + 4); k++) { m = IsNaNOrInf(map[i + k], 0.0f); if(m == 0) { out[i + k] = 0; continue; } inp = IsNaNOrInf(inputs[i + k], 0.0f); if(inp == 0) { out[i + k] = 0; continue; } out[i + k] = IsNaNOrInf(inp * m, 0); } } //+------------------------------------------------------------------+ ///\ingroup neuron_norm Kernels of Batch normalization process /// Describes the process of Batch normalization. (#CNeuronBatchNormOCL) ///\details Detailed description on the link. /// @param[in] inputs Input data tenzor /// @param[in,out] options Tenzor of variables /// @param[out] output Tenzor of output data /// @param[in] batch Batch size /// @param[in] optimization Optimization type /// @param[in] activation Activation type //+------------------------------------------------------------------+ __kernel void BatchFeedForward(__global float *inputs, __global float *options, __global float *output, int batch, int optimization, int activation, int training) { int n = get_global_id(0); int shift = n * (optimization == 0 ? 7 : 9); //--- float inp = IsNaNOrInf(inputs[n], 0); if(training == 0) { const float mean = IsNaNOrInf(options[shift], 0); float variance = IsNaNOrInf(options[shift + 1], 1); if(variance <= 0) variance = 1; float gamma = IsNaNOrInf(options[shift + 3], 1); if(gamma == 0) gamma = 1; const float betta = IsNaNOrInf(options[shift + 4], 0); const float nx = (inp - mean) / sqrt(variance); output[n] = fActivation(gamma * nx + betta, activation); return; } if(batch <= 1) return; float mean = (IsNaNOrInf(options[shift], 0) * max((float)batch - 1.0f, 0.0f) + inp) / max((float)batch, 1.0f); float delt = inp - mean; float variance = (IsNaNOrInf(options[shift + 1] * max((float)batch - 1.0f, 0.0f), 0) + delt * delt) / max((float)batch, 1.0f); if(variance <= 0) variance = 1; float nx = delt / sqrt(variance); //--- float gamma = IsNaNOrInf(options[shift + 3], 1); if(gamma == 0) { options[shift + 3] = 1; gamma = 1; } float betta = IsNaNOrInf(options[shift + 4], 0); //--- options[shift] = mean; options[shift + 1] = variance; options[shift + 2] = nx; output[n] = fActivation(gamma * nx + betta, activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_gr /// Kernel of the Batch neuron to transfer gradient to previous layer /// (#CNeuronBatchNormOCL) ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientBatch(__global float *options, ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_i, ///<[in] Tensor of previous layer output __global float *matrix_ig, ///<[out] Tensor of gradients at previous layer int activation, ///< Activation type (#ENUM_ACTIVATION) int batch, ///< Batch size int optimization ///< Optimization type ) { if(batch <= 1) return; //--- int n = get_global_id(0); int shift = n * (optimization == 0 ? 7 : 9); //--- float variance = IsNaNOrInf(options[shift + 1], 1); //--- float inp = IsNaNOrInf(matrix_i[n], 0); float gnx = IsNaNOrInf(matrix_g[n], 0) * IsNaNOrInf(options[shift + 3], 1); float temp = (variance > 0 ? 1.0f / sqrt(variance) : 0); float gmu = (-temp) * gnx; float gvar = (variance > 0 ? (IsNaNOrInf(options[shift], 0) * inp) / (2 * pow(variance, 3.0f / 2.0f)) * gnx : 0); float batch_ratio = max((float)(batch - 1), 0.0f) / max((float)batch, 1.0f); float gx = temp * gnx + gmu / max(batch, 1) + gvar * 2 * inp / max(batch, 1) * batch_ratio * batch_ratio; //--- matrix_ig[n] = Deactivation(gx, inp, activation);; } //+------------------------------------------------------------------+ ///\ingroup neuron_opt Batch normalization Neuron SGD optimization Updating /// options kernel /// Describes the process of SGD optimization options for the Batch /// normalization Neuron (#CNeuronBatchNormOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void UpdateBatchOptionsMomentum(__global float *options, ///<[in,out] Options matrix m*7, where m - Number of neurons in previous layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer float learning_rates, ///< Learning rates float momentum ///< Momentum multiplier ) { const int n = get_global_id(0); const int inputs = get_global_size(0); const int shift = n * 7; float grad = clamp(IsNaNOrInf(matrix_g[n], 0), -MAX_GRAD, MAX_GRAD); //--- float2 delta = learning_rates * grad * (float2)(IsNaNOrInf(options[shift + 2], 0), 1) + momentum * (float2)(IsNaNOrInf(options[shift + 5], 0), IsNaNOrInf(options[shift + 6], 0)); //--- delta.s0 = IsNaNOrInf(delta.s0, 0); delta.s1 = IsNaNOrInf(delta.s1, 0); options[shift + 5] = delta.s0; float value = IsNaNOrInf(options[shift + 3], 1); options[shift + 3] = value + delta.s0 - learning_rates * (l1 * sign(value) + l2 * value / inputs); //--- options[shift + 6] = delta.s1; value = IsNaNOrInf(options[shift + 4], 0); options[shift + 4] = value + delta.s1 - learning_rates * (l1 * sign(value) + l2 * value / inputs); } //+------------------------------------------------------------------+ ///\ingroup neuron_opt Batch normalization Neuron Adam optimization Updating /// options kernel /// Describes the process of Adam optimization options for the Batch /// normalization Neuron (#CNeuronBatchNormOCL). ///\details Detailed description on the link. //+------------------------------------------------------------------+ __kernel void UpdateBatchOptionsAdam(__global float *options, ///<[in,out] Options matrix m*9, where m - Number of neurons in previous layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int n = get_global_id(0); int inputs = get_global_size(0); const int shift = n * 9; float grad = clamp(IsNaNOrInf(matrix_g[n], 0), -MAX_GRAD, MAX_GRAD); //--- float nx = IsNaNOrInf(options[shift + 2], 0); float gamma = IsNaNOrInf(options[shift + 3], 1); if(gamma == 0) gamma = 1; float betta = IsNaNOrInf(options[shift + 4], 0); //--- float gamma_m1 = IsNaNOrInf(options[shift + 5], 0); float betta_m1 = IsNaNOrInf(options[shift + 6], 0); float gamma_m2 = IsNaNOrInf(options[shift + 7], 0); float betta_m2 = IsNaNOrInf(options[shift + 8], 0); //--- float2 mt = b1 * (float2)(gamma_m1, betta_m1) + (1 - b1) * (float2)(grad * nx, grad); float2 grad2 = (float2)(grad * nx, grad); float2 vt = b2 * (float2)(gamma_m2, betta_m2) + (1 - b2) * (grad2 * grad2); vt.s0 = IsNaNOrInf(vt.s0, 1); vt.s1 = IsNaNOrInf(vt.s1, 1); float2 delta = l * mt / sqrt(vt); delta.s0 = IsNaNOrInf(delta.s0, 0); delta.s1 = IsNaNOrInf(delta.s0, 0); float2 weight = delta - (l1 * sign((float2)(gamma, betta)) + l2 * (float2)(gamma, betta) / inputs); //--- options[shift + 3] = IsNaNOrInf(gamma + weight.s0, 1); options[shift + 4] = IsNaNOrInf(betta + weight.s1, 0); options[shift + 5] = IsNaNOrInf(mt.s0, 0); options[shift + 6] = IsNaNOrInf(mt.s1, 0); options[shift + 7] = vt.s0; options[shift + 8] = vt.s1; } //+------------------------------------------------------------------+ //| Runs the vae feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void VAE_FeedForward(__global float *inputs, __global float *random, __global float *outputs, __global float *numeric_flags, const int flags_enabled) { uint i = (uint)get_global_id(0); uint total = (uint)get_global_size(0); if(flags_enabled) { //--- Read the raw VAE operands before this work-item updates a potentially aliased output. const float mean = inputs[i]; const float log_var = inputs[i + total]; const float epsilon = random[i]; const float half_log_var = 0.5f * log_var; const float sigma = exp(half_log_var); const float product = IsNaNOrInf(sigma, 0) * epsilon; const float raw_result = IsNaNOrInf(mean, 0) + product; const int invalid = isnan(mean) || isinf(mean) || isnan(log_var) || isinf(log_var) || isnan(epsilon) || isinf(epsilon) || isnan(half_log_var) || isinf(half_log_var) || isnan(sigma) || isinf(sigma) || isnan(product) || isinf(product) || isnan(raw_result) || isinf(raw_result); numeric_flags[i] = (invalid ? 1.0f : 0.0f); } outputs[i] = IsNaNOrInf(inputs[i], 0) + IsNaNOrInf(exp(0.5f * inputs[i + total]), 0) * random[i]; } //+------------------------------------------------------------------+ //| Computes the vae calc hidden gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void VAE_CalcHiddenGradient(__global float *inputs, __global float *inp_grad, __global float *random, __global float *gradient, const float kld_mult) { uint i = (uint)get_global_id(0); uint total = (uint)get_global_size(0); float log_var = IsNaNOrInf(inputs[i + total], 0); float mean = IsNaNOrInf(inputs[i], 0); float sigma = exp(0.5f * log_var); float variance = exp(log_var); float grad = clamp(IsNaNOrInf(gradient[i], 0), -MAX_GRAD, MAX_GRAD); //--- gradient stores the negative downstream derivative used by additive updates. //--- Add the negative derivative of kld_mult * KL(N(mean,variance)||N(0,1)). inp_grad[i] = IsNaNOrInf(grad - kld_mult * mean, 0); inp_grad[i + total] = IsNaNOrInf(0.5f * grad * sigma * random[i] - 0.5f * kld_mult * (variance - 1.0f), 0); } //+------------------------------------------------------------------+ //| Runs the lstm feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void LSTM_FeedForward(__global const float *inputs, int inputs_size, __global const float *weights, __global float *concatenated, __global float *memory, __global float *output) { uint id = (uint)get_global_id(0); uint total = (uint)get_global_size(0); uint id2 = (uint)get_local_id(1); uint idv = (uint)get_global_id(2); uint total_v = (uint)get_global_size(2); //--- __local float Temp[4]; //--- float sum = 0; uint shift_in = idv * inputs_size; uint shift_out = idv * total; uint shift = (inputs_size + total + 1) * (id2 + id); //--- for(uint i = 0; i < total; i += 4) { if(total - i > 4) sum += IsNaNOrInf( dot((float4)(output[shift_out + i], output[shift_out + i + 1], output[shift_out + i + 2], output[shift_out + i + 3]), (float4)(weights[shift + i], weights[shift + i + 1], weights[shift + i + 2], weights[shift + i + 3])), 0); else for(uint k = i; k < total; k++) sum += IsNaNOrInf(output[shift_out + k] * weights[shift + k], 0); } //--- shift += total; //--- for(uint i = 0; i < inputs_size; i += 4) { if(total - i > 4) sum += IsNaNOrInf( dot((float4)(inputs[shift_in + i], inputs[shift_in + i + 1], inputs[shift_in + i + 2], inputs[shift_in + i + 3]), (float4)(weights[shift + i], weights[shift + i + 1], weights[shift + i + 2], weights[shift + i + 3])), 0); else for(uint k = i; k < total; k++) sum += IsNaNOrInf(inputs[shift_in + k] * weights[shift + k], 0); } sum += IsNaNOrInf(weights[shift + inputs_size], 0); if(id2 < 3) sum = fActivation(sum, 1); else sum = fActivation(sum, 0); Temp[id2] = sum; concatenated[4 * shift_out + id2 * total + id] = sum; //--- BarrierLoc if(id2 == 0) { float mem = memory[shift_out + id + total_v * total] = memory[shift_out + id]; float fg = Temp[0]; float ig = Temp[1]; float og = Temp[2]; float nc = Temp[3]; //--- memory[shift_out + id] = mem = IsNaNOrInf(mem * fg + ig * nc, 0); output[shift_out + id] = IsNaNOrInf(og * fActivation(mem, 0), 0); } } //+------------------------------------------------------------------+ //| Computes the lstm concatenated gradient gradient contributions on| //| the GPU. | //+------------------------------------------------------------------+ __kernel void LSTM_ConcatenatedGradient(__global float *gradient, __global float *concatenated_gradient, __global float *memory, __global float *concatenated) { uint id = (uint)get_global_id(0); uint total = (uint)get_global_size(0); uint idv = (uint)get_global_id(1); uint total_v = (uint)get_global_size(1); //--- uint shift_out = idv * total; float t = tanh(memory[shift_out + id]); //--- concatenated_gradient[4 * shift_out + id + 2 * total] = gradient[shift_out + id] * t; // output gate //--- float memory_gradient = gradient[shift_out + id] * concatenated[4 * shift_out + id + 2 * total]; memory_gradient *= 1 - t * t; //--- concatenated_gradient[4 * shift_out + id + 3 * total] = memory_gradient * concatenated[4 * shift_out + id + total]; // new content //--- concatenated_gradient[4 * shift_out + id + total] = memory_gradient * concatenated[4 * shift_out + id + 3 * total]; // input gate //--- concatenated_gradient[4 * shift_out + id] = memory_gradient * memory[shift_out + id + total_v * total]; // forgat gate } //+------------------------------------------------------------------+ //| Computes the lstm hidden gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void LSTM_HiddenGradient(__global float *concatenated_gradient, __global float *inputs_gradient, __global float *weights_gradient, __global float *hidden_state, __global float *inputs, __global float *weights, __global float *output, const int hidden_size, const int inputs_size) { uint id = get_global_id(0); uint total = get_global_size(0); uint idv = (uint)get_global_id(1); uint total_v = (uint)get_global_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; uint ls = min(total_v, (uint)LOCAL_ARRAY_SIZE); //--- uint shift_in = idv * inputs_size; uint shift_out = idv * total; uint weights_step = hidden_size + inputs_size + 1; //--- //--- for(int i = id; i < (hidden_size + inputs_size); i += total) { float inp = 0; if(i < hidden_size) { inp = hidden_state[shift_out + i]; hidden_state[shift_out + i] = output[shift_out + i]; } else { inp = inputs[shift_in + i - hidden_size]; float grad = 0; for(uint g = 0; g < 3 * hidden_size; g++) { float temp = concatenated_gradient[4 * shift_out + g]; grad += temp * (1 - temp) * weights[i + g * weights_step]; } for(uint g = 3 * hidden_size; g < 4 * hidden_size; g++) { float temp = concatenated_gradient[4 * shift_out + g]; grad += temp * (1 - temp * temp) * weights[i + g * weights_step]; } inputs_gradient[shift_in + i - hidden_size] = grad; } //--- for(uint g = 0; g < 3 * hidden_size; g++) { float temp = concatenated_gradient[4 * shift_out + g]; if(idv < ls) Temp[idv % ls] = 0; BarrierLoc for(uint v = 0; v < total_v; v += ls) { if(idv >= v && idv < v + ls) Temp[idv % ls] += temp * (1 - temp) * inp; BarrierLoc } if(idv == 0) { temp = Temp[0]; for(int v = 1; v < ls; v++) temp += Temp[v]; weights_gradient[i + g * weights_step] = temp; } BarrierLoc } for(uint g = 3 * hidden_size; g < 4 * hidden_size; g++) { float temp = concatenated_gradient[4 * shift_out + g]; if(idv < ls) Temp[idv % ls] = 0; BarrierLoc for(uint v = 0; v < total_v; v += ls) { if(idv >= v && idv < v + ls) Temp[idv % ls] += temp * (1 - temp * temp) * inp; BarrierLoc } if(idv == 0) { temp = Temp[0]; for(int v = 1; v < ls; v++) temp += Temp[v]; weights_gradient[i + g * weights_step] = temp; } BarrierLoc } } //--- for(int i = id; i < 4 * hidden_size; i += total) { if(idv < ls) Temp[idv % ls] = 0; BarrierLoc float temp = concatenated_gradient[4 * shift_out + i]; if(i < 3 * hidden_size) { for(uint v = 0; v < total_v; v += ls) { if(idv >= v && idv < v + ls) Temp[idv % ls] += temp * (1 - temp); BarrierLoc } } else { for(uint v = 0; v < total_v; v += ls) { if(idv >= v && idv < v + ls) Temp[idv % ls] += 1 - temp * temp; BarrierLoc } } if(idv == 0) { temp = Temp[0]; for(int v = 1; v < ls; v++) temp += Temp[v]; weights_gradient[(i + 1) * weights_step] = temp; } BarrierLoc } } //+------------------------------------------------------------------+ ///\ingroup LSTM_opt LSTM Adam Updating Weights Calculation kernel /// Describes the process of Adam optimization weights for the Neuron LSTM /// (#CNeuronLSTMOCL). //+------------------------------------------------------------------+ __kernel void LSTM_UpdateWeightsAdam(__global float *weights, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global float *weights_gradient, ///<[in] Tensor of gradients at current layer __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const uint id = get_global_id(0); const uint total = get_global_size(0); const uint id1 = get_global_id(1); const uint wi = id1 * total + id; float g = clamp(IsNaNOrInf(weights_gradient[wi], 0), -MAX_GRAD, MAX_GRAD); float mt = b1 * IsNaNOrInf(matrix_m[wi], 0) + (1 - b1) * g; float vt = b2 * IsNaNOrInf(matrix_v[wi], 1) + (1 - b2) * (g * g); float weight = IsNaNOrInf(weights[wi], 0); float delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight / total)); weights[wi] = IsNaNOrInf(weight + delta, 0); matrix_m[wi] = mt; matrix_v[wi] = vt; } //+------------------------------------------------------------------+ //| Runs the soft max feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void SoftMax_FeedForward(__global float *inputs, __global float *outputs) { const uint total = (uint)get_local_size(0); const uint l = (uint)get_local_id(0); const uint h = (uint)get_global_id(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; uint shift_head = h * total; //--- outputs[shift_head + l] = LocalSoftMax(IsNaNOrInf(inputs[shift_head + l], MIN_VALUE), 0, Temp); } //+------------------------------------------------------------------+ //| Computes the soft max hidden gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void SoftMax_HiddenGradient(__global float *outputs, __global float *output_gr, __global float *input_gr) { size_t i = get_local_id(0); size_t outputs_total = get_local_size(0); size_t h = get_global_id(1); __local float Temp[LOCAL_ARRAY_SIZE]; //--- uint shift = h * outputs_total; float output = IsNaNOrInf(outputs[shift + i], 0); float grad = IsNaNOrInf(output_gr[shift + i], 0); input_gr[shift + i] = LocalSoftMaxGrad(output, grad, 0, Temp); } //+------------------------------------------------------------------+ //| Computes the soft max output gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void SoftMax_OutputGradient(__global float *outputs, __global float *targets, __global float *output_gr) { size_t i = get_global_id(0); output_gr[i] = (outputs[i] == 0 ? 0 : targets[i] / outputs[i]); } //+------------------------------------------------------------------+ //| Runs the fqf cosine kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void FQF_Cosine(__global float *softmax, __global float *output) { size_t i = get_global_id(0); size_t total = get_global_size(0); size_t action = get_global_id(1); int shift = action * total; //--- float result = 0; //--- for(int it = 0; it < i; it++) result += softmax[shift + it]; result += softmax[shift + i] / 2.0f; output[shift + i] = cos(i * M_PI_F * result); } //+------------------------------------------------------------------+ //| Runs the fqf output kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void FQF_Output(__global float *quantiles, __global float *delta_taus, __global float *output, int total) { size_t action = get_global_id(0); int shift = action * total; //--- float result = 0; //--- for(int i = 0; i < total; i++) result += quantiles[shift + i] * delta_taus[shift + i]; output[action] = result; } //+------------------------------------------------------------------+ //| Computes the fqf output gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void FQF_OutputGradient(__global float *quantiles, __global float *delta_taus, __global float *output_gr, __global float *quantiles_gr, __global float *taus_gr) { size_t i = get_global_id(0); size_t total = get_global_size(0); size_t action = get_global_id(1); int shift = action * total; //--- float gradient = output_gr[action]; quantiles_gr[shift + i] = gradient * delta_taus[shift + i]; taus_gr[shift + i] = gradient * quantiles[shift + i]; } //+------------------------------------------------------------------+ //| Computes the fqf quantile gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void FQF_QuantileGradient(__global float *state_embeding, __global float *taus_embeding, __global float *quantiles_gr, __global float *state_gr, __global float *taus_gr) { size_t i = get_global_id(0); size_t total = get_global_size(0); size_t action = get_global_id(1); int shift = action * total; //--- float gradient = quantiles_gr[shift + i]; state_gr[shift + i] = gradient * taus_embeding[shift + i]; taus_gr[shift + i] = gradient * state_embeding[shift + i]; } //+------------------------------------------------------------------+ //| Computes the fqf cosine gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void FQF_CosineGradient(__global float *softmax, __global float *output_gr, __global float *softmax_gr) { size_t i = get_global_id(0); size_t total = get_global_size(0); size_t action = get_global_id(1); int shift = action * total; //--- float cumul = 0; //--- for(int it = 0; it < i; it++) cumul += softmax[shift + it]; float result = -M_PI_F * i * sin(M_PI_F * i * (cumul + softmax[shift + i] / 2)) * output_gr[shift + i]; //--- for(int it = i + 1; it < total; it++) { cumul += softmax[shift + it - 1]; float temp = cumul + softmax[shift + it] / 2; result += -M_PI_F * it * sin(M_PI_F * it * temp) * output_gr[shift + it] * softmax[shift + it] / temp; } softmax_gr[shift + i] += result; } //+------------------------------------------------------------------+ //| Computes the exact cosine-feature gradient used ONLY by the | //| Complexity: O(N^2) per task (each row i sums N-1 neighbor terms). | //| CNeuronACSRM specialized backward ::propagateACSRMGradients. | //| c_i = cos(pi*i*tau_i), tau_i = sum_{j= 0.0f && selected_index < (float)total); const size_t k = (found ? (size_t)selected_index : total); //--- const float pi = IsNaNOrInf(probabilities[RCtoFlat(v,i,total_v,total,0)], 0.0f); //--- value gradient quantiles_gr[RCtoFlat(v,i,total_v,total,0)] = (found && i == k ? value_weight * (target - zi) : 0.0f); //--- probability gradient at SoftMax output probabilities_gr[RCtoFlat(v,i,total_v,total,0)] = (found && i == k ? probability_weight / (pi + 1.0e-37f) : 0.0f); } //+------------------------------------------------------------------+ //| FQF Sample | //+------------------------------------------------------------------+ //| Samples one scalar realization per independent distribution. | //| One work-group owns one task: it reads its N probabilities once, | //| builds the local inclusive CDF with a prefix scan, locates u_t | //| inside the CDF and only the winning item reads its centroid. | //+------------------------------------------------------------------+ __kernel void FQF_Sample(__global float *quantiles, __global float *probabilities, __global float *random, __global float *result) { const size_t i = get_local_id(0); const size_t total = get_local_size(0); const size_t task = get_group_id(1); const size_t tasks = get_num_groups(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- row = task, column = quantile, row width = total (single variable) const size_t index = RCtoFlat(task, i, tasks, total, 0); //--- every task has its own uniform random value u_t in [0, 1] const float rnd = fmin(fmax(IsNaNOrInf(random[task], 0.5f), 0.0f), 1.0f); //--- one probability read per element; NaN/Inf/negative maps to 0 const float pi = fmax(IsNaNOrInf(probabilities[index], 0.0f), 0.0f); //--- local inclusive prefix scan: Temp[i] = CDF_i = sum_{j<=i} p_j Temp[i] = pi; BarrierLoc; for(size_t offset = 1; offset < total; offset <<= 1) { const float add = (i >= offset ? Temp[i - offset] : 0.0f); BarrierLoc; if(i >= offset) Temp[i] += add; BarrierLoc; } //--- destroyed or non-finite distribution (mass ~ 0, or the float32 sum //--- overflowed to +Inf/NaN): neutral sample 0 written by i == 0 const float mass = IsNaNOrInf(Temp[total - 1], 0.0f); if(mass <= 1.0e-6f) { if(i == 0) result[task] = 0.0f; return; } //--- first interval containing u_t: CDF_i >= u_t and CDF_{i-1} < u_t const float threshold = rnd * mass; const float cumulative = Temp[i]; const float previous = (i == 0 ? 0.0f : Temp[i - 1]); bool selected = cumulative >= threshold && (i == 0 || previous < threshold); //--- float rounding fallback: force the last element if(i == total - 1 && cumulative < threshold) selected = true; //--- the parallel prefix scan is float32; the CDF can locally decrease by //--- one ulp, so several items may satisfy the crossing test. Keep the //--- candidate index in a private register, then LocalMin selects one //--- deterministic winner (the smallest index). The barrier after the //--- candidate guarantees every work-item already consumed its CDF value //--- (Temp) before the reduction reuses the scratch buffer. const float candidate = (selected ? (float)i : (float)total); BarrierLoc; const float winner = LocalMin(candidate, 0, Temp); BarrierLoc; //--- only the single winning item reads its centroid through RCtoFlat if(winner == (float)i) { const float value = quantiles[index]; result[task] = IsNaNOrInf(value, 0.0f); } } //+------------------------------------------------------------------+ //| FQF SRM Cumulative Spectral Function | //+------------------------------------------------------------------+ //| Returns the cumulative spectral mass Phi(u) = int_0^u phi(v) dv | //| for the selected spectral risk measure. The result is clamped | //| to [0, 1] and the kernel always stays finite; parameter | //| validation happens on the MQL5 side before the launch. | //+------------------------------------------------------------------+ float SRMCumulative(const float u, const int risk_type, const float risk_parameter, const float mean_weight) { const float tau = fmin(fmax(u, 0.0f), 1.0f); //--- switch(risk_type) { case SRM_MEAN: return tau; case SRM_CVAR: { const float alpha = fmin(fmax(risk_parameter, SRM_PARAMETER_EPS), 1.0f); return fmin(fmax(fmin(tau, alpha) / alpha, 0.0f), 1.0f); } case SRM_MEAN_CVAR: { const float alpha = fmin(fmax(risk_parameter, SRM_PARAMETER_EPS), 1.0f); const float omega = fmin(fmax(mean_weight, 0.0f), 1.0f); return fmin(fmax(omega * tau + (1.0f - omega) * fmin(tau, alpha) / alpha, 0.0f), 1.0f); } case SRM_EXPONENTIAL: { //--- expm1 is supported by the target OpenCL compiler const float lambda = fmax(risk_parameter, 0.0f); if(lambda <= SRM_PARAMETER_EPS) return tau; const float denominator = -expm1(-lambda); const float numerator = -expm1(-lambda * tau); return fmin(fmax(numerator / denominator, 0.0f), 1.0f); } case SRM_DUAL_POWER: { const float nu = fmax(risk_parameter, 1.0f); return fmin(fmax(1.0f - pow(1.0f - tau, nu), 0.0f), 1.0f); } default: return 0.0f; } } //+------------------------------------------------------------------+ //| FQF SRM Density | //+------------------------------------------------------------------+ //| Returns the spectral density phi(u) = dPhi(u)/du of the selected | //| risk measure; it is the exact derivative of ::SRMCumulative. | //| CVaR and Mean-CVaR use the right-hand derivative at u == alpha, | //| Exponential falls back to the Mean density at lambda <= EPS. | //+------------------------------------------------------------------+ float SRMDensity(const float u, const int risk_type, const float risk_parameter, const float mean_weight) { const float tau = fmin(fmax(u, 0.0f), 1.0f); //--- switch(risk_type) { case SRM_MEAN: return 1.0f; case SRM_CVAR: { const float alpha = fmin(fmax(risk_parameter, SRM_PARAMETER_EPS), 1.0f); return (tau < alpha ? 1.0f / alpha : 0.0f); } case SRM_MEAN_CVAR: { const float alpha = fmin(fmax(risk_parameter, SRM_PARAMETER_EPS), 1.0f); const float omega = fmin(fmax(mean_weight, 0.0f), 1.0f); return (tau < alpha ? omega + (1.0f - omega) / alpha : omega); } case SRM_EXPONENTIAL: { //--- expm1 is supported by the target OpenCL compiler const float lambda = fmax(risk_parameter, 0.0f); if(lambda <= SRM_PARAMETER_EPS) return 1.0f; const float denominator = -expm1(-lambda); return fmax(lambda * exp(-lambda * tau) / denominator, 0.0f); } case SRM_DUAL_POWER: { const float nu = fmax(risk_parameter, 1.0f); if(nu == 1.0f) return 1.0f; return fmax(nu * pow(1.0f - tau, nu - 1.0f), 0.0f); } default: return 0.0f; } } //+------------------------------------------------------------------+ //| FQF SRM Forward | //+------------------------------------------------------------------+ //| Convolves one FQF distribution per work-group into the scalar | //| spectral risk measure rho_t = sum_i w_i z_i. The probabilities | //| are prefix-scanned into the local inclusive CDF, the adaptive | //| interval bounds are normalized by the total mass and the exact | //| spectral mass of every interval is Phi(tau_high) - Phi(tau_low). | //+------------------------------------------------------------------+ __kernel void FQF_SRMForward(__global float *quantiles, __global float *probabilities, __global float *result, const int risk_type, const float risk_parameter, const float mean_weight) { const size_t i = get_local_id(0); const size_t total = get_local_size(0); const size_t task = get_group_id(1); const size_t tasks = get_num_groups(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- row = task, column = quantile, row width = total (single variable) const size_t index = RCtoFlat(task, i, tasks, total, 0); //--- one probability read per element; NaN/Inf/negative maps to 0 float pi = probabilities[index]; pi = fmax(IsNaNOrInf(pi, 0.0f), 0.0f); //--- local inclusive prefix scan: Temp[i] = CDF_i = sum_{j<=i} p_j Temp[i] = pi; BarrierLoc; for(size_t offset = 1; offset < total; offset <<= 1) { const float add = (i >= offset ? Temp[i - offset] : 0.0f); BarrierLoc; if(i >= offset) Temp[i] += add; BarrierLoc; } //--- destroyed distribution (total mass ~ 0): neutral risk 0 const float mass = Temp[total - 1]; if(IsNaNOrInf(mass, 0.0f) <= 1.0e-37f) { if(i == 0) result[task] = 0.0f; return; } //--- normalize the probability axis by the actual total mass const float inv_mass = 1.0f / mass; const float tau_high = fmin(fmax(Temp[i] * inv_mass, 0.0f), 1.0f); const float tau_low = (i == 0 ? 0.0f : fmin(fmax(Temp[i - 1] * inv_mass, 0.0f), 1.0f)); //--- exact spectrum mass of the adaptive interval const float spectral_high = SRMCumulative(tau_high, risk_type, risk_parameter, mean_weight); const float spectral_low = SRMCumulative(tau_low, risk_type, risk_parameter, mean_weight); const float weight = fmax(spectral_high - spectral_low, 0.0f); //--- distribution value of the same FQF grid index const float zi = IsNaNOrInf(quantiles[index], 0.0f); const float contribution = weight * zi; //--- CDF is no longer needed BarrierLoc; //--- reduce the weighted returns over the work-group const float risk = LocalSum(contribution, 0, Temp); if(i == 0) result[task] = IsNaNOrInf(risk, 0.0f); } //+------------------------------------------------------------------+ //| FQF SRM Gradient | //+------------------------------------------------------------------+ //| Back-propagates the upstream scalar risk-score gradient g_t of | //| rho_t = sum_i w_i z_i. One work-group per task: the centroid | //| gradient is g_t * (Phi(tau_high) - Phi(tau_low)) and the | //| probability gradient is assembled from a reverse scan over the | //| adaptive CDF-boundary derivatives as (A_k - B) / M, using the | //| same mass normalization and sanitization as ::FQF_SRMForward. | //+------------------------------------------------------------------+ __kernel void FQF_SRMGradient(__global float *quantiles, __global float *probabilities, __global float *result_gradient, __global float *quantiles_gradient, __global float *probabilities_gradient, const int risk_type, const float risk_parameter, const float mean_weight) { const size_t i = get_local_id(0); const size_t total = get_local_size(0); const size_t task = get_group_id(1); const size_t tasks = get_num_groups(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- row = task, column = quantile, row width = total (single variable) const size_t index = RCtoFlat(task, i, tasks, total, 0); //--- sanitize inputs; validity flags repeat the forward masking const float raw_pi = probabilities[index]; const bool p_valid = !(isnan(raw_pi) || isinf(raw_pi)) && raw_pi >= 0.0f; const float pi = fmax(IsNaNOrInf(raw_pi, 0.0f), 0.0f); const float raw_zi = quantiles[index]; const bool z_valid = !(isnan(raw_zi) || isinf(raw_zi)); const float zi = IsNaNOrInf(raw_zi, 0.0f); const float upstream = IsNaNOrInf(result_gradient[task], 0.0f); //--- zero upstream: gradients vanish identically (avoids 0*Inf NaN paths) if(upstream == 0.0f) { quantiles_gradient[index] = 0.0f; probabilities_gradient[index] = 0.0f; return; } //--- local inclusive prefix scan: Temp[i] = CDF_i = sum_{j<=i} p_j Temp[i] = pi; BarrierLoc; for(size_t offset = 1; offset < total; offset <<= 1) { const float add = (i >= offset ? Temp[i - offset] : 0.0f); BarrierLoc; if(i >= offset) Temp[i] += add; BarrierLoc; } //--- destroyed distribution (total mass ~ 0): neutral gradients only const float mass = IsNaNOrInf(Temp[total - 1], 0.0f); if(mass <= 1.0e-37f) { quantiles_gradient[index] = 0.0f; probabilities_gradient[index] = 0.0f; return; } //--- normalize the probability axis by the actual total mass const float inv_mass = 1.0f / mass; const float tau_high = fmin(fmax(Temp[i] * inv_mass, 0.0f), 1.0f); const float tau_low = (i == 0 ? 0.0f : fmin(fmax(Temp[i - 1] * inv_mass, 0.0f), 1.0f)); //--- exact spectrum mass of the adaptive interval (same as forward) const float spectral_high = SRMCumulative(tau_high, risk_type, risk_parameter, mean_weight); const float spectral_low = SRMCumulative(tau_low, risk_type, risk_parameter, mean_weight); const float weight = fmax(spectral_high - spectral_low, 0.0f); //--- direct centroid gradient: g_t * w_i (invalid centroid -> 0) quantiles_gradient[index] = (z_valid ? IsNaNOrInf(upstream * weight, 0.0f) : 0.0f); //--- stage sanitized centroids into local memory for neighbor reads BarrierLoc; Temp[i] = zi; BarrierLoc; const float next_z = (i + 1 < total ? Temp[i + 1] : zi); //--- derivative of rho w.r.t. the adaptive CDF boundary tau_i const float boundary_gradient = (i + 1 < total ? IsNaNOrInf(upstream * (zi - next_z) * SRMDensity(tau_high, risk_type, risk_parameter, mean_weight), 0.0f) : 0.0f); //--- reverse inclusive scan: Temp[i] = A_i = sum_{j>=i} a_j BarrierLoc; Temp[i] = boundary_gradient; BarrierLoc; for(size_t offset = 1; offset < total; offset <<= 1) { const float add = (i + offset < total ? Temp[i + offset] : 0.0f); BarrierLoc; if(i + offset < total) Temp[i] += add; BarrierLoc; } const float suffix = Temp[i]; //--- normalization correction B = sum_q q_i A_i (one local reduction) const float normalized_probability = pi * inv_mass; const float correction = LocalSum(normalized_probability * suffix, 0, Temp); //--- direct probability gradient: (A_i - B) / M (invalid p -> 0) probabilities_gradient[index] = (p_valid ? IsNaNOrInf((suffix - correction) * inv_mass, 0.0f) : 0.0f); } //+------------------------------------------------------------------+ //| FQF Fraction Gradient | //+------------------------------------------------------------------+ //| Exact input gradient of the fraction proposal head. For the | //| plain linear fraction layer the full graph derivative d rho/d x | //| receives the probability branch (dP/dx)^T g_P = W^T * g_probs. | //| Unlike ::CalcHiddenGradient this kernel applies no activation | //| clipping: it mirrors the clamp-free arithmetic of | //| ::FQF_QuantileGradient so the composite FQF layer propagates the | //| exact chain-rule gradient to its input. | //+------------------------------------------------------------------+ __kernel void FQF_FractionGradient(__global float *matrix_w, __global float *gradient, __global float *input_gr, const int outputs) { const uint i = get_global_id(0); const uint inputs = get_global_size(0); const uint loc = get_local_id(1); const uint total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- float sum = 0; for(int k = (int)loc; k < outputs; k += (int)total_loc) sum += matrix_w[k * (inputs + 1) + i] * gradient[k]; if(total_loc > 1) sum = LocalSum(sum, 1, Temp); if(loc == 0) input_gr[i] = IsNaNOrInf(sum, 0.0f); } //+------------------------------------------------------------------+ //| Sparse Attention | //+------------------------------------------------------------------+ __kernel void MHSparseAttentionScore(__global float *qkv, ///<[in] Matrix of Querys, Keys, Values __global float *score, ///<[out] Matrix of Scores int dimension, ///< Dimension of Key float sparse ///< less than 1.0 coefficient of sparse ) { int q = get_global_id(0); int h = get_global_id(1); int units = get_global_size(0); int heads = get_global_size(1); //--- int shift_q = dimension * (h + 3 * q * heads); int shift_s = units * (h + q * heads); int active_units = (int)max((float)(units * sparse), min((float)units, 3.0f)); //--- float koef = sqrt((float)dimension); if(koef < 1) koef = 1; float sum = 0.0f; float min_s = 0.0f; float max_s = 0.0f; //--- for(int k = 0; k < units; k++) { float result = 0; int shift_k = dimension * (h + heads * (3 * k + 1)); for(int i = 0; i < dimension; i++) { if((dimension - i) > 4) { result += dot((float4)(qkv[shift_q + i], qkv[shift_q + i + 1], qkv[shift_q + i + 2], qkv[shift_q + i + 3]), (float4)(qkv[shift_k + i], qkv[shift_k + i + 1], qkv[shift_k + i + 2], qkv[shift_k + i + 3])); i += 3; } else result += (qkv[shift_q + i] * qkv[shift_k + i]); } score[shift_s + k] = result; if(k == 0) min_s = max_s = result; else { max_s = max(max_s, result); min_s = min(min_s, result); } } //--- int count = units; //--- while(count > active_units && min_s < max_s) { count = 0; float temp = max_s; for(int k = 0; k < units; k++) { float value = score[shift_s + k]; if(value < min_s) continue; count++; if(value < temp && value > min_s) temp = value; } if(count > active_units) min_s = temp; } //--- if(max_s == 0.0f) max_s = 1.0f; //--- for(int k = 0; k < units; k++) { float value = score[shift_s + k]; if(value < min_s) { score[shift_s + k] = 0.0f; continue; } value = exp(value / max_s / koef); score[shift_s + k] = value; sum += value; } //--- for(int k = 0; (k < units && sum > 1); k++) { float temp = score[shift_s + k]; if(temp == 0.0f) continue; score[shift_s + k] = temp / sum; } } //+------------------------------------------------------------------+ //| Implements the mh sparse attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHSparseAttentionOut(__global float *scores, ///<[in] Matrix of Scores __global float *qkv, ///<[in] Matrix of Values __global float *out, ///<[out] Output tensor int dimension ///< Dimension of Value ) { int u = get_global_id(0); int units = get_global_size(0); int h = get_global_id(1); int heads = get_global_size(1); //--- int shift_s = units * (h + heads * u); int shift_out = dimension * (h + heads * u); //--- for(int d = 0; d < dimension; d++) { float result = 0; for(int v = 0; v < units; v++) { float cur_score = scores[shift_s + v]; if(cur_score == 0) continue; int shift_v = dimension * (h + heads * (3 * v + 2)) + d; result += cur_score * qkv[shift_v]; } out[shift_out + d] = result; } } //+------------------------------------------------------------------+ //| Runs the feed forward multi models feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardMultiModels(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number of neurons in layer and n - number of outputs (neurons in next layer) __global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_o, ///<[out] Output tensor int inputs, ///< Number of inputs int activation ///< Activation type (#ENUM_ACTIVATION) ) { int i = get_global_id(0); int outputs = get_global_size(0); int m = get_global_id(1); int models = get_global_size(1); //--- float sum = 0; float4 inp, weight; int shift = (inputs + 1) * (i + outputs * m); int shift_in = inputs * m; int shift_out = outputs * m; //--- for(int k = 0; k <= inputs; k = k + 4) { switch(inputs - k) { case 0: inp = (float4)(1, 0, 0, 0); weight = (float4)(matrix_w[shift + k], 0, 0, 0); break; case 1: inp = (float4)(matrix_i[shift_in + k], 1, 0, 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], 0, 0); break; case 2: inp = (float4)(matrix_i[shift_in + k], matrix_i[shift_in + k + 1], 1, 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], 0); break; case 3: inp = (float4)(matrix_i[shift_in + k], matrix_i[shift_in + k + 1], matrix_i[shift_in + k + 2], 1); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], matrix_w[shift + k + 3]); break; default: inp = (float4)(matrix_i[shift_in + k], matrix_i[shift_in + k + 1], matrix_i[shift_in + k + 2], matrix_i[shift_in + k + 3]); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], matrix_w[shift + k + 3]); break; } float d = dot(inp, weight); if(isnan(sum + d)) continue; sum += d; } if(isnan(sum)) sum = 0; //--- matrix_o[shift_out + i] = fActivation(sum, activation); } //+------------------------------------------------------------------+ //| Computes the calc hidden gradient multi models gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientMultiModels(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number ///< of neurons in previous layer and n - number ///< of neurons in current layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_o, ///<[in] Previous layer Output tensor __global float *matrix_ig, ///<[out] Tensor of gradients at previous layer int outputs, ///< Number of outputs int activation, ///< Activation type (#ENUM_ACTIVATION), int model) { int i = get_global_id(0); int inputs = get_global_size(0); int m = get_global_id(1); int models = get_global_size(1); //--- int shift_in = inputs * m; if(model >= 0 && model != m) { matrix_ig[shift_in + i] = 0; return; } //--- int shift_out = outputs * m; int shift_w = (inputs + 1) * outputs * m; float sum = 0; float out = matrix_o[shift_in + i]; float4 grad, weight; //--- for(int k = 0; k < outputs; k += 4) { switch(outputs - k) { case 1: weight = (float4)(matrix_w[shift_w + k * (inputs + 1) + i], 0, 0, 0); grad = (float4)(matrix_g[shift_out + k], 0, 0, 0); break; case 2: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + 1], 0, 0); weight = (float4)(matrix_w[shift_w + k * (inputs + 1) + i], matrix_w[shift_w + (k + 1) * (inputs + 1) + i], 0, 0); break; case 3: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + 1], matrix_g[shift_out + k + 2], 0); weight = (float4)(matrix_w[shift_w + k * (inputs + 1) + i], matrix_w[shift_w + (k + 1) * (inputs + 1) + i], matrix_w[shift_w + (k + 2) * (inputs + 1) + i], 0); break; default: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + 1], matrix_g[shift_out + k + 2], matrix_g[shift_out + k + 3]); weight = (float4)(matrix_w[shift_w + k * (inputs + 1) + i], matrix_w[shift_w + (k + 1) * (inputs + 1) + i], matrix_w[shift_w + (k + 2) * (inputs + 1) + i], matrix_w[shift_w + (k + 3) * (inputs + 1) + i]); break; } sum += dot(grad, weight); } //--- matrix_ig[shift_in + i] = Deactivation(sum, out, activation); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights adam multi | //| models step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsAdamMultiModels( __global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2, ///< Second momentum multiplier const int model) { const int outputs = get_global_size(0); const int i = get_global_id(0); const int j = get_global_id(1); const int wi = (i + outputs * model) * (inputs + 1) + j * 4; float4 m, v, weight, inp; int shift_in = j * 4 + inputs * model; if((inputs + 1 - j * 4) < 0) return; switch(inputs + 1 - j * 4) { case 0: inp = (float4)(1, 0, 0, 0); weight = (float4)(matrix_w[wi], 0, 0, 0); m = (float4)(matrix_m[wi], 0, 0, 0); v = (float4)(matrix_v[wi], 0, 0, 0); break; case 1: inp = (float4)(matrix_i[shift_in], 1, 0, 0); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], 0, 0); m = (float4)(matrix_m[wi], matrix_m[wi + 1], 0, 0); v = (float4)(matrix_v[wi], matrix_v[wi + 1], 0, 0); break; case 2: inp = (float4)(matrix_i[shift_in], matrix_i[shift_in + 1], 1, 0); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], 0); m = (float4)(matrix_m[wi], matrix_m[wi + 1], matrix_m[wi + 2], 0); v = (float4)(matrix_v[wi], matrix_v[wi + 1], matrix_v[wi + 2], 0); break; case 3: inp = (float4)(matrix_i[shift_in], matrix_i[shift_in + 1], matrix_i[shift_in + 2], 1); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], matrix_w[wi + 3]); m = (float4)(matrix_m[wi], matrix_m[wi + 1], matrix_m[wi + 2], matrix_m[wi + 3]); v = (float4)(matrix_v[wi], matrix_v[wi + 1], matrix_v[wi + 2], matrix_v[wi + 3]); break; default: inp = (float4)(matrix_i[shift_in], matrix_i[shift_in + 1], matrix_i[shift_in + 2], matrix_i[shift_in + 3]); weight = (float4)(matrix_w[wi], matrix_w[wi + 1], matrix_w[wi + 2], matrix_w[wi + 3]); m = (float4)(matrix_m[wi], matrix_m[wi + 1], matrix_m[wi + 2], matrix_m[wi + 3]); v = (float4)(matrix_v[wi], matrix_v[wi + 1], matrix_v[wi + 2], matrix_v[wi + 3]); break; } float4 g = (float4)(matrix_g[(outputs + 1) * model + i]) * inp; float4 mt = b1 * m + (1 - b1) * g; float4 vt = b2 * v + (1 - b2) * (g * g); float4 delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); switch(min(inputs + 1 - j * 4, 3)) { case 3: if(fabs(delta.s3) > 0) matrix_w[wi + 3] = matrix_w[wi + 3] + delta.s3; matrix_m[wi + 3] = mt.s3; matrix_v[wi + 3] = vt.s3; case 2: if(fabs(delta.s2) > 0) matrix_w[wi + 2] = matrix_w[wi + 2] + delta.s2; matrix_m[wi + 2] = mt.s2; matrix_v[wi + 2] = vt.s2; case 1: if(fabs(delta.s1) > 0) matrix_w[wi + 1] = matrix_w[wi + 1] + delta.s1; matrix_m[wi + 1] = mt.s1; matrix_v[wi + 1] = vt.s1; case 0: if(fabs(delta.s0) > 0) matrix_w[wi] = matrix_w[wi] + delta.s0; matrix_m[wi] = mt.s0; matrix_v[wi] = vt.s0; break; } } //+------------------------------------------------------------------+ //| Runs the concat feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void Concat_FeedForward(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number ///< of neurons in layer and n - number of outputs ///< (neurons in next layer) __global float *matrix_i1, ///<[in] Inputs 1 tensor __global float *matrix_i2, ///<[in] Inputs 2 tensor __global float *matrix_o, ///<[out] Output tensor int inputs1, ///< Number of inputs int inputs2, ///< Number of inputs int activation ///< Activation type (#ENUM_ACTIVATION) ) { int i = get_global_id(0); float sum = 0; float4 inp, weight; int shift = (inputs1 + inputs2 + 1) * i; //--- //--- for(int k = 0; k < inputs1; k += 4) { switch(inputs1 - k) { case 1: inp = (float4)(matrix_i1[k], 0, 0, 0); weight = (float4)(matrix_w[shift + k], 0, 0, 0); break; case 2: inp = (float4)(matrix_i1[k], matrix_i1[k + 1], 0, 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], 0, 0); break; case 3: inp = (float4)(matrix_i1[k], matrix_i1[k + 1], matrix_i1[k + 2], 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], 0); break; default: inp = (float4)(matrix_i1[k], matrix_i1[k + 1], matrix_i1[k + 2], matrix_i1[k + 3]); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], matrix_w[shift + k + 3]); break; } float d = dot(inp, weight); if(isnan(sum + d)) continue; sum += d; } //--- shift += inputs1; //--- for(int k = 0; k < inputs2; k += 4) { switch(inputs2 - k) { case 1: inp = (float4)(matrix_i2[k], 0, 0, 0); weight = (float4)(matrix_w[shift + k], 0, 0, 0); break; case 2: inp = (float4)(matrix_i2[k], matrix_i2[k + 1], 0, 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], 0, 0); break; case 3: inp = (float4)(matrix_i2[k], matrix_i2[k + 1], matrix_i2[k + 2], 0); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], 0); break; default: inp = (float4)(matrix_i2[k], matrix_i2[k + 1], matrix_i2[k + 2], matrix_i2[k + 3]); weight = (float4)(matrix_w[shift + k], matrix_w[shift + k + 1], matrix_w[shift + k + 2], matrix_w[shift + k + 3]); break; } float d = dot(inp, weight); if(isnan(sum + d)) continue; sum += d; } sum += matrix_w[shift + inputs2]; //--- if(isnan(sum)) sum = 0; //--- matrix_o[i] = fActivation(sum, activation);; } //+------------------------------------------------------------------+ //| Computes the concat hidden gradient gradient contributions on the| //| GPU. | //+------------------------------------------------------------------+ __kernel void Concat_HiddenGradient(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_o1, ///<[in] Previous layer Output tensor __global float *matrix_o2, ///<[in] Previous layer Output tensor __global float *matrix_ig1, ///<[out] Tensor of gradients at previous layer __global float *matrix_ig2, ///<[out] Tensor of gradients at previous layer int outputs, ///< Number of outputs int inputs1, int inputs2, int activation1, ///< Activation type (#ENUM_ACTIVATION) int activation2 ///< Activation type (#ENUM_ACTIVATION) ) { int i = get_global_id(0); if(i >= (inputs1 + inputs2)) return; int inputs = inputs1 + inputs2; float sum = 0; float out = (i < inputs1 ? matrix_o1[i] : matrix_o2[i - inputs1]); float4 grad, weight; //--- for(int k = 0; k < outputs; k += 4) { switch(outputs - k) { case 1: weight = (float4)(matrix_w[k * (inputs + 1) + i], 0, 0, 0); grad = (float4)(matrix_g[k], 0, 0, 0); break; case 2: grad = (float4)(matrix_g[k], matrix_g[k + 1], 0, 0); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], 0, 0); break; case 3: grad = (float4)(matrix_g[k], matrix_g[k + 1], matrix_g[k + 2], 0); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], matrix_w[(k + 2) * (inputs + 1) + i], 0); break; default: grad = (float4)(matrix_g[k], matrix_g[k + 1], matrix_g[k + 2], matrix_g[k + 3]); weight = (float4)(matrix_w[k * (inputs + 1) + i], matrix_w[(k + 1) * (inputs + 1) + i], matrix_w[(k + 2) * (inputs + 1) + i], matrix_w[(k + 3) * (inputs + 1) + i]); break; } sum += dot(grad, weight); } if(isnan(sum)) sum = 0; if(i < inputs1) matrix_ig1[i] = Deactivation(sum, out, activation1); else matrix_ig2[i - inputs1] = Deactivation(sum, out, activation2); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the concat update weights | //| momentum step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void Concat_UpdateWeightsMomentum(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer __global float *matrix_g, ///<[in] Tensor of gradients at current layer __global float *matrix_i1, ///<[in] Inputs tensor __global float *matrix_i2, ///<[in] Inputs tensor __global float *matrix_dw, ///<[in,out] Matrix of delta weights in last correction int inputs1, ///< Number of inputs int inputs2, ///< Number of inputs float learning_rates, ///< Learning rates float momentum ///< Momentum multiplier ) { int i = get_global_id(0); int j = get_global_id(1); if(j > (inputs1 + inputs2)) return; int wi = i * (inputs1 + inputs2 + 1) + j; float inp = (j < inputs1 ? matrix_i1[j] : ((j - inputs1) < inputs2 ? matrix_i2[j - inputs1] : 1)); float delta = learning_rates * matrix_g[i] * inp + momentum * matrix_dw[wi]; if(!isnan(delta)) { matrix_dw[wi] = delta; if(fabs(delta) > 0) matrix_w[wi] = matrix_w[wi] + delta; } } //+------------------------------------------------------------------+ //| Updates the trainable weights of the concat update weights adam | //| step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void Concat_UpdateWeightsAdam(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i1, ///<[in] Inputs tensor __global const float *matrix_i2, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const int inputs1, ///< Number of inputs const int inputs2, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int i = get_global_id(0); const int j = get_global_id(1); if(j > (inputs1 + inputs2)) return; const int wi = i * (inputs1 + inputs2 + 1) + j; float inp = (j < inputs1 ? matrix_i1[j] : ((j - inputs1) < inputs2 ? matrix_i2[j - inputs1] : 1)); float weight = matrix_w[wi]; float g = matrix_g[i] * inp; float mt = b1 * matrix_m[wi] + (1 - b1) * g; float vt = b2 * matrix_v[wi] + (1 - b2) * (g * g); float delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(fabs(delta) > 0) matrix_w[wi] = matrix_w[wi] + delta; matrix_m[wi] = mt; matrix_v[wi] = vt; } //+------------------------------------------------------------------+ //| Implements the soft update kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SoftUpdate(__global float *target, ///<[in,out] Target matrix __global const float *source, ///<[in] Source matrix const float tau ///<[in] Multiplicator Tau ) { const int i = get_global_id(0); target[i] = source[i] * tau + (1.0f - tau) * target[i]; } //+------------------------------------------------------------------+ //| Updates the trainable weights of the soft update adam step using | //| the optimizer rule. | //+------------------------------------------------------------------+ __kernel void SoftUpdateAdam(__global float *target, __global const float *source, __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const float tau, ///<[in] Multiplicator Tau const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int i = get_global_id(0); float m, v, weight; m = matrix_m[i]; v = matrix_v[i]; weight = target[i]; float g = source[i] - weight; m = b1 * m + (1 - b1) * g; v = b2 * v + (1 - b2) * (g * g); float delta = (1 - tau) * m / (v != 0.0f ? sqrt(v) : 1.0f); if(fabs(delta) > 0) target[i] = weight + delta; matrix_m[i] = m; matrix_v[i] = v; } //+------------------------------------------------------------------+ //| Implements the sac alpha log probs kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SAC_AlphaLogProbs(__global float *outputs, __global float *quantiles, __global float *probs, __global float *alphas, __global float *log_probs, __global float *random, const int count_quants, const int activation) { const int i = get_global_id(0); int shift = i * count_quants; float prob = 0; float value = 0; float sum = 0; float rnd = random[i]; //--- for(int r = 0; r < count_quants; r++) { prob = probs[shift + r]; sum += prob; if(sum >= rnd || r == (count_quants - 1)) { value = quantiles[shift + r]; break; } } //--- outputs[i] = fActivation(value, activation); log_probs[i] = -alphas[i] * log(prob); } //+------------------------------------------------------------------+ //| Computes the sac alpha gradients gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SAC_AlphaGradients(__global float *outputs, __global float *gradient, __global float *log_probs, __global float *alphas_grad, const int activation) { const int i = get_global_id(0); float out = outputs[i]; //--- float grad = -gradient[i] * log_probs[i]; //--- alphas_grad[i] = Deactivation(grad, out, activation); } //+------------------------------------------------------------------+ //| Computes the sac output gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SAC_OutputGradient(__global float *quantiles, __global float *delta_taus, __global float *output_gr, __global float *quantiles_gr, __global float *taus_gr, __global float *output, const int count_quants, const int activation) { size_t action = get_global_id(0); int shift = action * count_quants; float quant1 = -1e37f; float quant2 = 1e37f; int pos1 = -1; int pos2 = -1; float value = output[action]; //--- for(int i = 0; i < count_quants; i++) { float quant = fActivation(quantiles[shift + i], activation); if(value >= quant && quant1 < quant) { quant1 = quant; pos1 = shift + i; } if(value < quant && quant2 > quant) { quant2 = quant; pos2 = shift + i; } quantiles_gr[shift + i] = 0.0f; taus_gr[shift + i] = 0.0f; } float gradient = output_gr[action]; if(quant1 > -1e37f) { quantiles_gr[pos1] = gradient * delta_taus[pos1]; taus_gr[pos1] = gradient * quant1; } if(quant2 < 1e37f) { quantiles_gr[pos2] = gradient * delta_taus[pos2]; taus_gr[pos2] = gradient * quant2; } } //+------------------------------------------------------------------+ //| Implements the sac calc log probs kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SAC_CalcLogProbs(__global float *outputs, __global float *quantiles, __global float *probs, __global float *alphas, __global float *log_probs, const int count_quants, const int activation) { const int i = get_global_id(0); int shift = i * count_quants; float quant1 = -1e37f; float quant2 = 1e37f; float prob1 = 0; float prob2 = 0; float value = outputs[i]; //--- for(int q = 0; q < count_quants; q++) { float quant = fActivation(quantiles[shift + q], activation); if(value >= quant && quant1 < quant) { quant1 = quant; prob1 = probs[shift + q]; } if(value < quant && quant2 > quant) { quant2 = quant; prob2 = probs[shift + q]; } } //--- float prob = fabs(value - quant1) / fabs(quant2 - quant1); prob = clamp((1 - prob) * prob1 + prob * prob2, 1.0e-3f, 1.0f); log_probs[i] = -alphas[i] * log(prob); } //+------------------------------------------------------------------+ //| Implements the embedding kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void Embedding(__global float *inputs, __global float *outputs, __global float *weights, __global int *windows, __global float *std, const int stack_size) { const int window_out = get_global_size(0); const int pos = get_global_id(0); const int emb = get_global_id(1); const int emb_total = get_global_size(1); const int shift_out = emb * window_out + pos; const int step = emb_total * window_out; const uint ls = min((uint)get_local_size(0), (uint)LOCAL_ARRAY_SIZE); //--- for(int i = stack_size - 1; i > 0; i--) outputs[i * step + shift_out] = outputs[(i - 1) * step + shift_out]; int shift_in = 0; //--- for(int i = 0; i < emb; i++) shift_in += windows[i]; const int window_in = windows[emb]; const int shift_weights = (shift_in + emb) * window_out + (window_in + 1) * pos; //--- __local float temp[LOCAL_ARRAY_SIZE]; if(pos < LOCAL_ARRAY_SIZE) temp[pos] = 0; BarrierLoc //--- float value = weights[shift_weights + window_in]; //--- for(int i = 0; i < window_in; i++) value += inputs[shift_in + i] * weights[shift_weights + i]; //--- for(int i = 0; i < window_out; i += ls) { if(pos >= i && pos < (i + ls)) temp[pos % ls] += value; BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(pos + count < ls) { if(pos < count) temp[pos] += temp[pos + count]; temp[pos + count] = 0; } BarrierLoc } while(count > 1); //--- value -= temp[0] / (float)window_out; BarrierLoc //--- if(pos < LOCAL_ARRAY_SIZE) temp[pos] = 0; BarrierLoc //--- for(int i = 0; i < window_out; i += ls) { if(pos >= i && pos < (i + ls)) temp[pos % ls] += (value * value) / (float)window_out; BarrierLoc } //--- count = ls; do { count = (count + 1) / 2; if(pos + count < ls) { if(pos < count) temp[pos] += temp[pos + count]; temp[pos + count] = 0; } BarrierLoc } while(count > 1); //--- if(temp[0] > 0) value /= sqrt(temp[0]); //--- outputs[shift_out] = value; if(pos == 0) std[emb] = sqrt(temp[0]); } //+------------------------------------------------------------------+ //| Computes the embedding hidden gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void EmbeddingHiddenGradient(__global float *inputs_gradient, __global float *outputs_gradient, __global float *weights, __global int *windows, __global float *std, const int window_out) { const int pos = get_global_id(0); int emb = -1; int count = 0; do { emb++; count += windows[emb]; } while(count < pos); const int shift_out = emb * window_out; const int shift_weights = pos + (count - windows[emb] + emb) * window_out; //--- float value = 0; //--- for(int i = 0; i < window_out; i++) value += outputs_gradient[shift_out + i] * weights[shift_weights + i * window_out]; float s = std[emb]; if(s > 0) value /= s; //--- inputs_gradient[pos] = value; } //+------------------------------------------------------------------+ //| Updates the trainable weights of the embedding update weights | //| adam step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void EmbeddingUpdateWeightsAdam(__global float *weights, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *gradient, ///<[in] Tensor of gradients at current layer __global const float *inputs, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum __global int *windows, __global float *std, const int window_out, const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int i = get_global_id(0); int emb = -1; int count = 0; int shift = 0; int window_in = 0; do { emb++; shift = count; window_in = windows[emb]; count += (window_in + 1) * window_out; } while(count <= i); const int shift_out = emb * window_out; int shift_in = shift / window_out - emb; shift = (i - shift) % (window_in + 1); float inp = 1.0f; if(shift < window_in) inp = inputs[shift_in + shift]; //--- float weight = weights[i]; float g = gradient[shift_out] * inp / std[emb]; float mt = b1 * matrix_m[i] + (1 - b1) * g; float vt = b2 * matrix_v[i] + (1 - b2) * (g * g); float delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(fabs(delta) > 0) weights[i] = weights[i] + delta; matrix_m[i] = mt; matrix_v[i] = vt; } //+------------------------------------------------------------------+ //| Performs the transpose data-layout operation on the GPU. | //+------------------------------------------------------------------+ __kernel void Transpose(__global float *matrix_in, ///<[in] Input matrix __global float *matrix_out ///<[out] Output matrix ) { const int r = get_global_id(0); const int c = get_global_id(1); const int rows = get_global_size(0); const int cols = get_global_size(1); //--- matrix_out[c * rows + r] = matrix_in[r * cols + c]; } //+------------------------------------------------------------------+ //| Implements the mh2 attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MH2AttentionOut(__global float *q, ///<[in] Matrix of Querys __global float *kv, ///<[in] Matrix of Keys __global float *score, ///<[out] Matrix of Scores __global float *out, ///<[out] Matrix of attention int dimension, ///< Dimension of Key int heads_kv, int mask ///< 1 - calc only previous units, 0 - calc all ) { //--- init const int q_id = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_local_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_k = dimension * (2 * heads_kv * k + h_kv); const int shift_v = dimension * (2 * heads_kv * k + heads_kv + h_kv); const int shift_s = kunits * (q_id * heads + h) + k; const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); float koef = sqrt((float)dimension); if(koef < 1) koef = 1; __local float temp[LOCAL_ARRAY_SIZE]; //--- Score float sum = MIN_VALUE; if(mask == 0 || q_id >= k) { sum = 0; for(int d = 0; d < dimension; d++) sum += q[shift_q + d] * kv[shift_k + d]; } float sc = LocalSoftMax(IsNaNOrInf(sum / koef, MIN_VALUE), 1, temp); score[shift_s] = sc; //--- out for(int d = 0; d < dimension; d++) { BarrierLoc sum = LocalSum(IsNaNOrInf(kv[shift_v + d ] * sc, 0), 1, temp); //--- if(k == 0) out[shift_q + d] = sum; } } //+------------------------------------------------------------------+ //| Computes the mh2 attention inside gradients gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MH2AttentionInsideGradients(__global float *q, __global float *q_g, __global float *kv, __global float *kv_g, __global float *scores, __global float *gradient, int kunits, int heads_kv) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = q_id * kunits * heads + h * kunits; const int shift_g = h * dimension + d; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v = q_id; v < kunits; v += qunits) { float grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v; for(int g = 0; g < qunits; g++) grad += gradient[shift_g + dimension * (hq - h + g * heads)] * scores[shift_score + g * step_score]; } int shift_v = dimension * (2 * heads_kv * v + heads_kv + h) + d; kv_g[shift_v] = grad; } } //--- Calculating Query's gradients float grad = 0; float out_g = gradient[shift_g + q_id * dimension]; int shift_val = (heads_kv + h_kv) * dimension + d; int shift_key = h_kv * dimension + d; //--- for(int k = 0; k < kunits; k++) { float sc_g = 0; float sc = scores[shift_s + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_s + v] * out_g * kv[shift_val + 2 * v * heads_kv * dimension] * ((float)(k == v) - sc); grad += sc_g * kv[shift_key + 2 * k * heads_kv * dimension]; } q_g[shift_q] = grad / koef; //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k = q_id; k < kunits; k += qunits) { int shift_k = dimension * (2 * heads_kv * k + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq++) { int shift_score = hq * kunits + k; float val = kv[shift_k + heads_kv * dimension]; for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_sc + v] * gradient[shift_g + scr * dimension] * val * ((float)(k == v) - sc); grad += sc_g * q[shift_q + scr * dimension]; } } kv_g[shift_k] = grad / koef; } } } //+------------------------------------------------------------------+ //| Computes the cg conv hidden gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void CGConv_HiddenGradient(__global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_f, ///<[in] Previous layer Output tensor __global const float *matrix_s, ///<[in] Previous layer Output tensor __global float *matrix_fg, ///<[out] Tensor of gradients at previous layer __global float *matrix_sg, ///<[out] Tensor of gradients at previous layer const int activationf, ///< Activation type (#ENUM_ACTIVATION) const int activations ///< Activation type (#ENUM_ACTIVATION) ) { int i = get_global_id(0); //--- float grad = matrix_g[i]; float f = matrix_f[i]; float s = matrix_s[i]; //--- float sg = grad * f; float fg = grad * s; //--- matrix_fg[i] = Deactivation(fg, f, activationf); matrix_sg[i] = Deactivation(sg, s, activations); } //+------------------------------------------------------------------+ //| Runs the x ci t feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void XCiTFeedForward(__global float *qkv, __global float *score, __global float *out) { const size_t d = get_local_id(0); const size_t dimension = get_local_size(0); const size_t u = get_local_id(1); const size_t units = get_local_size(1); const size_t h = get_global_id(2); const size_t heads = get_global_size(2); //--- const uint ls_u = min((uint)units, (uint)LOCAL_ARRAY_SIZE); const uint ls_d = min((uint)dimension, (uint)LOCAL_ARRAY_SIZE); __local float q[LOCAL_ARRAY_SIZE][LOCAL_ARRAY_SIZE]; __local float k[LOCAL_ARRAY_SIZE][LOCAL_ARRAY_SIZE]; //--- Normalize Query and Key for(int cur_d = 0; cur_d < dimension; cur_d += ls_d) { float q_val = 0; float k_val = 0; //--- if(d < ls_d && (cur_d + d) < dimension && u < ls_u) { for(int count = u; count < units; count += ls_u) { int shift = count * dimension * heads * 3 + dimension * h + cur_d + d; q_val += qkv[shift] * qkv[shift]; k_val += qkv[shift + dimension * heads] * qkv[shift + dimension * heads]; } q[u][d] = q_val; k[u][d] = k_val; } BarrierLoc //--- uint count = ls_u; do { count = (count + 1) / 2; if(d < ls_d) { if(u < ls_u && u < count && (u + count) < units) { float q_val = q[u][d] + q[u + count][d]; float k_val = k[u][d] + k[u + count][d]; q[u + count][d] = 0; k[u + count][d] = 0; q[u][d] = q_val; k[u][d] = k_val; } } BarrierLoc } while(count > 1); //--- int shift = u * dimension * heads * 3 + dimension * h + cur_d; qkv[shift] = qkv[shift] / sqrt(q[0][d]); qkv[shift + dimension * heads] = qkv[shift + dimension * heads] / sqrt(k[0][d]); BarrierLoc } //--- Score int step = dimension * heads * 3; //--- for(int cur_r = 0; cur_r < dimension; cur_r += ls_u) { for(int cur_d = 0; cur_d < dimension; cur_d += ls_d) { if(u < ls_d && d < ls_d) q[u][d] = 0; BarrierLoc //--- if((cur_r + u) < ls_d && (cur_d + d) < ls_d) { int shift_q = dimension * h + cur_d + d; int shift_k = dimension * (heads + h) + cur_r + u; float scr = 0; for(int i = 0; i < units; i++) scr += qkv[shift_q + i * step] * qkv[shift_k + i * step]; scr = exp(scr / sqrt((float)units)); score[(cur_r + u) * dimension * heads + dimension * h + cur_d + d] = scr; q[u][d] += scr; } } BarrierLoc //--- int count = ls_d; do { count = (count + 1) / 2; if(u < ls_d) { if(d < ls_d && d < count && (d + count) < dimension) q[u][d] += q[u][d + count]; if(d + count < ls_d) q[u][d + count] = 0; } BarrierLoc } while(count > 1); //--- if((cur_r + u) < ls_d) score[(cur_r + u) * dimension * heads + dimension * h + d] /= q[u][0]; BarrierLoc } //--- int shift_out = dimension * (u * heads + h) + d; int shift_s = dimension * (heads * d + h); int shift_v = dimension * (heads * (u * 3 + 2) + h); float sum = 0; //--- for(int i = 0; i < dimension; i++) sum += qkv[shift_v + i] * score[shift_s + i]; out[shift_out] = sum; } //+------------------------------------------------------------------+ //| Computes the x ci t inside gradients gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void XCiTInsideGradients(__global float *qkv, __global float *qkv_g, __global float *scores, __global float *gradient) { //--- init const int q = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int units = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int shift_q = dimension * (heads * 3 * q + h); const int shift_k = dimension * (heads * (3 * q + 1) + h); const int shift_v = dimension * (heads * (3 * q + 2) + h); const int shift_g = dimension * (heads * q + h); int shift_score = dimension * h; int step_score = dimension * heads; //--- Calculating Value's gradients float sum = 0; //--- for(int i = 0; i < dimension; i++) sum += gradient[shift_g + i] * scores[shift_score + d + i * step_score]; qkv_g[shift_v + d] = sum; //--- Calculating Query's gradients float grad = 0; float val = qkv[shift_v + d]; //--- for(int k = 0; k < dimension; k++) { float sc_g = 0; float sc = scores[shift_score + k]; for(int v = 0; v < dimension; v++) sc_g += scores[shift_score + v] * val * gradient[shift_g + v * dimension] * ((float)(k == v) - sc); grad += sc_g * qkv[shift_k + k]; } qkv_g[shift_q + d] = grad / sqrt((float)units); //--- Calculating Key's gradients grad = 0; float out_g = gradient[shift_g]; //--- for(int scr = 0; scr < dimension; scr++) { float sc_g = 0; int shift_sc = scr * dimension * heads; float sc = scores[shift_sc + d]; for(int v = 0; v < dimension; v++) sc_g += scores[shift_sc + v] * out_g * qkv[shift_v + v] * ((float)(d == v) - sc); grad += sc_g * qkv[shift_q + scr]; } qkv_g[shift_k + d] = grad / sqrt((float)units); } //+------------------------------------------------------------------+ //| Runs the dot feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void DOTFeedForward(__global float *qkv, __global float *score, __global float *rpb, __global float *out) { const size_t d = get_local_id(0); const size_t dimension = get_local_size(0); const size_t u = get_global_id(1); const size_t units = get_global_size(1); const size_t h = get_global_id(2); const size_t heads = get_global_size(2); //--- uint step = 3 * dimension * heads; uint start = max((int)u - 1, 0); uint stop = min((int)u + 1, (int)units - 1); uint shift_q = u * step + h * dimension; uint shift_k = start * step + dimension * (heads + h); uint shift_score = u * 3 * heads; //--- const uint ls_d = min((uint)dimension, (uint)LOCAL_ARRAY_SIZE); __local float temp[LOCAL_ARRAY_SIZE][3]; //--- Score if(d < ls_d) { //--- for(uint pos = start; pos <= stop; pos++) temp[d][pos - start] = 0; //--- for(uint dim = d; dim < dimension; dim += ls_d) { float q = qkv[shift_q + dim]; for(uint pos = start; pos <= stop; pos++) { uint i = pos - start; temp[d][i] = temp[d][i] + q * qkv[shift_k + i * step + dim]; } } BarrierLoc //--- int count = ls_d; //--- do { count = (count + 1) / 2; if(d < count && (d + count) < dimension) for(uint i = 0; i <= (stop - start); i++) { temp[d][i] += temp[d + count][i]; temp[d + count][i] = 0; } BarrierLoc } while(count > 1); } //--- if(d == 0) { float sum = 0; //--- for(uint i = 0; i <= (stop - start); i++) { temp[0][i] = exp(temp[0][i] + rpb[shift_score + i]); sum += temp[0][i]; } //--- for(uint i = 0; i <= (stop - start); i++) { temp[0][i] = temp[0][i] / sum; score[shift_score + i] = temp[0][i]; } } BarrierLoc //--- int shift_out = dimension * (u * heads + h) + d; int shift_v = dimension * (heads * (u * 3 + 2) + h); float sum = 0; //--- for(uint i = 0; i <= (stop - start); i++) sum += qkv[shift_v + i] * temp[0][i]; out[shift_out] = sum; } //+------------------------------------------------------------------+ //| Computes the dot inside gradients gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void DOTInsideGradients(__global float *qkv, __global float *qkv_g, __global float *scores, __global float *rpb, __global float *rpb_g, __global float *gradient) { //--- init const uint u = get_global_id(0); const uint d = get_global_id(1); const uint h = get_global_id(2); const uint units = get_global_size(0); const uint dimension = get_global_size(1); const uint heads = get_global_size(2); //--- uint step = 3 * dimension * heads; uint start = max((int)u - 1, 0); uint stop = min((int)u + 1, (int)units - 1); const uint shift_q = u * step + dimension * h + d; const uint shift_k = u * step + dimension * (heads + h) + d; const uint shift_v = u * step + dimension * (2 * heads + h) + d; //--- Calculating Value's gradients float sum = 0; //--- for(uint i = start; i <= stop; i++) { int shift_score = i * 3 * heads; if(u == i) { shift_score += (uint)(u > 0); } else { if(u > i) shift_score += (uint)(start > 0) + 1; } uint shift_g = dimension * (i * heads + h) + d; sum += gradient[shift_g] * scores[shift_score]; } qkv_g[shift_v] = sum; //--- Calculating Query's gradients float grad = 0; uint shift_score = u * heads * 3; //--- for(int k = start; k <= stop; k++) { float sc_g = 0; float sc = scores[shift_score + k - start]; for(int v = start; v <= stop; v++) for(int dim = 0; dim < dimension; dim++) sc_g += scores[shift_score + v - start] * qkv[v * step + dimension * (2 * heads + h) + dim] * gradient[dimension * (u * heads + h) + dim] * ((float)(k == v) - sc); grad += sc_g * qkv[k * step + dimension * (heads + h) + d]; if(d == 0) rpb_g[shift_score + k - start] = sc_g; } qkv_g[shift_q] = grad; //--- Calculating Key's gradients grad = 0; //--- for(int q = start; q <= stop; q++) { float sc_g = 0; shift_score = q * heads * 3; if(u == q) shift_score += (uint)(u > 0); else { if(u > q) shift_score += (uint)(start > 0) + 1; } float sc = scores[shift_score]; for(int v = start; v <= stop; v++) { shift_score = v * heads * 3; if(u == v) shift_score += (uint)(u > 0); else { if(u > v) shift_score += (uint)(start > 0) + 1; } for(int dim = 0; dim < dimension; dim++) sc_g += scores[shift_score] * qkv[shift_v - d + dim] * gradient[dimension * (v * heads + h) + d] * ((float)(d == v) - sc); } grad += sc_g * qkv[q * step + dimension * h + d]; } qkv_g[shift_k] = grad; } //+------------------------------------------------------------------+ //| Updates the trainable weights of the rpb update adam step using | //| the optimizer rule. | //+------------------------------------------------------------------+ __kernel void RPBUpdateAdam(__global float *target, __global const float *gradient, __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int i = get_global_id(0); float m, v, weight; m = matrix_m[i]; v = matrix_v[i]; weight = target[i]; float g = gradient[i]; m = b1 * m + (1 - b1) * g; v = b2 * v + (1 - b2) * (g * g); float delta = m / (v != 0.0f ? sqrt(v) : 1.0f); target[i] = weight + delta; matrix_m[i] = m; matrix_v[i] = v; } //+------------------------------------------------------------------+ //| Runs the gte feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void GTEFeedForward(__global float *qkv, __global float *score, __global float *out, int dimension) { const size_t cur_q = get_global_id(0); const size_t units_q = get_global_size(0); const size_t cur_k = get_local_id(1); const size_t units_k = get_local_size(1); const size_t h = get_global_id(2); const size_t heads = get_global_size(2); //--- int shift_q = dimension * (cur_q + h * units_q); int shift_k = (cur_k + h * units_k + heads * units_q); int shift_v = dimension * (h * units_k + heads * (units_q + units_k)); int shift_score_con = units_k * (cur_q * 2 * heads + h) + cur_k; int shift_score_notcon = units_k * (cur_q * 2 * heads + heads + h) + cur_k; int shift_out_con = dimension * (cur_q + h * units_q); int shift_out_notcon = dimension * (cur_q + units_q * (h + heads)); //--- const uint ls_score = min((uint)units_k, (uint)LOCAL_ARRAY_SIZE); __local float local_score[LOCAL_ARRAY_SIZE][2]; //--- Score float scr = 0; //--- for(int d = 0; d < dimension; d++) scr += qkv[shift_q + d] * qkv[shift_k + d]; scr = exp(min(scr / sqrt((float)dimension), 30.0f)); if(cur_q == cur_k) { score[shift_score_con] = scr; score[shift_score_notcon] = scr; if(cur_k < ls_score) { local_score[cur_k][0] = scr; local_score[cur_k][1] = scr; } } else { if(abs(cur_q - cur_k) == 1) { score[shift_score_con] = scr; score[shift_score_notcon] = 0; if(cur_k < ls_score) { local_score[cur_k][0] = scr; local_score[cur_k][1] = 0; } } else { score[shift_score_con] = 0; score[shift_score_notcon] = scr; if(cur_k < ls_score) { local_score[cur_k][0] = 0; local_score[cur_k][1] = scr; } } } BarrierLoc //--- for(int k = ls_score; k < units_k; k += ls_score) { if((cur_k + k) < units_k) { local_score[cur_k][0] += score[shift_score_con + k]; local_score[cur_k][1] += score[shift_score_notcon + k]; } } BarrierLoc //--- int count = ls_score; do { count = (count + 1) / 2; if(cur_k < count) { if((cur_k + count) < units_k) { local_score[cur_k][0] += local_score[cur_k + count][0]; local_score[cur_k][1] += local_score[cur_k + count][1]; local_score[cur_k + count][0] = 0; local_score[cur_k + count][1] = 0; } } BarrierLoc } while(count > 1); BarrierLoc //--- score[shift_score_con] /= local_score[0][0]; score[shift_score_notcon] /= local_score[0][1]; BarrierLoc //--- shift_score_con -= cur_k; shift_score_notcon -= cur_k; //--- for(int d = 0; d < dimension; d += ls_score) { if((cur_k + d) < dimension) { float sum_con = 0; float sum_notcon = 0; for(int v = 0; v < units_k; v++) { sum_con += qkv[shift_v + v * dimension + cur_k + d] * score[shift_score_con + v]; sum_notcon += qkv[shift_v + v * dimension + cur_k + d] * score[shift_score_notcon + v]; } out[shift_out_con + cur_k + d] = sum_con; out[shift_out_notcon + cur_k + d] = sum_notcon; } } } //+------------------------------------------------------------------+ //| Computes the gte inside gradients gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void GTEInsideGradients(__global float *qkv, __global float *qkv_g, __global float *scores, __global float *gradient) { //--- init const uint u = get_global_id(0); const uint d = get_global_id(1); const uint h = get_global_id(2); const uint units = get_global_size(0); const uint dimension = get_global_size(1); const uint heads = get_global_size(2); //--- Calculating Value's gradients { int shift_out_con = dimension * h * units + d; int shift_out_notcon = dimension * units * (h + heads) + d; int shift_score_con = units * h + u; int shift_score_notcon = units * (heads + h) + u; int step_score = units * 2 * heads; int shift_v = dimension * (h * units + 2 * heads * units + u) + d; //--- float sum = 0; //--- for(uint i = 0; i <= units; i++) { sum += gradient[shift_out_con + i * dimension] * scores[shift_score_con + i * step_score]; sum += gradient[shift_out_notcon + i * dimension] * scores[shift_score_notcon + i * step_score]; } qkv_g[shift_v] = sum; } //--- Calculating Query's gradients { int shift_q = dimension * (u + h * units) + d; int shift_out_con = dimension * (h * units + u) + d; int shift_out_notcon = dimension * (u + units * (h + heads)) + d; int shift_score_con = units * h; int shift_score_notcon = units * (heads + h); int shift_v = dimension * (h * units + 2 * heads * units); float grad = 0; //--- for(int k = 0; k < units; k++) { int shift_k = (k + h * units + heads * units) + d; float sc_g = 0; float sc_con = scores[shift_score_con + k]; float sc_notcon = scores[shift_score_notcon + k]; for(int v = 0; v < units; v++) for(int dim = 0; dim < dimension; dim++) { sc_g += scores[shift_score_con + v] * qkv[shift_v + v * dimension + dim] * gradient[shift_out_con + dim] * ((float)(k == v) - sc_con); sc_g += scores[shift_score_notcon + v] * qkv[shift_v + v * dimension + dim] * gradient[shift_out_notcon + dim] * ((float)(k == v) - sc_notcon); } grad += sc_g * qkv[shift_k]; } qkv_g[shift_q] = grad; } //--- Calculating Key's gradients { int shift_k = (u + (h + heads) * units) + d; int shift_out_con = dimension * h * units + d; int shift_out_notcon = dimension * units * (h + heads) + d; int shift_score_con = units * h + u; int shift_score_notcon = units * (heads + h) + u; int step_score = units * 2 * heads; int shift_v = dimension * (h * units + 2 * heads * units); float grad = 0; //--- for(int q = 0; q < units; q++) { int shift_q = dimension * (q + h * units) + d; float sc_g = 0; float sc_con = scores[shift_score_con + u + q * step_score]; float sc_notcon = scores[shift_score_notcon + u + q * step_score]; for(int g = 0; g < units; g++) { for(int dim = 0; dim < dimension; dim++) { sc_g += scores[shift_score_con + g] * qkv[shift_v + u * dimension + dim] * gradient[shift_out_con + g * dimension + dim] * ((float)(u == g) - sc_con); sc_g += scores[shift_score_notcon + g] * qkv[shift_v + u * dimension + dim] * gradient[shift_out_notcon + g * dimension + dim] * ((float)(u == g) - sc_notcon); } } grad += sc_g * qkv[shift_q]; } qkv_g[shift_k] = grad; } } //+------------------------------------------------------------------+ //| Runs the feed forward nodef feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardNODEF(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_o, ///<[out] Output tensor int dimension, ///< input dimension float step, ///< h int activation ///< Activation type (#ENUM_ACTIVATION) ) { int d = get_global_id(0); int dimension_out = get_global_size(0); int v = get_global_id(1); int variables = get_global_size(1); int i = get_global_id(2); int lenth = get_global_size(2); //--- int shift = variables * i + v; int input_shift = shift * dimension; int output_shift = shift * dimension_out + d; int weight_shift = (v * dimension_out + d) * (dimension + 2); //--- float sum = matrix_w[dimension + 1 + weight_shift] + matrix_w[dimension + weight_shift] * step; //--- for(int w = 0; w < dimension; w++) sum += matrix_w[w + weight_shift] * matrix_i[input_shift + w]; //--- if(isnan(sum)) sum = 0; //--- matrix_o[output_shift] = fActivation(sum, activation); } //+------------------------------------------------------------------+ //| Runs the feed forward node inp k feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardNODEInpK(__global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_k1, ///<[in] K1 tensor __global float *matrix_k2, ///<[in] K2 tensor __global float *matrix_k3, ///<[in] K3 tensor __global float *matrix_k4, ///<[in] K4 tensor __global float *matrix_k5, ///<[in] K5 tensor __global float *matrix_k6, ///<[in] K6 tensor __global float *matrix_beta, ///<[in] beta tensor __global float *matrix_o ///<[out] Output tensor ) { int i = get_global_id(0); //--- float sum = matrix_i[i]; //--- for(int b = 0; b < 6; b++) { float beta = matrix_beta[b]; if(beta == 0.0f || isnan(beta)) continue; //--- float val = 0.0f; switch(b) { case 0: val = matrix_k1[i]; break; case 1: val = matrix_k2[i]; break; case 2: val = matrix_k3[i]; break; case 3: val = matrix_k4[i]; break; case 4: val = matrix_k5[i]; break; case 5: val = matrix_k6[i]; break; } if(val == 0.0f || isnan(val)) continue; //--- sum += val * beta; } //--- matrix_o[i] = sum; } //+------------------------------------------------------------------+ //| Computes the hidden gradient node inp k gradient contributions on| //| the GPU. | //+------------------------------------------------------------------+ __kernel void HiddenGradientNODEInpK(__global float *matrix_ig, ///<[in] Inputs tensor __global float *matrix_k1g, ///<[in] K1 tensor __global float *matrix_k2g, ///<[in] K2 tensor __global float *matrix_k3g, ///<[in] K3 tensor __global float *matrix_k4g, ///<[in] K4 tensor __global float *matrix_k5g, ///<[in] K5 tensor __global float *matrix_k6g, ///<[in] K6 tensor __global float *matrix_beta, ///<[in] beta tensor __global float *matrix_og ///<[out] Output tensor ) { int i = get_global_id(0); //--- float grad = IsNaNOrInf(matrix_og[i], 0); matrix_ig[i] = grad; //--- for(int b = 0; b < 6; b++) { float beta = IsNaNOrInf(matrix_beta[b], 0.0f); //--- float val = IsNaNOrInf(beta * grad, 0.0f); switch(b) { case 0: matrix_k1g[i] = val; break; case 1: matrix_k2g[i] = val; break; case 2: matrix_k3g[i] = val; break; case 3: matrix_k4g[i] = val; break; case 4: matrix_k5g[i] = val; break; case 5: matrix_k6g[i] = val; break; } } } //+------------------------------------------------------------------+ //| Computes the hidden gradient nodef gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void HiddenGradientNODEF(__global float *matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global float *matrix_g, ///<[in] Gradient tensor __global float *matrix_i, ///<[in] Inputs tensor __global float *matrix_ig, ///<[out] Inputs Gradient tensor int dimension_out, ///< output dimension int activation ///< Input Activation type (#ENUM_ACTIVATION) ) { int d = get_global_id(0); int dimension = get_global_size(0); int v = get_global_id(1); int variables = get_global_size(1); int i = get_global_id(2); int lenth = get_global_size(2); //--- int shift = variables * i + v; int input_shift = shift * dimension + d; int output_shift = shift * dimension_out; int weight_step = (dimension + 2); int weight_shift = (v * dimension_out) * weight_step + d; //--- float sum = 0; //--- for(int k = 0; k < dimension_out; k++) sum += matrix_g[output_shift + k] * matrix_w[weight_shift + k * weight_step]; if(isnan(sum)) sum = 0; //--- float out = matrix_i[input_shift]; //--- matrix_ig[input_shift] = Deactivation(sum, out, activation); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the nodef update weights adam | //| step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void NODEF_UpdateWeightsAdam(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_gk1, ///<[in] Tensor of gradients at k1 __global const float *matrix_gk2, ///<[in] Tensor of gradients at k2 __global const float *matrix_gk3, ///<[in] Tensor of gradients at k3 __global const float *matrix_gk4, ///<[in] Tensor of gradients at k4 __global const float *matrix_gk5, ///<[in] Tensor of gradients at k5 __global const float *matrix_gk6, ///<[in] Tensor of gradients at k6 __global const float *matrix_ik1, ///<[in] Inputs tensor __global const float *matrix_ik2, ///<[in] Inputs tensor __global const float *matrix_ik3, ///<[in] Inputs tensor __global const float *matrix_ik4, ///<[in] Inputs tensor __global const float *matrix_ik5, ///<[in] Inputs tensor __global const float *matrix_ik6, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum __global const float *alpha, ///< h const int lenth, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { const int d_in = get_global_id(0); const int dimension_in = get_global_size(0); const int d_out = get_global_id(1); const int dimension_out = get_global_size(1); const int v = get_global_id(2); const int variables = get_global_id(2); //--- const int weight_shift = (v * dimension_out + d_out) * dimension_in; const int input_step = variables * (dimension_in - 2); const int input_shift = v * (dimension_in - 2) + d_in; const int output_step = variables * dimension_out; const int output_shift = v * dimension_out + d_out; //--- float weight = matrix_w[weight_shift]; float g = 0; //--- for(int i = 0; i < lenth; i++) { int shift_g = i * output_step + output_shift; int shift_i = i * input_step + input_shift; switch(dimension_in - d_in) { case 1: g += matrix_gk1[shift_g] + matrix_gk2[shift_g] + matrix_gk3[shift_g] + matrix_gk4[shift_g] + matrix_gk5[shift_g] + matrix_gk6[shift_g]; break; case 2: g += matrix_gk1[shift_g] * alpha[0] + matrix_gk2[shift_g] * alpha[1] + matrix_gk3[shift_g] * alpha[2] + matrix_gk4[shift_g] * alpha[3] + matrix_gk5[shift_g] * alpha[4] + matrix_gk6[shift_g] * alpha[5]; break; default: g += matrix_gk1[shift_g] * matrix_ik1[shift_i] + matrix_gk2[shift_g] * matrix_ik2[shift_i] + matrix_gk3[shift_g] * matrix_ik3[shift_i] + matrix_gk4[shift_g] * matrix_ik4[shift_i] + matrix_gk5[shift_g] * matrix_ik5[shift_i] + matrix_gk6[shift_g] * matrix_ik6[shift_i]; break; } } //--- float mt = b1 * matrix_m[weight_shift] + (1 - b1) * g; float vt = b2 * matrix_v[weight_shift] + (1 - b2) * (g * g); float delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(fabs(delta) > 0) matrix_w[weight_shift] = matrix_w[weight_shift] + delta; matrix_m[weight_shift] = mt; matrix_v[weight_shift] = vt; } //+------------------------------------------------------------------+ //| Implements the time derivative kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void TimeDerivative(__global float *qkv, __global float *dqkv, int dimension) { const size_t pos = get_global_id(0); const size_t variable = get_global_id(1); const size_t head = get_global_id(2); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); const size_t heads = get_global_size(2); //--- const int shift = 3 * heads * variables * dimension; const int shift_query = pos * shift + (3 * variable * heads + head) * dimension; const int shift_key = shift_query + heads * dimension; //--- //--- for(int i = 0; i < dimension; i++) { //--- dQ/dt { int count = 0; float delta = 0; float value = qkv[shift_query + i]; if(pos > 0) { delta = value - qkv[shift_query + i - shift]; count++; } if(pos < (total - 1)) { delta += qkv[shift_query + i + shift] - value; count++; } if(count > 0) dqkv[shift_query + i] = delta / count; } //--- dK/dt { int count = 0; float delta = 0; float value = qkv[shift_key + i]; if(pos > 0) { delta = value - qkv[shift_key + i - shift]; count++; } if(pos < (total - 1)) { delta += qkv[shift_key + i + shift] - value; count++; } if(count > 0) dqkv[shift_key + i] = delta / count; } } } //+------------------------------------------------------------------+ //| Computes the hidden gradient time derivative gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void HiddenGradientTimeDerivative(__global float *qkv_g, __global float *dqkv_g, int dimension) { const size_t pos = get_global_id(0); const size_t variable = get_global_id(1); const size_t head = get_global_id(2); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); const size_t heads = get_global_size(2); //--- const int shift = 3 * heads * variables * dimension; const int shift_query = pos * shift + (3 * variable * heads + head) * dimension; const int shift_key = shift_query + heads * dimension; //--- //--- for(int i = 0; i < dimension; i++) { //--- dQ/dt { int count = 0; float grad = 0; float current = dqkv_g[shift_query + i]; if(pos > 0) { grad += current - dqkv_g[shift_query + i - shift]; count++; } if(pos < (total - 1)) { grad += dqkv_g[shift_query + i + shift] - current; count++; } if(count > 0) grad /= count; qkv_g[shift_query + i] += grad; } //--- dK/dt { int count = 0; float grad = 0; float current = dqkv_g[shift_key + i]; if(pos > 0) { grad += current - dqkv_g[shift_key + i - shift]; count++; } if(pos < (total - 1)) { grad += dqkv_g[shift_key + i + shift] - current; count++; } if(count > 0) grad /= count; qkv_g[shift_key + i] += dqkv_g[shift_key + i] + grad; } } } //+------------------------------------------------------------------+ //| Runs the feed forward cont att feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardContAtt(__global float *qkv, __global float *dqkv, __global float *score, __global float *out, int dimension, int heads) { const size_t query = get_global_id(0); const size_t key = get_global_id(1); const size_t variable = get_global_id(2); const size_t queris = get_global_size(0); const size_t keis = get_global_size(1); const size_t variables = get_global_size(2); //--- const uint ls_score = min((uint)keis, (uint)LOCAL_ARRAY_SIZE); __local float local_score[LOCAL_ARRAY_SIZE]; //--- //--- for(int head = 0; head < heads; head++) { const int shift = 3 * heads * variables * dimension; const int shift_query = query * shift + (3 * variable * heads + head) * dimension; const int shift_key = key * shift + (3 * variable * heads + heads + head) * dimension; const int shift_out = dimension * (heads * (query * variables + variable) + head); int shift_score = keis * (heads * (query * variables + variable) + head) + key; //--- Score float scr = 0; for(int d = 0; d < dimension; d++) scr += qkv[shift_query + d] * dqkv[shift_key + d] + qkv[shift_key + d] * dqkv[shift_query + d]; scr = exp(min(scr / sqrt((float)dimension), 30.0f)); score[shift_score] = scr; BarrierLoc //--- if(key < ls_score) { local_score[key] = scr; for(int k = ls_score + key; k < keis; k += ls_score) local_score[key] += score[shift_score + k]; } BarrierLoc //--- int count = ls_score; do { count = (count + 1) / 2; if(key < count) { if((key + count) < keis) { local_score[key] += local_score[key + count]; local_score[key + count] = 0; } } BarrierLoc } while(count > 1); //--- score[shift_score] /= local_score[0]; BarrierLoc //--- shift_score -= key; for(int d = key; d < dimension; d += keis) { float sum = 0; int shift_value = (3 * variable * heads + 2 * heads + head) * dimension + d; for(int v = 0; v < keis; v++) sum += qkv[shift_value + v * shift] * score[shift_score + v]; out[shift_out + d] = sum; } BarrierLoc } //--- } //+------------------------------------------------------------------+ //| Computes the hidden gradient cont att gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void HiddenGradientContAtt(__global float *qkv, __global float *qkv_g, __global float *dqkv, __global float *dqkv_g, __global float *score, __global float *out_g, const int dimension) { const size_t pos = get_global_id(0); const size_t variable = get_global_id(1); const size_t head = get_global_id(2); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); const size_t heads = get_global_size(2); //--- Value gradient { const int shift_value = dimension * (heads * (3 * variables * pos + 3 * variable + 2) + head); const int shift_out = dimension * (head + variable * heads); const int shift_score = total * (variable * heads + head); const int step_out = variables * heads * dimension; const int step_score = variables * heads * total; //--- //--- for(int d = 0; d < dimension; d++) { float sum = 0; for(int g = 0; g < total; g++) sum += out_g[shift_out + g * step_out + d] * score[shift_score + g * step_score]; qkv_g[shift_value + d] = sum; } } //--- Query gradient { const int shift_out = dimension * (heads * (pos * variables + variable) + head); const int step = 3 * variables * heads * dimension; const int shift_query = dimension * (3 * heads * variable + head) + pos * step; const int shift_key = dimension * (heads * (3 * variable + 1) + head); const int shift_value = dimension * (heads * (3 * variable + 2) + head); const int shift_score = total * (heads * (pos * variables + variable) + head); //--- Score gradient //--- for(int k = 0; k < total; k++) { float score_grad = 0; float scr = score[shift_score + k]; for(int v = 0; v < total; v++) { float grad = 0; for(int d = 0; d < dimension; d++) grad += qkv[shift_value + v * step + d] * out_g[shift_out + d]; score_grad += score[shift_score + v] * grad * ((float)(pos == v) - scr); } score_grad /= sqrt((float)dimension); //--- Query gradient for(int d = 0; d < dimension; d++) { if(k == 0) { dqkv_g[shift_query + d] = score_grad * qkv[shift_key + k * step + d]; qkv_g[shift_query + d] = score_grad * dqkv[shift_key + k * step + d]; } else { dqkv_g[shift_query + d] += score_grad * qkv[shift_key + k * step + d]; qkv_g[shift_query + d] += score_grad * dqkv[shift_key + k * step + d]; } } } } //--- Key gradient { const int shift_key = dimension * (heads * (3 * variables * pos + 3 * variable + 1) + head); const int shift_out = dimension * (head + variable * heads); const int step_out = variables * heads * dimension; const int step = 3 * variables * heads * dimension; const int shift_query = dimension * (3 * heads * variable + head); const int shift_value = dimension * (heads * (3 * variable + 2) + head) + pos * step; const int shift_score = total * (heads * variable + head); const int step_score = variables * heads * total; //--- Score gradient //--- for(int q = 0; q < total; q++) { float score_grad = 0; float scr = score[shift_score + q * step_score]; for(int g = 0; g < total; g++) { float grad = 0; for(int d = 0; d < dimension; d++) grad += qkv[shift_value + d] * out_g[shift_out + d + g * step_out] / sqrt((float)dimension); score_grad += score[shift_score + q * step_score + g] * grad * ((float)(q == pos) - scr); } //--- Key gradient for(int d = 0; d < dimension; d++) { if(q == 0) { dqkv_g[shift_key + d] = qkv[shift_query + q * step + d] * score_grad; qkv_g[shift_key + d] = score_grad * dqkv[shift_query + q * step + d]; } else { qkv_g[shift_key + d] += score_grad * dqkv[shift_query + q * step + d]; dqkv_g[shift_key + d] += score_grad * qkv[shift_query + q * step + d]; } } } } } //+------------------------------------------------------------------+ //| Runs the rev in feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void RevInFeedForward(__global float *inputs, __global float *options, __global float *output, int options_size, int optimization) { int n = get_global_id(0); int shift = (n * (optimization == 0 ? 7 : 9)) % options_size; //--- float mean = options[shift]; float variance = options[shift + 1]; float k = options[shift + 3]; //--- float res = 0; res = sqrt(variance) * (inputs[n] - options[shift + 4]) / fmax(k, 0.001f) + mean; if(isnan(res)) res = 0; //--- output[n] = res; } //+------------------------------------------------------------------+ //| Implements the rev in hidden graddient kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RevInHiddenGraddient(__global float *inputs, __global float *inputs_gr, __global float *options, __global float *output_gr, int options_size, int optimization, int activation) { int n = get_global_id(0); int shift = (n * (optimization == 0 ? 7 : 9)) % options_size; //--- float variance = options[shift + 1]; float inp = inputs[n]; float k = options[shift + 3]; //--- float res = sqrt(variance) * output_gr[n]; if(fabs(k) > 1) res /= k; if(isnan(res)) res = 0; //--- inputs_gr[n] = Deactivation(res, inp, activation); } //+------------------------------------------------------------------+ //| Implements the activation kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void Activation(__global const float *inputs, __global float *outputs, const int activation) { int n = get_global_id(0); //--- float res = IsNaNOrInf(inputs[n], 0); //--- outputs[n] = fActivation(res, activation); } //+------------------------------------------------------------------+ //| Implements the de activation kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DeActivation(__global const float *inputs, __global float *inputs_gr, __global const float *output_gr, const int activation) { int n = get_global_id(0); //--- float inp = inputs[n]; float res = IsNaNOrInf(output_gr[n], 0); //--- inputs_gr[n] = Deactivation(res, inp, activation); } //+------------------------------------------------------------------+ //| Implements the patch create kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PatchCreate(__global float *inputs, __global float *weights, __global float *outputs, int inputs_total, int window_in, int step, int activation ) { const int i = get_global_id(0); const int w = get_global_id(1); const int v = get_global_id(2); const int window_out = get_global_size(1); const int variables = get_global_size(2); //--- const int shift_in = i * step * variables + v; const int shift_out = (i * variables + v) * window_out + w; const int shift_weights = (window_in + 1) * (v * window_out + w); //--- float res = weights[shift_weights + window_in]; //--- for(int p = 0; p < window_in; p++) if((shift_in + p * variables) < inputs_total) res += inputs[shift_in + p * variables] * weights[shift_weights + p]; if(isnan(res)) res = 0; //--- outputs[shift_out] = fActivation(res, activation); } //+------------------------------------------------------------------+ //| Computes the patch hidden gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void PatchHiddenGradient(__global float *inputs, __global float *inputs_gr, __global float *weights, __global float *outputs_gr, int window_in, int step, int window_out, int outputs_total, int activation ) { const int i = get_global_id(0); const int v = get_global_id(1); const int variables = get_global_size(1); //--- const int w_start = i % step; const int r_start = max((i - window_in + step) / step, 0); int total = (window_in - w_start + step - 1) / step; total = min((i + step) / step, total); //--- float grad = 0; //--- for(int p = 0; p < total; p ++) { int row = r_start + p; if(row >= outputs_total) break; for(int wo = 0; wo < window_out; wo++) { int shift_g = (row * variables + v) * window_out + wo; int shift_w = v * (window_in + 1) * window_out + w_start + (total - p - 1) * step + wo * (window_in + 1); grad += outputs_gr[shift_g] * weights[shift_w]; } } //--- float inp = inputs[i * variables + v]; //--- inputs_gr[i * variables + v] = Deactivation(grad, inp, activation); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the patch update weights adam | //| step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void PatchUpdateWeightsAdam(__global float *weights, __global const float *outputs_gr, __global const float *inputs, __global float *weights_m, __global float *weights_v, const int inputs_total, const float l, const float b1, const float b2, int step ) { const int c = get_global_id(0); const int r = get_global_id(1); const int v = get_global_id(2); const int window_in = get_global_size(0) - 1; const int window_out = get_global_size(1); const int variables = get_global_size(2); //--- const int start_input = c * variables + v; const int step_input = step * variables; const int start_out = v * window_out + r; const int step_out = variables * window_out; const int total = inputs_total / (variables * step); //--- float grad = 0; //--- for(int p = 0; p < total; p++) { int i = start_input + i * step_input; int o = start_out + i * step_out; grad += (c == window_in ? 1 : inputs[i]) * outputs_gr[0]; } if(isnan(grad)) grad = 0; //--- const int shift_weights = (window_in + 1) * (window_out * v + r) + c; //--- float weight = weights[shift_weights]; float mt = b1 * weights_m[shift_weights] + (1 - b1) * grad; float vt = b2 * weights_v[shift_weights] + (1 - b2) * (grad * grad); float delta = l * (mt / (sqrt(vt) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(fabs(delta) > 0) weights[shift_weights] = weight + delta; weights_m[shift_weights] = mt; weights_v[shift_weights] = vt; } //+------------------------------------------------------------------+ //| Implements the mat mult kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MatMult(__global const float *matr1, __global const float *matr2, __global float *result, int dimension, int multvarsecond, int offset_matr1, int offset_matr2, int offset_result, __global float *numeric_flags, const int flags_enabled) { size_t row = get_global_id(0); size_t col = get_global_id(1); size_t var = get_global_id(2); size_t rows = get_global_size(0); size_t cols = get_global_size(1); //--- int shift1 = offset_matr1 + RCtoFlat(row, 0, rows, dimension, var); int shift2 = offset_matr2 + RCtoFlat(0, col, dimension, cols, multvarsecond * var); int shift_out = offset_result + RCtoFlat(row, col, rows, cols, var); //--- float res = 0; int invalid = 0; //--- for(int i = 0; i < dimension; i++) { if(flags_enabled) { const float value1 = matr1[shift1 + i]; const float value2 = matr2[shift2 + i * cols]; const float product = value1 * value2; invalid |= isnan(value1) || isinf(value1) || isnan(value2) || isinf(value2) || isnan(product) || isinf(product); } //--- Preserve the legacy sanitized accumulation after checking its raw operands. res += IsNaNOrInf(matr1[shift1 + i] * matr2[shift2 + i * cols], 0); if(flags_enabled) invalid |= isnan(res) || isinf(res); } //--- if(flags_enabled) { invalid |= isnan(res) || isinf(res); numeric_flags[RCtoFlat(row, col, rows, cols, var)] = (invalid ? 1.0f : 0.0f); } result[shift_out] = res; } //+------------------------------------------------------------------+ //| Computes the mat mult grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MatMultGrad(__global const float *matr1, __global float *matr1_gr, __global const float *matr2, __global float *matr2_gr, __global const float *result_gr, int dimension, int multvarsecond, int offset_matr1, int offset_matr1_gr, int offset_matr2, int offset_matr2_gr, int offset_result_gr) { size_t row = get_global_id(0); size_t col = get_global_id(1); size_t var = get_global_id(2); size_t rows = get_global_size(0); size_t cols = get_global_size(1); //--- Tensor offsets select the current row and variable views. int shift1 = offset_matr1 + (row + var * rows) * dimension; int shift1_gr = offset_matr1_gr + (row + var * rows) * dimension; int shift2 = offset_matr2 + var * dimension * cols * multvarsecond; int shift2_gr = offset_matr2_gr + var * dimension * cols * multvarsecond; int shift_out = offset_result_gr + (row + var * rows) * cols; //--- First-matrix gradients use disjoint full-feature lanes. for(int k = 0; k * cols < dimension; k++) { int feature = col + k * cols; if(feature >= dimension) continue; float grad = 0; for(int i = 0; i < cols; i++) grad += IsNaNOrInf(result_gr[shift_out + i] * matr2[shift2 + feature * cols + i], 0); matr1_gr[shift1_gr + feature] = IsNaNOrInf(grad, 0); } //--- Second-matrix gradients reduce rows and all shared variables per entry. for(int k = 0; k * rows < dimension; k++) { int feature = row + k * rows; if(feature >= dimension) continue; if(multvarsecond || var == 0) { float grad = 0; if(multvarsecond) { shift_out = offset_result_gr + var * rows * cols + col; shift1 = offset_matr1 + var * rows * dimension + feature; for(int i = 0; i < rows; i++) grad += IsNaNOrInf(result_gr[shift_out + i * cols] * matr1[shift1 + i * dimension], 0); } else { for(int v = 0; v < get_global_size(2); v++) { shift_out = offset_result_gr + v * rows * cols + col; shift1 = offset_matr1 + v * rows * dimension + feature; for(int i = 0; i < rows; i++) grad += IsNaNOrInf(result_gr[shift_out + i * cols] * matr1[shift1 + i * dimension], 0); } } matr2_gr[shift2_gr + col + feature * cols] = IsNaNOrInf(grad, 0); } } } //+------------------------------------------------------------------+ //| Runs the fft kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void FFT(__global float *inputs_re, __global float *inputs_im, __global float *outputs_re, __global float *outputs_im, const int input_window, const int input_complex, const int output_window, const int reverse ) { size_t variable = get_global_id(0); //--- const ulong N = output_window; const ulong N2 = N / 2; const ulong inp_shift = input_window * variable; const ulong out_shift = output_window * variable; //--- uint target = 0; //--- for(uint position = 0; position < N; position++) { if(target > position) { outputs_re[out_shift + position] = (target < input_window ? inputs_re[inp_shift + target] : 0); outputs_im[out_shift + position] = ((target < input_window && input_complex) ? inputs_im[inp_shift + target] : 0); outputs_re[out_shift + target] = inputs_re[inp_shift + position]; outputs_im[out_shift + target] = (input_complex ? inputs_im[inp_shift + position] : 0); } else { outputs_re[out_shift + position] = inputs_re[inp_shift + position]; outputs_im[out_shift + position] = (input_complex ? inputs_im[inp_shift + position] : 0); } unsigned int mask = N; while(target & (mask >>= 1)) target &= ~mask; target |= mask; } float real = 0, imag = 0; //--- for(int len = 2; len <= (int)N; len <<= 1) { float w_real = (float)cos(2 * M_PI_F / len); float w_imag = (float)sin(2 * M_PI_F / len); for(int i = 0; i < (int)N; i += len) { float cur_w_real = 1; float cur_w_imag = 0; for(int j = 0; j < len / 2; j++) { real = cur_w_real * outputs_re[out_shift + i + j + len / 2] - cur_w_imag * outputs_im[out_shift + i + j + len / 2]; imag = cur_w_imag * outputs_re[out_shift + i + j + len / 2] + cur_w_real * outputs_im[out_shift + i + j + len / 2]; outputs_re[out_shift + i + j + len / 2] = outputs_re[out_shift + i + j] - real; outputs_im[out_shift + i + j + len / 2] = outputs_im[out_shift + i + j] - imag; outputs_re[out_shift + i + j] += real; outputs_im[out_shift + i + j] += imag; real = cur_w_real * w_real - cur_w_imag * w_imag; cur_w_imag = cur_w_imag * w_real + cur_w_real * w_imag; cur_w_real = real; } } } //--- if(reverse) { outputs_re[0] /= N; outputs_im[0] /= N; outputs_re[N2] /= N; outputs_im[N2] /= N; //--- for(int i = 1; i < N2; i++) { real = outputs_re[i] / N; imag = outputs_im[i] / N; outputs_re[i] = outputs_re[N - i] / N; outputs_im[i] = outputs_im[N - i] / N; outputs_re[N - i] = real; outputs_im[N - i] = imag; } } } //+------------------------------------------------------------------+ //| Implements the complex layer kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexLayer(__global float *inputs_re, __global float *inputs_im, __global float *outputs_re, __global float *outputs_im ) { size_t i = get_global_id(0); size_t j = get_global_id(1); size_t total_i = get_global_size(0); size_t total_j = get_global_size(1); uint shift = i * total_j + j; //--- outputs_re[shift] = inputs_re[shift] - inputs_im[shift]; outputs_im[shift] = inputs_im[shift] + inputs_re[shift]; } //+------------------------------------------------------------------+ //| Computes the complex layer gradient gradient contributions on the| //| GPU. | //+------------------------------------------------------------------+ __kernel void ComplexLayerGradient(__global float *inputs_re, __global float *inputs_im, __global float *outputs_re, __global float *outputs_im ) { size_t i = get_global_id(0); size_t j = get_global_id(1); size_t total_i = get_global_size(0); size_t total_j = get_global_size(1); uint shift = i * total_j + j; //--- inputs_re[shift] = outputs_re[shift] + outputs_im[shift]; inputs_im[shift] = outputs_im[shift] - outputs_re[shift]; } //+------------------------------------------------------------------+ //| Computes the gradient msa gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void GradientMSA(__global float *matrix_t, ///<[in] Target tensor __global float *matrix_o, ///<[in] Forecast tensor __global float *matrix_g ///<[out] Tensor of gradients ) { int i = get_global_id(0); matrix_g[i] = matrix_t[i] - matrix_o[i]; } //+------------------------------------------------------------------+ //| Computes the cumulative gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void CumulativeGradient(__global float *gradient1, __global float *gradient2, __global float *gradient_out, float alpha ) { int i = get_global_id(0); gradient_out[i] = alpha * gradient1[i] + (1 - alpha) * gradient2[i]; } //+------------------------------------------------------------------+ //| Implements the complex mul helper on the GPU. | //+------------------------------------------------------------------+ inline float2 ComplexMul(const float2 a, const float2 b) { float2 result = 0; result.x = IsNaNOrInf(a.x * b.x - a.y * b.y, 0); result.y = IsNaNOrInf(a.x * b.y + a.y * b.x, 0); return result; } //+------------------------------------------------------------------+ //| Implements the complex div helper on the GPU. | //+------------------------------------------------------------------+ inline float2 ComplexDiv(const float2 a, const float2 b) { float2 result = 0; float z = IsNaNOrInf(b.x * b.x + b.y * b.y, 1); if(z > 0) { result.x = IsNaNOrInf(a.x * b.x + a.y * b.y, 0) / z; result.y = IsNaNOrInf(a.y * b.x - a.x * b.y, 0) / z; } return result; } //+------------------------------------------------------------------+ //| Implements the complex abs helper on the GPU. | //+------------------------------------------------------------------+ inline float ComplexAbs(float2 a) { return sqrt(IsNaNOrInf(a.x * a.x + a.y * a.y, 0)); } //+------------------------------------------------------------------+ //| Implements the complex sqrt helper on the GPU. | //+------------------------------------------------------------------+ inline float2 ComplexSqrt(float2 a) { float2 result = 0; float z = ComplexAbs(a); result.x = sqrt((z + IsNaNOrInf(a.x, 0)) / 2); result.y = sqrt((z - IsNaNOrInf(a.x, 0)) / 2); if(a.y < 0) result.y *= (-1); //--- return result; } //+------------------------------------------------------------------+ //| Implements the complex exp helper on the GPU. | //+------------------------------------------------------------------+ inline float2 ComplexExp(float2 a) { float2 result = exp(clamp(IsNaNOrInf(a.x, 0), -20.0f, 20.0f)); result.x *= IsNaNOrInf(cos(a.y), 0); result.y *= IsNaNOrInf(sin(a.y), 0); return result; } //+------------------------------------------------------------------+ //| Implements the complex tanh helper on the GPU. | //+------------------------------------------------------------------+ inline float2 ComplexTanh(float2 a) { float sinh_re = sinh(a.x); float cosh_re = cosh(a.x); float sin_im = sin(a.y); float cos_im = cos(a.y); //--- float2 sinh_a = 0; float2 cosh_a = 0; sinh_a.x = sinh_re * cos_im; sinh_a.y = cosh_re * sin_im; cosh_a.x = cosh_re * cos_im; cosh_a.y = sinh_re * sin_im; //--- return ComplexDiv(sinh_a, cosh_a); } //+------------------------------------------------------------------+ //| Runs the feed forward complex conv feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardComplexConv(__global const float2* __attribute__((aligned(8))) matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global const float2* __attribute__((aligned(8))) matrix_i, ///<[in] Inputs tensor __global float2* __attribute__((aligned(8))) matrix_o, ///<[out] Output tensor const int inputs, ///< Number of inputs const int step, ///< Step size const int window_in, ///< Size of input window const int activation ///< Activation type (#ENUM_ACTIVATION) ) { const size_t i = get_global_id(0); const size_t units = get_global_size(0); const size_t out = get_global_id(1); const size_t w_out = get_global_size(1); const size_t var = get_global_id(2); const size_t variables = get_global_size(2); //--- int w_in = window_in; int shift_out = w_out * (i + units * var); int shift_in = step * i; int shift = (w_in + 1) * (out + var * w_out); int stop = (w_in <= (inputs - shift_in) ? w_in : (inputs - shift_in)); shift_in += + inputs * var; //--- float2 sum = ComplexMul((float2)(1, 0), matrix_w[shift + w_in]); //--- for(int k = 0; k < stop; k ++) sum += IsNaNOrInf2(ComplexMul(matrix_i[shift_in + k], matrix_w[shift + k]), (float2)0); //--- switch(activation) { case 0: sum = ComplexTanh(sum); break; case 1: sum = ComplexDiv((float2)(1, 0), (float2)(1, 0) + ComplexExp(-sum)); break; case 2: if(sum.x < 0) { sum.x *= 0.01f; sum.y *= 0.01f; } break; default: break; } matrix_o[out + shift_out] = sum; } //+------------------------------------------------------------------+ //| Computes the calc hidden gradient complex conv gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientComplexConv(__global const float2* __attribute__((aligned(8))) matrix_w, ///<[in] Weights matrix (m+1)*n, where m - input ///< window and n - output window __global const float2* __attribute__((aligned(8))) matrix_g, ///<[in] Tensor of gradients at current layer __global const float2* __attribute__((aligned(8))) matrix_o, ///<[in] Output tensor __global float2* __attribute__((aligned(8))) matrix_ig, ///<[out] Tensor of gradients at previous layer const int outputs, ///< Number of outputs const int step, ///< Step size const int window_in, ///< Size of input window const int window_out, ///< Size of output window const int activation, ///< Activation type (#ENUM_ACTIVATION) const int shift_out ///< Shift in output and gradient buffer ) { const size_t i = get_global_id(0); const size_t inputs = get_global_size(0); const size_t var = get_global_id(1); const size_t variables = get_global_size(1); //--- float2 sum = (float2)0; float2 out = matrix_o[i]; int start = i - window_in + step; start = max((start - start % step) / step, 0) + var * inputs; int stop = (i + step - 1) / step; if(stop > (outputs / window_out)) stop = outputs / window_out; stop += var * outputs; //--- for(int h = 0; h < window_out; h ++) { for(int k = start; k < stop; k++) { int shift_g = k * window_out + h; int shift_w = (stop - k - 1) * step + i % step + h * (window_in + 1); if(shift_g >= outputs || shift_w >= (window_in + 1) * window_out) break; sum += ComplexMul(matrix_g[shift_out + shift_g], matrix_w[shift_w]); } } sum = IsNaNOrInf2(sum, (float2)0); //--- switch(activation) { case 0: sum = ComplexMul(sum, (float2)1.0f - ComplexMul(out, out)); break; case 1: sum = ComplexMul(sum, ComplexMul(out, (float2)1.0f - out)); break; case 2: if(out.x < 0.0f) { sum.x *= 0.01f; sum.y *= 0.01f; } break; default: break; } matrix_ig[i] = clamp(sum, (float2)(-MAX_GRAD), (float2)MAX_GRAD); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights complex conv | //| momentum step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsComplexConvMomentum(__global float2* __attribute__((aligned(8))) matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global float2* __attribute__((aligned(8))) matrix_g, ///<[in] Tensor of gradients at current layer __global float2* __attribute__((aligned(8))) matrix_i, ///<[in] Inputs tensor __global float2* __attribute__((aligned(8))) matrix_dw, ///<[in,out] Matrix of delta weights in last correction int inputs, ///< Number of inputs float learning_rates, ///< Learning rates float momentum, ///< Momentum multiplier int window_in, ///< Size of input window int window_out, ///< Size of output window int step ///< Step size ) { const size_t i = get_global_id(0); const size_t total_w = get_global_size(0); const size_t var = get_global_id(1); const size_t variables = get_global_size(1); const int shift = i % (window_in + 1); int shift_out = (i - shift) / (window_in + 1); int total = (inputs - window_in) % step; total = (inputs - window_in - total) / step + (total > 0 ? 1 : 0); shift_out += total * window_out * var; float2 grad = 0; //--- for(int t = 0; t < total; t++) { if(shift != window_in && (shift + t * window_in) >= inputs) break; grad += ComplexMul(matrix_g[t * window_out + shift_out], (shift == window_in ? (float2)(1, 0) : matrix_i[inputs * var + shift + t * step])); } float2 delta = ComplexMul((float2)(learning_rates, 0), clamp(grad, (float2) - MAX_GRAD, (float2)MAX_GRAD)) + ComplexMul((float2)(momentum, 0), matrix_dw[i + total_w * var]); if(!(isnan(delta.x) || isnan(delta.y) || isinf(delta.x) || isinf(delta.y))) { matrix_dw[i + total_w * var] = delta; matrix_w[i + total_w * var] = matrix_w[i + total_w * var] + delta; } } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights complex conv | //| adam step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsComplexConvAdam(__global float2* __attribute__((aligned(8))) matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global const float2* __attribute__((aligned(8))) matrix_g, ///<[in] Tensor of gradients at current layer __global const float2* __attribute__((aligned(8))) matrix_i, ///<[in] Inputs tensor __global float2* __attribute__((aligned(8))) matrix_m, ///<[in] Matrix of first momentum __global float2* __attribute__((aligned(8))) matrix_v, ///<[in] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2, ///< Second momentum multiplier int window_in, ///< Size of input window int window_out, ///< Size of output window int step ///< Step size ) { const size_t i = get_global_id(0); const size_t total_w = get_global_size(0); const size_t var = get_global_id(1); const size_t variables = get_global_size(1); //--- const int shift = i % (window_in + 1); int shift_out = (i - shift) / (window_in + 1); int total = (inputs - window_in + step - 1) / step; shift_out += total * window_out * var; const int shift_var_in = var * inputs; const int shift_var_out = var * total * window_out; //--- float2 grad = 0; //--- for(int t = 0; t < total; t++) { if(shift != window_in && (shift + t * window_in) >= inputs) break; grad += IsNaNOrInf2(ComplexMul(matrix_g[t * window_out + shift_out + shift_var_out], (shift == window_in ? (float2)(1, 0) : matrix_i[shift + t * step + shift_var_in])), (float2)0); } grad = clamp(grad, (float2) - MAX_GRAD, (float2)MAX_GRAD); float2 mt = IsNaNOrInf2(b1 * matrix_m[i + total_w * var] + (1 - b1) * grad, (float2)0); float2 vt = IsNaNOrInf2(b2 * matrix_v[i + total_w * var] + (1 - b2) * ComplexMul(grad, grad), (float2)(1.0e-6f, 0)); float2 weight = matrix_w[i + total_w * var] + IsNaNOrInf2(l * ComplexDiv(mt, ComplexSqrt(vt)), (float2)0); matrix_w[i + total_w * var] = weight; matrix_m[i + total_w * var] = mt; matrix_v[i + total_w * var] = vt; } //+------------------------------------------------------------------+ //| Runs the complex soft max feed forward feed-forward pass on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void ComplexSoftMax_FeedForward(__global float2* __attribute__((aligned(8))) inputs, __global float2* __attribute__((aligned(8))) outputs, const int total) { const uint i = (uint)get_global_id(0); const uint l = (uint)get_local_id(0); const uint h = (uint)get_global_id(1); const uint ls = min((uint)get_local_size(0), (uint)LOCAL_ARRAY_SIZE); uint shift_head = h * total; //--- __local float2 temp[LOCAL_ARRAY_SIZE]; uint count = 0; if(l < ls) do { uint shift = shift_head + count * ls + l; if(shift < ((h + 1) * total)) temp[l].x = (count > 0 ? fmax(ComplexAbs(inputs[shift]), temp[l].x) : ComplexAbs(inputs[shift])); count++; } while((count * ls + l) < total); BarrierLoc float max_value = temp[0].x; //--- for(int i = 1; i < ls; i++) max_value = fmax(max_value, temp[i].x); //--- count = 0; if(l < ls) do { uint shift = shift_head + count * ls + l; temp[l] = (count > 0 ? temp[l] : (float2)0) + (shift < ((h + 1) * total) ? ComplexExp(ComplexDiv(inputs[shift], (float2)(max_value, 0))) : (float2)0); count++; } while((count * ls + l) < total); BarrierLoc count = min(ls, (uint)total); do { count = (count + 1) / 2; if(l < ls) temp[l] += (l < count && (l + count) < total ? temp[l + count] : (float2)0); if(l + count < ls) temp[l + count] = (float2)0; BarrierLoc } while(count > 1); //--- float2 sum = temp[0]; if(ComplexAbs(sum) > 0) { count = 0; while((count * ls + l) < total) { uint shift = shift_head + count * ls + l; if(shift < ((h + 1) * total)) outputs[shift] = ComplexDiv(ComplexExp(ComplexDiv(inputs[shift], (float2)(max_value, 0))), sum); count++; } } } //+------------------------------------------------------------------+ //| Computes the complex soft max hidden gradient gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexSoftMax_HiddenGradient(__global float2* __attribute__((aligned(8))) outputs, __global float2* __attribute__((aligned(8))) output_gr, __global float2* __attribute__((aligned(8))) input_gr) { size_t i = get_global_id(0); size_t outputs_total = get_global_size(0); size_t h = get_global_id(1); uint shift = h * outputs_total; float2 output = outputs[shift + i]; float2 result = 0; //--- for(int j = 0; j < outputs_total; j++) result += ComplexMul(ComplexMul(outputs[shift + j], output_gr[shift + j]), ((i == j ? (float2)(1, 0) : (float2)0) - output)); input_gr[shift + i] = result; } //+------------------------------------------------------------------+ //| Computes the complex soft max output gradient gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexSoftMax_OutputGradient(__global float2* __attribute__((aligned(8))) outputs, __global float2* __attribute__((aligned(8))) targets, __global float2* __attribute__((aligned(8))) output_gr) { size_t i = get_global_id(0); if(ComplexAbs(outputs[i]) == 0) output_gr[i] = (float2)0; else output_gr[i] = ComplexDiv(targets[i], outputs[i]); } //+------------------------------------------------------------------+ //| Implements the complex mh attention score kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexMHAttentionScore(__global float2* __attribute__((aligned(8))) qkv, ///<[in] Matrix of Querys, Keys, Values __global float2* __attribute__((aligned(8))) score, ///<[out] Matrix of Scores int dimension, ///< Dimension of Key int mask ///< 1 - calc only previous units, 0 - calc all ) { int q = get_global_id(0); int h = get_global_id(1); int units = get_global_size(0); int heads = get_global_size(1); //--- int shift_q = dimension * (h + 3 * q * heads); int shift_s = units * (h + q * heads); //--- float2 koef = (float2)(sqrt((float)dimension), 0); if(koef.x < 1) koef.x = 1; float2 sum = 0; //--- for(int k = 0; k < units; k++) { if(mask > 0 && k > q) { score[shift_s + k] = (float2)0; continue; } float2 result = (float2)0; int shift_k = dimension * (h + heads * (3 * k + 1)); for(int i = 0; i < dimension; i++) result += ComplexMul(qkv[shift_q + i], qkv[shift_k + i]); result = ComplexExp(ComplexDiv(result, koef)); if(isnan(result.x) || isnan(result.y) || isinf(result.x) || isinf(result.y)) result = (float2)0; score[shift_s + k] = result; sum += result; } if(ComplexAbs(sum) > 0) { //--- for(int k = 0; k < units; k++) score[shift_s + k] = ComplexDiv(score[shift_s + k], sum); } } //+------------------------------------------------------------------+ //| Implements the complex mh attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexMHAttentionOut(__global float2* __attribute__((aligned(8))) scores, ///<[in] Matrix of Scores __global float2* __attribute__((aligned(8))) qkv, ///<[in] Matrix of Values __global float2* __attribute__((aligned(8))) out, ///<[out] Output tensor int dimension ///< Dimension of Value ) { int u = get_global_id(0); int units = get_global_size(0); int h = get_global_id(1); int heads = get_global_size(1); //--- int shift_s = units * (h + heads * u); int shift_out = dimension * (h + heads * u); //--- //--- for(int d = 0; d < dimension; d++) { float2 result = (float2)0; for(int v = 0; v < units; v++) { int shift_v = dimension * (h + heads * (3 * v + 2)) + d; result += ComplexMul(scores[shift_s + v], qkv[shift_v]); } out[shift_out + d] = result; } } //+------------------------------------------------------------------+ //| Computes the complex mh attention gradients gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexMHAttentionGradients(__global float2* __attribute__((aligned(8))) qkv, __global float2* __attribute__((aligned(8))) qkv_g, __global float2* __attribute__((aligned(8))) scores, __global float2* __attribute__((aligned(8))) gradient ) { size_t u = get_global_id(0); size_t h = get_global_id(1); size_t d = get_global_id(2); size_t units = get_global_size(0); size_t heads = get_global_size(1); size_t dimension = get_global_size(2); //--- float2 koef = (float2)(sqrt((float)dimension), 0); if(koef.x < 1) koef.x = 1; //--- init const int shift_q = dimension * (heads * 3 * u + h); const int shift_k = dimension * (heads * (3 * u + 1) + h); const int shift_v = dimension * (heads * (3 * u + 2) + h); const int shift_g = dimension * (heads * u + h); int shift_score = h * units; int step_score = units * heads; //--- Calculating Value's gradients float2 sum = (float2)0; //--- for(int i = 0; i < units; i++) sum += ComplexMul(gradient[(h + i * heads) * dimension + d], scores[shift_score + u + i * step_score]); qkv_g[shift_v + d] = sum; //--- Calculating Query's gradients shift_score = h * units + u * step_score; float2 grad = 0; float2 grad_out = gradient[shift_g + d]; //--- for(int k = 0; k < units; k++) { float2 sc_g = (float2)0; float2 sc = scores[shift_score + k]; for(int v = 0; v < units; v++) sc_g += ComplexMul( ComplexMul(scores[shift_score + v], ComplexMul(qkv[dimension * (heads * (3 * v + 2) + h)], grad_out)), ((float2)(k == v, 0) - sc) ); grad += ComplexMul(ComplexDiv(sc_g, koef), qkv[dimension * (heads * (3 * k + 1) + h) + d]); } qkv_g[shift_q + d] = grad; //--- Calculating Key's gradients grad = 0; //--- for(int q = 0; q < units; q++) { shift_score = h * units + q * step_score; float2 sc_g = (float2)0; float2 sc = scores[shift_score + u]; float2 grad_out = gradient[dimension * (heads * q + h) + d]; for(int v = 0; v < units; v++) sc_g += ComplexMul( ComplexMul(scores[shift_score + v], ComplexMul(qkv[dimension * (heads * (3 * v + 2) + h)], grad_out)), ((float2)(u == v, 0) - sc) ); grad += ComplexMul(ComplexDiv(sc_g, koef), qkv[dimension * (heads * 3 * q + h) + d]); } qkv_g[shift_k + d] = grad; } //+------------------------------------------------------------------+ //| Implements the complex normalize kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexNormalize(__global float2* __attribute__((aligned(8))) inputs, __global float2* __attribute__((aligned(8))) outputs, __global float2* __attribute__((aligned(8))) means, __global float *vars, int dimension) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; const float2 dim = (float2)(dimension, 0); //--- float2 mean = 0; //--- for(int i = 0; i < dimension; i++) mean = IsNaNOrInf2(inputs[shift + i], (float2)0); means[n] = mean = ComplexDiv(mean, dim); float variance = 0; //--- for(int i = 0; i < dimension; i++) { float abs_delta = ComplexAbs(inputs[shift + i] - mean); variance += abs_delta * abs_delta; } vars[n] = variance = sqrt(IsNaNOrInf(variance / dimension, 1)); float2 v = (float2)(variance, 0); //--- for(int i = 0; i < dimension; i++) { float2 val = IsNaNOrInf2(ComplexDiv((inputs[shift + i] - mean), v), (float2)0); outputs[shift + i] = val; } } //+------------------------------------------------------------------+ //| Computes the complex normalize gradient gradient contributions on| //| the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexNormalizeGradient(__global float2* __attribute__((aligned(8))) inputs_gr, __global float2* __attribute__((aligned(8))) outputs_gr, __global float *vars, int dimension) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float v = vars[n]; float2 variance = (float2)((v > 0 ? v : 1.0f), 0); //--- for(int i = 0; i < dimension; i++) { float2 val = ComplexDiv(outputs_gr[shift + i], variance); if(isnan(val.x) || isinf(val.x) || isnan(val.y) || isinf(val.y)) val = (float2)0; inputs_gr[shift + i] = val; } } //+------------------------------------------------------------------+ //| Implements the complex un normalize kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexUnNormalize(__global float2* __attribute__((aligned(8))) inputs, __global float2* __attribute__((aligned(8))) outputs, __global float2* __attribute__((aligned(8))) means, __global float *vars, int dimension) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float v = vars[n]; float2 variance = (float2)((v > 0 ? v : 1.0f), 0); float2 mean = means[n]; //--- for(int i = 0; i < dimension; i++) { float2 val = ComplexMul(inputs[shift + i], variance) + mean; if(isnan(val.x) || isinf(val.x) || isnan(val.y) || isinf(val.y)) val = (float2)0; outputs[shift + i] = val; } } //+------------------------------------------------------------------+ //| Computes the complex un normalize gradient gradient contributions| //| on the GPU. | //+------------------------------------------------------------------+ __kernel void ComplexUnNormalizeGradient(__global float2* __attribute__((aligned(8))) inputs_gr, __global float2* __attribute__((aligned(8))) outputs_gr, __global float *vars, int dimension ) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float v = vars[n]; float2 variance = (float2)((v > 0 ? v : 1.0f), 0); //--- for(int i = 0; i < dimension; i++) { float2 val = ComplexMul(outputs_gr[shift + i], variance); if(isnan(val.x) || isinf(val.x) || isnan(val.y) || isinf(val.y)) val = (float2)0; inputs_gr[shift + i] = val; } } //+------------------------------------------------------------------+ //| Implements the main freq weight kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MainFreqWeight(__global float2* __attribute__((aligned(8))) freq, __global float *weight, int dimension ) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float max_f = 0; float total = 0; float energy; //--- for(int i = 0; i < dimension; i++) { energy = ComplexAbs(freq[shift + i]); total += energy; max_f = fmax(max_f, energy); } weight[n] = max_f / (total > 0 ? total : 1); } //+------------------------------------------------------------------+ //| Implements the weighted sum kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void WeightedSum(__global float *inputs1, __global float *inputs2, __global float *outputs, __global float *weight, int dimension ) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float w = weight[n]; //--- for(int i = 0; i < dimension; i++) outputs[shift + i] = inputs1[shift + i] * w + inputs2[shift + i] * (1 - w); } //+------------------------------------------------------------------+ //| Computes the weighted sum gradient gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void WeightedSumGradient(__global float *inputs_gr1, __global float *inputs_gr2, __global float *outputs_gr, __global float *weight, int dimension ) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float w = weight[n]; float w1 = 1 - weight[n]; //--- for(int i = 0; i < dimension; i++) { float grad = outputs_gr[shift + i]; inputs_gr1[shift + i] = grad * w; inputs_gr2[shift + i] = grad * w1; } } //+------------------------------------------------------------------+ //| Runs the feed forward s3 feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardS3(__global float *inputs, __global float *probability, __global float *weights, __global float *outputs, __global float *positions, const int window, const int total ) { int pos = get_global_id(0); int segments = get_global_size(0); //--- if((segments * window) > total) segments--; //--- int segment = 0; if(pos < segments) { const float prob = probability[pos]; //--- for(int i = 0; i < pos; i++) { if(probability[i] <= prob) segment++; } //--- for(int i = pos + 1; i < segments; i++) { if(probability[i] < prob) segment++; } } else segment = pos; //--- const int shift_in = segment * window; const int shift_out = pos * window; const float w1 = weights[0]; const float w2 = weights[1]; positions[pos] = (float)segment; //--- for(int i = 0; i < window; i++) { if((shift_in + i) >= total || (shift_out + i) >= total) break; outputs[shift_out + i] = w1 * inputs[shift_in + i] + w2 * inputs[shift_out + i]; } } //+------------------------------------------------------------------+ //| Computes the inside gradient s3 gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void InsideGradientS3(__global float *inputs, __global float *inputs_gr, __global float *probability, __global float *probability_gr, __global float *weights, __global float *outputs_gr, __global float *positions, const int window, const int total ) { size_t pos = get_global_id(0); //--- int segment = (int)positions[pos]; float prob = probability[pos]; const float w1 = weights[0]; const float w2 = weights[1]; const int shift_in = segment * window; const int shift_out = pos * window; //--- float grad = 0; float temp = 0; //--- for(int i = 0; i < window; i++) { if((shift_out + i) >= total) break; temp = outputs_gr[shift_out + i] * w1; grad += temp * inputs[shift_in + i]; inputs_gr[shift_in + i] = temp + outputs_gr[shift_in + i] * w2; } probability_gr[segment] = grad / prob; } //+------------------------------------------------------------------+ //| Computes the weight gradient s3 gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void WeightGradientS3(__global float *inputs, __global float *positions, __global float *outputs_gr, __global float *weights_gr, const int window, const int total ) { size_t l = get_local_id(0); size_t w = get_global_id(1); size_t ls = min((uint)get_local_size(0), (uint)LOCAL_ARRAY_SIZE); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- if(l < ls) { float val = 0; //--- for(int i = l; i < total; i += ls) { int shift_in = i; if(w == 0) { int pos = i / window; shift_in = positions[pos] * window + i % window; } val += outputs_gr[i] * inputs[shift_in]; } temp[l] = val; } BarrierLoc //--- int t = ls; do { t = (t + 1) / 2; if(l < t && (l + t) < ls) { temp[l] += temp[l + t]; temp[l + t] = 0; } BarrierLoc } while(t > 1); //--- if(l == 0) weights_gr[w] = temp[0]; } //+------------------------------------------------------------------+ //| Implements the mh2 pyr attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MH2PyrAttentionOut(__global float *q, __global float *kv, __global float *score, __global float *out, const int dimension, const int heads_kv, const int window ) { //--- init const int q_id = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_k = dimension * (2 * heads_kv * k + h_kv); const int shift_v = dimension * (2 * heads_kv * k + heads_kv + h_kv); const int shift_s = kunits * (q_id * heads + h) + k; const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); const int delta_win = (window + 1) / 2; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; __local float temp[LOCAL_ARRAY_SIZE]; //--- Score float sum = 0; for(int d = 0; d < dimension; d++) sum = q[shift_q + d] * kv[shift_k + d]; float sc = LocalSoftMax(IsNaNOrInf(sum / koef, MIN_VALUE), 1, temp); score[shift_s] = sc; BarrierLoc //--- out for(int d = 0; d < dimension; d++) { uint count = 0; if(k < ls) do { if((count * ls) < (kunits - k)) { sum = 0; if(abs(count * ls + k - q_id) <= delta_win) { int sh_v = 2 * dimension * heads_kv * count * ls; sum = kv[shift_v + d + sh_v] * (count == 0 ? sc : score[shift_s + count * ls]); if(isnan(sum)) sum = 0; } temp[k] = (count > 0 ? temp[k] : 0) + sum; } count++; } while((count * ls + k) < kunits); BarrierLoc //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k < ls) temp[k] += (k < count && (k + count) < kunits ? temp[k + count] : 0); if(k + count < ls) temp[k + count] = 0; BarrierLoc } while(count > 1); //--- if(k == 0) out[shift_q + d] = temp[0]; } } //+------------------------------------------------------------------+ //| Implements the plr kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PLR(__global const float *inputs, __global float *outputs, __global int *isttp, const int transpose, const float min_step ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- constants const int shift_in = ((bool)transpose ? (i * variables + v) : (v * lenth + i)); const int step_in = ((bool)transpose ? variables : 1); //--- look for ttp float value = inputs[shift_in]; bool bttp = false; if(i == 0 || i == lenth - 1) bttp = true; else { float prev = value; int prev_pos = i; float max_v = value; float max_pos = i; float min_v = value; float min_pos = i; while(fmax(fabs(prev - max_v), fabs(prev - min_v)) < min_step && prev_pos > 0) { prev_pos--; prev = inputs[shift_in - (i - prev_pos) * step_in]; if(prev >= max_v && (prev - min_v) < min_step) { max_v = prev; max_pos = prev_pos; } if(prev <= min_v && (max_v - prev) < min_step) { min_v = prev; min_pos = prev_pos; } } //--- float next = value; int next_pos = i; while(fmax(fabs(next - max_v), fabs(next - min_v)) < min_step && next_pos < (lenth - 1)) { next_pos++; next = inputs[shift_in + (next_pos - i) * step_in]; if(next > max_v && (next - min_v) < min_step) { max_v = next; max_pos = next_pos; } if(next < min_v && (max_v - next) < min_step) { min_v = next; min_pos = next_pos; } } //--- if( (value >= prev && value > next) || (value > prev && value == next) || (value <= prev && value < next) || (value < prev && value == next) ) if(max_pos == i || min_pos == i) bttp = true; } //--- isttp[shift_in] = (int)bttp; outputs[shift_in] = 0; BarrierLoc //--- calc position int pos = -1; int prev_in = 0; int prev_ttp = 0; if(bttp) { pos = 0; //--- for(int p = 0; p < i; p++) { int current_in = ((bool)transpose ? (p * variables + v) : (v * lenth + p)); if((bool)isttp[current_in]) { pos++; prev_ttp = p; prev_in = current_in; } } } //--- cacl tendency if(pos > 0 && pos < (lenth / 3)) { float sum_x = 0; float sum_y = 0; float sum_xy = 0; float sum_xx = 0; int dist = i - prev_ttp; //--- for(int p = 0; p < dist; p++) { float x = (float)(p); float y = inputs[prev_in + p * step_in]; sum_x += x; sum_y += y; sum_xy += x * y; sum_xx += x * x; } float slope = (dist * sum_xy - sum_x * sum_y) / (dist > 1 ? (dist * sum_xx - sum_x * sum_x) : 1); float intercept = (sum_y - slope * sum_x) / dist; int shift_out = ((bool)transpose ? ((pos - 1) * 3 * variables + v) : (v * lenth + (pos - 1) * 3)); outputs[shift_out] = slope; outputs[shift_out + step_in] = intercept; outputs[shift_out + 2 * step_in] = ((float)dist) / lenth; } else { if(pos == (lenth / 3)) { float sum_x = 0; float sum_y = 0; float sum_xy = 0; float sum_xx = 0; int dist = lenth - prev_ttp; //--- for(int p = 0; p < dist; p++) { float x = (float)(p); float y = inputs[prev_in + p * step_in]; sum_x += x; sum_y += y; sum_xy += x * y; sum_xx += x * x; } float slope = (dist * sum_xy - sum_x * sum_y) / (dist > 1 ? (dist * sum_xx - sum_x * sum_x) : 1); float intercept = (sum_y - slope * sum_x) / dist; int shift_out = ((bool)transpose ? ((pos - 1) * 3 * variables + v) : (v * lenth + (pos - 1) * 3)); outputs[shift_out] = slope; outputs[shift_out + step_in] = intercept; outputs[shift_out + 2 * step_in] = ((float)dist) / lenth; } } } //+------------------------------------------------------------------+ //| Computes the plr gradient gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void PLRGradient(__global float *inputs_gr, __global const float *outputs, __global const float *outputs_gr, const int transpose ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- constants const int shift_in = ((bool)transpose ? (i * variables + v) : (v * lenth + i)); const int step_in = ((bool)transpose ? variables : 1); const int shift_out = ((bool)transpose ? v : (v * lenth)); const int step_out = 3 * step_in; //--- calc position int pos = -1; int prev_in = 0; int dist = 0; do { pos++; prev_in += dist; dist = (int)fmax(outputs[shift_out + pos * step_out + 2 * step_in] * lenth, 1); } while(!(prev_in <= i && (prev_in + dist) > i)); //--- calc constants float sum_x = 0; float sum_xx = 0; //--- for(int p = 0; p < dist; p++) { float x = (float)(p); sum_x += x; sum_xx += x * x; } //--- get output gradient float grad_slope = outputs_gr[shift_out + pos * step_out]; float grad_intercept = outputs_gr[shift_out + pos * step_out + step_in]; //--- calc gradient grad_slope -= sum_x / dist * grad_intercept; grad_slope /= fmax(dist * sum_xx - sum_x * sum_x, 1); float grad = grad_intercept / dist; grad += (dist * (i - prev_in) - sum_x) * grad_slope; if(isnan(grad) || isinf(grad)) grad = 0; //--- save result inputs_gr[shift_in] = grad; } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights adam mini | //| step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsAdamMini(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< number of neurons in previous layer and n - ///< number of neurons in current layer __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in,out] Matrix of first momentum __global float *matrix_v, ///<[in,out] Matrix of seconfd momentum const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2 ///< Second momentum multiplier ) { //--- inputs const size_t i = get_local_id(0); const size_t inputs = get_local_size(0) - 1; //--- outputs const size_t o = get_global_id(1); const size_t outputs = get_global_size(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((uint)LOCAL_ARRAY_SIZE, (uint)inputs); const float inp = (i < inputs ? matrix_i[i] : 1.0f); int count = 0; do { if(count == (i / ls)) { int shift = i % ls; temp[shift] = (count == 0 ? 0 : temp[shift]) + ((isnan(inp) || isinf(inp)) ? 0 : inp * inp) / inputs; } count++; BarrierLoc } while(count * ls < inputs); //--- sum count = (ls + 1) / 2; do { if(i < count && (i + count) < ls) { temp[i] += temp[i + count]; temp[i + count] = 0; } count = (count + 1) / 2; BarrierLoc } while(count > 1); //--- calc v if(i == 0) { temp[1] = matrix_g[o]; if(isnan(temp[1]) || isinf(temp[1])) temp[1] = 0; if(isnan(temp[0]) || isinf(temp[0])) temp[0] = 1; float v = matrix_v[o]; if(isnan(v) || isinf(v)) v = 1; temp[0] = b2 * v + (1 - b2) * (temp[1] * temp[1]) * temp[0]; matrix_v[o] = temp[0]; } BarrierLoc //--- const int wi = o * (inputs + 1) + i; float weight = matrix_w[wi]; if(isnan(weight) || isinf(weight)) weight = 0; //--- float m = matrix_m[wi]; if(isnan(m) || isinf(m)) m = 0; //--- calc m m = b1 * m + (1 - b1) * temp[1] * inp; if(isnan(m) || isinf(m)) m = 0; //--- float delta = l * (m / (sqrt(temp[0]) + 1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(isnan(delta) || isinf(delta)) delta = 0; if(delta > 0) matrix_w[wi] = weight + delta; matrix_m[wi] = m; } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights conv adam | //| mini step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsConvAdamMini(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in] Matrix of first momentum __global float *matrix_v, ///<[in] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2, ///< Second momentum multiplier int step ///< Step size ) { //--- window in const size_t i = get_global_id(0); const size_t window_in = get_global_size(0) - 1; //--- window out const size_t f = get_global_id(1); const size_t window_out = get_global_size(1); //--- head window out const size_t f_h = get_local_id(1); const size_t window_out_h = get_local_size(1); //--- variable const size_t v = get_global_id(2); const size_t variables = get_global_size(2); //--- constants const int total = (inputs - window_in + step - 1) / step; const int shift_var_in = v * inputs; const int shift_var_out = v * total * window_out; const int shift_w = (f + v * window_out) * (window_in + 1) + i; //--- __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((uint)window_in, (uint)LOCAL_ARRAY_SIZE); //--- calc gradient float grad = 0; //--- for(int t = 0; t < total; t++) { if(i != window_in && (i + t * window_in) >= inputs) break; float gt = matrix_g[t * window_out + f + shift_var_out] * (i == window_in ? 1 : matrix_i[i + t * step + shift_var_in]); if(!(isnan(gt) || isinf(gt))) grad += gt; } //--- calc sum grad int count; //--- for(int h = 0; h < window_out_h; h++) { count = 0; do { if(h == f_h) { if(count == (i / ls)) { int shift = i % ls; temp[shift] = ((count == 0 && h == 0) ? 0 : temp[shift]) + ((isnan(grad) || isinf(grad)) ? 0 : grad * grad) / (window_in * window_out_h); } } count++; BarrierLoc } while((count * ls) < window_in); } count = (ls + 1) / 2; do { if(i < count && (i + count) < ls && f_h == 0) { temp[i] += temp[i + count]; temp[i + count] = 0; } count = (count + 1) / 2; BarrierLoc } while(count > 1); //--- calc v if(i == 0 && f_h == 0) { if(isnan(temp[0]) || isinf(temp[0])) temp[0] = 1; int head = f / window_out_h; float v = matrix_v[head]; if(isnan(v) || isinf(v)) v = 1; temp[0] = clamp(b2 * v + (1 - b2) * temp[0], 1.0e-6f, 1.0e6f); matrix_v[head] = temp[0]; } BarrierLoc //--- calc m float mt = clamp(b1 * matrix_m[shift_w] + (1 - b1) * grad, -1.0e5f, 1.0e5f); if(isnan(mt) || isinf(mt)) mt = 0; float weight = matrix_w[shift_w] + l * mt / sqrt(temp[0]); if(!(isnan(weight) || isinf(weight))) matrix_w[shift_w] = weight; matrix_m[shift_w] = mt; } //+------------------------------------------------------------------+ //| Implements the cut trend and other kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CutTrendAndOther(__global const float *inputs, __global const float *plr, __global float *trend, __global float *other ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- constants const int shift_in = i * variables + v; const int step_in = variables; const int shift_plr = v; const int step_plr = 3 * step_in; //--- calc position int pos = -1; int prev_in = 0; int dist = 0; do { pos++; prev_in += dist; dist = (int)fmax(plr[shift_plr + pos * step_plr + 2 * step_in] * lenth, 1); } while(!(prev_in <= i && (prev_in + dist) > i)); //--- calc trend float sloat = plr[shift_plr + pos * step_plr]; float intercept = plr[shift_plr + pos * step_plr + step_in]; pos = i - prev_in; float trend_i = sloat * pos + intercept; float other_i = inputs[shift_in] - trend_i; //--- save result trend[shift_in] = trend_i; other[shift_in] = other_i; } //+------------------------------------------------------------------+ //| Computes the cut trend and other gradient gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void CutTrendAndOtherGradient(__global float *inputs_gr, __global const float *plr, __global float *plr_gr, __global const float *trend_gr, __global const float *other_gr ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- constants const int shift_in = i * variables + v; const int step_in = variables; const int shift_plr = v; const int step_plr = 3 * step_in; //--- calc position int pos = -1; int prev_in = 0; int dist = 0; do { pos++; prev_in += dist; dist = (int)fmax(plr[shift_plr + pos * step_plr + 2 * step_in] * lenth, 1); } while(!(prev_in <= i && (prev_in + dist) > i)); //--- get gradient float other_i_gr = other_gr[shift_in]; float trend_i_gr = trend_gr[shift_in] - other_i_gr; //--- calc plr gradient pos = i - prev_in; float sloat_gr = trend_i_gr * pos; float intercept_gr = trend_i_gr; //--- save result plr_gr[shift_plr + pos * step_plr] += sloat_gr; plr_gr[shift_plr + pos * step_plr + step_in] += intercept_gr; inputs_gr[shift_in] = other_i_gr; } //+------------------------------------------------------------------+ //| Implements the cut one from another kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CutOneFromAnother(__global const float *inputs, __global const float *cut, __global float *other ) { const size_t i = get_global_id(0); //--- save result other[i] = inputs[i] - cut[i]; } //+------------------------------------------------------------------+ //| Computes the cut one from another gradient gradient contributions| //| on the GPU. | //+------------------------------------------------------------------+ __kernel void CutOneFromAnotherGradient(__global float *inputs_gr, __global float *cut_gr, __global const float *other_gr ) { const size_t i = get_global_id(0); float gr = other_gr[i]; //--- save result inputs_gr[i] = gr; cut_gr[i] = (-gr); } //+------------------------------------------------------------------+ //| Implements the uni traj prepare kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void UniTrajPrepare(__global const float *history, __global const float *h_mask, __global const float *future, __global const float *f_mask, __global float *output, const int h_total, const int f_total ) { const size_t i = get_global_id(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- const int shift_in = i * variables + v; const int shift_out = 3 * shift_in; const int shift_f_out = 3 * (h_total * variables + v); //--- history if(i < h_total) { float mask = h_mask[shift_in]; float h = history[shift_in]; float v = (i < (h_total - 1) && mask != 0 ? (history[shift_in + variables] - h) * mask : 0); if(isnan(v) || isinf(v)) v = h = mask = 0; output[shift_out] = h * mask; output[shift_out + 1] = v; output[shift_out + 2] = mask; } //--- future if(i < f_total) { float mask = f_mask[shift_in]; float f = future[shift_in]; float v = (i < (f_total - 1) && mask != 0 ? (future[shift_in + variables] - f) * mask : 0); if(isnan(v) || isinf(v)) v = f = mask = 0; output[shift_f_out + shift_out] = f * mask; output[shift_f_out + shift_out + 1] = v; output[shift_f_out + shift_out + 2] = mask; } } //+------------------------------------------------------------------+ //| Computes the uni traj prepare grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void UniTrajPrepareGrad(__global float *history_gr, __global float *future_gr, __global const float *output, __global const float *output_gr, const int h_total, const int f_total ) { const size_t i = get_global_id(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- const int shift_in = i * variables + v; const int shift_out = 3 * shift_in; const int shift_f_out = 3 * (h_total * variables + v); //--- history if(i < h_total) { float mask = output[shift_out + 2]; float grad = 0; if(mask > 0) { grad = output_gr[shift_out] * mask; grad -= (i < (h_total - 1) && mask != 0 ? (output_gr[shift_out + 1]) * mask : 0); grad += (i > 0 ? output[shift_out + 1 - 3 * variables] * output[shift_out + 2 - 3 * variables] : 0); if(isnan(grad) || isinf(grad)) grad = 0; //--- } history_gr[shift_in] = grad; } //--- future if(i < f_total) { float mask = output[shift_f_out + shift_out + 2]; float grad = 0; if(mask > 0) { grad = output_gr[shift_f_out + shift_out] * mask; grad -= (i < (h_total - 1) && mask != 0 ? (output_gr[shift_f_out + shift_out + 1]) * mask : 0); grad += (i > 0 ? output[shift_f_out + shift_out + 1 - 3 * variables] * output[shift_f_out + shift_out + 2 - 3 * variables] : 0); if(isnan(grad) || isinf(grad)) grad = 0; //--- } future_gr[shift_in] = grad; } } //+------------------------------------------------------------------+ //| Implements the uni traj bts kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void UniTrajBTS(__global const float *concat_inp, __global float *d_forw, __global float *d_bakw, const int total ) { const size_t i = get_global_id(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- if(i == 0) { const int step = variables * 3; const int start = v * 3 + 2; float last = 0; d_forw[v] = 0; //--- for(int p = 1; p < total; p++) { float m = concat_inp[start + p * step]; d_forw[p * variables + v] = last = 1 + (1 - m) * last; } } else { const int step = -(variables * 3); const int start = (total - 1) * variables + v * 3 + 2; float last = 0; d_bakw[(total - 1) + v] = 0; //--- for(int p = 1; p < total; p++) { float m = concat_inp[start + p * step]; d_bakw[(total - 1 - p) * variables + v] = last = 1 + (1 - m) * last; } } } //+------------------------------------------------------------------+ //| Performs the rotate data-layout operation on the GPU. | //+------------------------------------------------------------------+ float2 Rotate(const float x, const float cos_theta, const float sin_theta) { float2 result = 0; result.s0 = cos_theta + x * sin_theta; result.s1 = x * cos_theta - sin_theta; return result; } //+------------------------------------------------------------------+ //| Implements the hi vt prepare kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void HiVTPrepare(__global const float *data, __global float2* __attribute__((aligned(8))) output ) { const size_t t = get_global_id(0); const size_t v = get_global_id(1); const size_t total_v = get_global_size(1); //--- const int shift_data = t * total_v; const int shift_out = shift_data * total_v; //--- float value = data[shift_data + v + total_v] - data[shift_data + v]; const float theta = atan(value); const float cos_theta = cos(theta); const float sin_theta = sin(theta); const float2 main = Rotate(value, cos_theta, sin_theta); //--- //--- for(int a = 0; a < total_v; a++) { float2 o = main; if(a != v) o -= Rotate(data[shift_data + a + total_v] - data[shift_data + a], cos_theta, sin_theta); output[shift_out + a] = o; } } //+------------------------------------------------------------------+ //| Implements the gate element mul kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void GateElementMul(__global const float *inputs1, __global const float *inputs2, __global const float *gate, __global float *out, __global float *numeric_flags, const int flags_enabled ) { const int i = get_global_id(0); //--- int invalid = 0; if(flags_enabled) { //--- Evaluate raw gate products before their operands are sanitized for legacy output. const float raw_gate = gate[i]; const float raw_input1 = inputs1[i]; const float raw_input2 = inputs2[i]; const float raw_product1 = raw_gate * raw_input1; const float raw_product2 = (1.0f - raw_gate) * raw_input2; invalid = isnan(raw_gate) || isinf(raw_gate) || isnan(raw_input1) || isinf(raw_input1) || isnan(raw_input2) || isinf(raw_input2) || isnan(raw_product1) || isinf(raw_product1) || isnan(raw_product2) || isinf(raw_product2); } const float g = IsNaNOrInf(gate[i], 0.5f); float result = 0; float inp = IsNaNOrInf(inputs1[i], 0.0f); if(inp != 0.0f && g != 0.0f) result += g * inp; inp = IsNaNOrInf(inputs2[i], 0.0f); if(inp != 0.0f && (1 - g) != 0.0f) result += (1 - g) * inp; //--- if(flags_enabled) { invalid |= isnan(result) || isinf(result); numeric_flags[i] = (invalid ? 1.0f : 0.0f); } out[i] = IsNaNOrInf(result, 0); } //+------------------------------------------------------------------+ //| Computes the gate element mul grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void GateElementMulGrad(__global const float *inputs1, __global float *inputs1_gr, __global const float *inputs2, __global float *inputs2_gr, __global const float *gate, __global float *gate_gr, __global const float *out_gr, const int activ1, const int activ2, const int activ_gate ) { const int i = get_global_id(0); //--- const float g = IsNaNOrInf(gate[i], 0.5f); const float i1 = IsNaNOrInf(inputs1[i], 0); const float i2 = IsNaNOrInf(inputs2[i], 0); const float grad = IsNaNOrInf(out_gr[i], 0); //--- float i1_gr = IsNaNOrInf(grad * g, 0); float i2_gr = IsNaNOrInf(grad * (1 - g), 0); float g_gr = IsNaNOrInf(grad * (i1 - i2), 0); //--- inputs1_gr[i] = Deactivation(i1_gr, i1, activ1); inputs2_gr[i] = Deactivation(i2_gr, i2, activ2); if(activ_gate == ActFunc_SIGMOID) gate_gr[i] = clamp(IsNaNOrInf(g_gr * g * (1.0f - g), 0), -MAX_GRAD, MAX_GRAD); else gate_gr[i] = Deactivation(g_gr, g, activ_gate); } //+------------------------------------------------------------------+ //| Performs the transpose rcd data-layout operation on the GPU. | //+------------------------------------------------------------------+ __kernel void TransposeRCD(__global const float *matrix_in, ///<[in] Input matrix __global float *matrix_out ///<[out] Output matrix ) { const int r = get_global_id(0); const int c = get_global_id(1); const int d = get_global_id(2); const int rows = get_global_size(0); const int cols = get_global_size(1); const int dimension = get_global_size(2); //--- matrix_out[(c * rows + r)*dimension + d] = matrix_in[(r * cols + c) * dimension + d]; } //+------------------------------------------------------------------+ //| Implements the orthoganal loss kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void OrthoganalLoss(__global const float *data, __global float *grad, const int add ) { const size_t r = get_global_id(0); const size_t c = get_local_id(1); const size_t cols = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; uint ls = min((uint)cols, (uint)LOCAL_ARRAY_SIZE); //--- const int shift1 = r * cols + c; const int shift2 = c * cols + r; float value1 = IsNaNOrInf(data[shift1], 0); float value2 = (shift1 == shift2 ? value1 : IsNaNOrInf(data[shift2], 0)); float v2 = IsNaNOrInf(value1 * value2, 0); //--- for(int i = 0; i < cols; i += ls) { //--- if(i <= c && (i + ls) > c) Temp[c - i] = (i == 0 ? 0 : Temp[c - i]) + v2; BarrierLoc } //--- uint count = min(ls, (uint)cols); do { count = (count + 1) / 2; if(c < ls) Temp[c] += (c < count && (c + count) < cols ? Temp[c + count] : 0); if(c + count < ls) Temp[c + count] = 0; BarrierLoc } while(count > 1); //--- const float sum = Temp[0]; float diff = (float)(r == c) - sum; float loss = -(diff * diff); float g = (2 * (sum - (float)(r == c))) * loss; g = 2 * value2 * g; if(isinf(g) || isnan(g)) g = 0; if(add == 1) grad[shift1] += g; else grad[shift1] = g; } //+------------------------------------------------------------------+ //| Calculates calc distance on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcDistance(__global const float *data, __global float *distance, const int dimension, const int groups, const int normalize, __global float *numeric_flags, const int flags_enabled ) { const size_t main = get_global_id(0); const size_t slave = get_local_id(1); const size_t group = get_global_id(2); const int total = (int)get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- if(group >= (size_t)groups) return; const int shift_group = (int)group * total; const int shift_main = (shift_group + main) * dimension; const int shift_slave = (shift_group + slave) * dimension; const int shift_dist = shift_group * total + main * total + slave; //--- calc distance float dist = 0; int invalid = 0; if(flags_enabled) for(int d = 0; d < dimension; d++) { const float value_main = data[shift_main + d]; const float value_slave = data[shift_slave + d]; invalid |= isnan(value_main) || isinf(value_main) || isnan(value_slave) || isinf(value_slave); } if(main != slave) { //--- for(int d = 0; d < dimension; d++) { float delta = data[shift_main + d] - data[shift_slave + d]; dist += delta * delta; if(flags_enabled) { const float square = delta * delta; invalid |= isnan(delta) || isinf(delta) || isnan(square) || isinf(square); } } } //--- Look Max float max_dist = LocalMax(dist, 1, Temp); if(flags_enabled) invalid |= isnan(dist) || isinf(dist) || isnan(max_dist) || isinf(max_dist); //--- Normalize if(normalize && max_dist > 0) dist /= Temp[0]; if(flags_enabled) invalid |= isnan(dist) || isinf(dist); if(flags_enabled) numeric_flags[shift_dist] = (invalid ? 1.0f : 0.0f); dist = IsNaNOrInf(dist, 1); //--- result distance[shift_dist] = dist; } //+------------------------------------------------------------------+ //| Runs the feed forward local max feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardLocalMax(__global const float *matrix_i, __global const float *distance, __global float *matrix_o, const float radius ) { const size_t i = get_global_id(0); const size_t total = get_global_size(0); const size_t d = get_global_id(1); const size_t dimension = get_global_size(1); //--- const int shift_dist = i * total; const int shift_out = i * dimension + d; //--- float result = -3.402823466e+38f; //--- for(int k = 0; k < total; k++) { if(distance[shift_dist + k] > radius) continue; int shift = k * dimension + d; result = max(result, matrix_i[shift]); } matrix_o[shift_out] = result; } //+------------------------------------------------------------------+ //| Computes the calc input gradient local max gradient contributions| //| on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcInputGradientLocalMax(__global const float *matrix_i, __global float *matrix_ig, __global const float *distance, __global const float *matrix_o, __global const float *matrix_g, const float radius ) { const size_t i = get_global_id(0); const size_t total = get_global_size(0); const size_t d = get_global_id(1); const size_t dimension = get_global_size(1); //--- float result = 0; float value = matrix_i[i * dimension + d]; //--- for(int k = 0; k < total; k++) { if(distance[k * total + i] > radius) continue; int shift = k * dimension + d; if(fabs(matrix_o[shift] - value) <= 1.192092896e-07f) result += matrix_g[shift]; } matrix_ig[i * dimension + d] = result; } //+------------------------------------------------------------------+ //| Implements the mh mask attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHMaskAttentionOut(__global const float *q, ///<[in] Matrix of Querys __global const float *kv, ///<[in] Matrix of Keys __global float *score, ///<[out] Matrix of Scores __global const float *mask, ///<[in] Mask Matrix __global float *out, ///<[out] Matrix of attention const int dimension, ///< Dimension of Key const int heads_kv, const float mask_level ) { //--- init const int q_id = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_local_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_k = dimension * (2 * heads_kv * k + h_kv); const int shift_v = dimension * (2 * heads_kv * k + heads_kv + h_kv); const int shift_s = kunits * (q_id * heads + h) + k; const bool b_mask = (mask[shift_s] < mask_level); const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); float koef = sqrt((float)dimension); if(koef < 1) koef = 1; __local float temp[LOCAL_ARRAY_SIZE]; //--- Score float sum = 0; if(b_mask) sum = MIN_VALUE; else for(int d = 0; d < dimension; d++) sum += q[shift_q + d] * kv[shift_k + d]; float sc = LocalSoftMax(IsNaNOrInf(sum / koef, MIN_VALUE), 1, temp); score[shift_s] = sc; BarrierLoc //--- out for(int d = 0; d < dimension; d++) { BarrierLoc sum = LocalSum(IsNaNOrInf(kv[shift_v + d] * sc, 0), 1, temp); if(k == 0) out[shift_q + d] = temp[0]; } } //+------------------------------------------------------------------+ //| Computes the mh mask attention inside gradients gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MHMaskAttentionInsideGradients(__global const float *q, __global float *q_g, __global const float *kv, __global float *kv_g, __global const float *mask, __global float *mask_g, __global const float *scores, __global const float *gradient, const int kunits, const int heads_kv, const float mask_level) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (q_id * heads + h) * kunits; const int shift_g = h * dimension + d; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v = q_id; v < kunits; v += qunits) { float grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v; for(int g = 0; g < qunits; g++) grad += gradient[shift_g + dimension * (hq - h + g * heads)] * scores[shift_score + g * step_score]; } int shift_v = dimension * (2 * heads_kv * v + heads_kv + h) + d; kv_g[shift_v] = grad; } } //--- Calculating Query's gradients float grad = 0; float out_g = gradient[shift_g + q_id * dimension]; int shift_val = (heads_kv + h_kv) * dimension + d; int shift_key = h_kv * dimension + d; //--- for(int k = 0; k < kunits; k++) { float sc_g = 0; float sc = scores[shift_s + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_s + v] * out_g * kv[shift_val + 2 * v * heads_kv * dimension] * ((float)(k == v) - sc); grad += sc_g * kv[shift_key + 2 * k * heads_kv * dimension]; } q_g[shift_q] = grad / koef; //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k = q_id; k < kunits; k += qunits) { int shift_k = dimension * (2 * heads_kv * k + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq++) { int shift_score = hq * kunits + k; float val = kv[shift_k + heads_kv * dimension]; for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_sc + v] * gradient[shift_g + scr * dimension] * val * ((float)(k == v) - sc); grad += sc_g * q[shift_q + scr * dimension]; } } kv_g[shift_k] = grad / koef; } } //--- Mask's gradient for(int k = q_id; k < kunits; k += qunits) { float m = mask[shift_s + k]; if(m < mask_level) mask_g[shift_s + k] = 0; else mask_g[shift_s + k] = 1 - m; } } //+------------------------------------------------------------------+ //| Calculates calc position bias on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcPositionBias(__global const float *data1, __global const float *data2, __global float *result, const int dimension ) { const size_t idx1 = get_global_id(0); const size_t idx2 = get_global_id(1); const size_t total1 = get_global_size(0); const size_t total2 = get_global_size(1); //--- const int shift1 = idx1 * dimension; const int shift2 = idx2 * dimension; const int shift_out = idx1 * total2 + idx2; //--- float res = 0; //--- for(int i = 0; i < dimension; i++) { float delta = data1[shift1 + i] - data2[shift2 + i]; res = delta * delta; } res = sqrt(res); res = exp(-res); if(isnan(res) || isinf(res)) res = 0; //--- result[shift_out] = res; } //+------------------------------------------------------------------+ //| Implements the mh pos bias attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHPosBiasAttentionOut(__global const float *q, ///<[in] Matrix of Querys __global const float *k, ///<[in] Matrix of Keys __global const float *v, ///<[in] Matrix of Values __global float *score, ///<[out] Matrix of Scores __global const float *pos_bias, ///<[in] Position Bias __global float *out, ///<[out] Matrix of attention const int dimension, ///< Dimension of Key const int heads_kv, const int use_pos_bias ) { //--- init const int q_id = get_global_id(0); const int k_id = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_kv = dimension * (heads_kv * k_id + h_kv); const int shift_s = kunits * (q_id * heads + h) + k_id; const int shift_pb = q_id * kunits + k_id; const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); float koef = sqrt((float)dimension); if(koef < 1) koef = 1; __local float temp[LOCAL_ARRAY_SIZE]; //--- sum of exp uint count = 0; if(k_id < ls) { temp[k_id] = 0; //--- do { if(q_id >= (count * ls + k_id)) if((count * ls) < (kunits - k_id)) { float sum = 0; int sh_k = dimension * heads_kv * count * ls; for(int d = 0; d < dimension; d++) sum = q[shift_q + d] * k[shift_kv + d + sh_k]; sum = exp(sum / koef); if(isnan(sum)) sum = 0; temp[k_id] = temp[k_id] + sum + (use_pos_bias > 0 ? pos_bias[shift_pb + count * ls] : 0); } count++; } while((count * ls + k_id) < kunits); } BarrierLoc count = min(ls, (uint)kunits); //--- //--- do { count = (count + 1) / 2; if(k_id < ls) temp[k_id] += (k_id < count && (k_id + count) < kunits ? temp[k_id + count] : 0); if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- score float sum = temp[0]; float sc = 0; if(q_id >= (count * ls + k_id)) if(sum != 0) { //--- for(int d = 0; d < dimension; d++) sc = q[shift_q + d] * k[shift_kv + d]; sc = (exp(sc / koef) + (use_pos_bias > 0 ? pos_bias[shift_pb] : 0)) / sum; if(isnan(sc)) sc = 0; } score[shift_s] = sc; BarrierLoc //--- out //--- for(int d = 0; d < dimension; d++) { uint count = 0; if(k_id < ls) do { if((count * ls) < (kunits - k_id)) { int sh_v = 2 * dimension * heads_kv * count * ls; float sum = v[shift_kv + d + sh_v] * (count == 0 ? sc : score[shift_s + count * ls]); if(isnan(sum)) sum = 0; temp[k_id] = (count > 0 ? temp[k_id] : 0) + sum; } count++; } while((count * ls + k_id) < kunits); BarrierLoc //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k_id < ls) temp[k_id] += (k_id < count && (k_id + count) < kunits ? temp[k_id + count] : 0); if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- out[shift_q + d] = temp[0]; } } //+------------------------------------------------------------------+ //| Computes the mh pos bias attention inside gradients gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MHPosBiasAttentionInsideGradients(__global const float *q, __global float *q_g, __global const float *k, __global float *k_g, __global const float *v, __global float *v_g, __global const float *scores, __global const float *gradient, const int kunits, const int heads_kv) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (q_id * heads + h) * kunits; const int shift_g = h * dimension + d; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v_id = q_id; v_id < kunits; v_id += qunits) { float grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v_id; for(int g = 0; g < qunits; g++) grad += gradient[shift_g + dimension * (hq - h + g * heads)] * scores[shift_score + g * step_score]; } int shift_v = dimension * (heads_kv * v_id + h) + d; v_g[shift_v] = grad; } } //--- Calculating Query's gradients float grad = 0; float out_g = gradient[shift_g + q_id * dimension]; int shift_val = h_kv * dimension + d; int shift_key = h_kv * dimension + d; //--- for(int k_id = 0; k_id < kunits; k_id++) { float sc_g = 0; float sc = scores[shift_s + k_id]; if(sc == 0) continue; for(int v_id = 0; v_id < kunits; v_id++) sc_g += scores[shift_s + v_id] * out_g * v[shift_val + v_id * heads_kv * dimension] * ((float)(k_id == v_id) - sc); grad += sc_g * k[shift_key + k_id * heads_kv * dimension]; } q_g[shift_q] = grad / koef; //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k_id = q_id; k_id < kunits; k_id += qunits) { int shift_k = dimension * (heads_kv * k_id + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + k_id; float val = v[shift_k]; for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k_id]; if(sc == 0) continue; for(int v_id = 0; v_id < kunits; v_id++) sc_g += scores[shift_sc + v_id] * gradient[shift_g + scr * dimension] * val * ((float)(k_id == v_id) - sc); grad += sc_g * q[shift_g + scr * heads * dimension]; } } k_g[shift_k] = grad / koef; } } } //+------------------------------------------------------------------+ //| Implements the diversity loss kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DiversityLoss(__global const float *data, __global float *grad, const int dimension, const int activation, const int add ) { const size_t main = get_global_id(0); const size_t loc = get_local_id(1); const size_t total = get_local_size(0); const size_t total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- float delts = 0; //--- for(int d = 0; d < dimension; d++) { const float value_main = IsNaNOrInf(data[main * dimension + d], 0); for(int i = loc; i < total; i += total_loc) { float value_slave = IsNaNOrInf(data[i * dimension + d], 0); float delta = value_main - value_slave; delts += IsNaNOrInf(delta * delta / total, 0); } } //--- float loss = exp(LocalSum(delts, 1, Temp)); float gr = 0; //--- for(int d = 0; d < dimension; d++) { const float value_main = IsNaNOrInf(data[main * dimension + d], 0); for(int i = loc; i < total; i += total_loc) { float value_slave = IsNaNOrInf(data[i * dimension + d], 0); gr += IsNaNOrInf(2 * loss * (value_main - value_slave) / total, 0); } //--- gr = LocalSum(gr, 1, Temp); if(loc == 0) { if(add > 0) grad[main * dimension + d] += Deactivation(gr, value_main, activation); else grad[main * dimension + d] = Deactivation(gr, value_main, activation); } } } //+------------------------------------------------------------------+ //| Implements the mh relative attention out kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHRelativeAttentionOut(__global const float *q, ///<[in] Matrix of Querys __global const float *k, ///<[in] Matrix of Keys __global const float *v, ///<[in] Matrix of Values __global const float *bk, ///<[in] Matrix of Positional Bias Keys __global const float *bv, ///<[in] Matrix of Positional Bias Values __global const float *gc, ///<[in] Global content bias vector __global const float *gp, ///<[in] Global positional bias vector __global float *score, ///<[out] Matrix of Scores __global float *out, ///<[out] Matrix of attention const int dimension ///< Dimension of Key ) { //--- init const int q_id = get_global_id(0); const int k_id = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_local_size(1); const int heads = get_global_size(2); const int shift_q = dimension * (q_id * heads + h); const int shift_kv = dimension * (heads * k_id + h); const int shift_gc = dimension * h; const int shift_s = kunits * (q_id * heads + h) + k_id; const int shift_pb = q_id * kunits + k_id; const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); float koef = sqrt((float)dimension); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- score float sc = 0; //--- for(int d = 0; d < dimension; d++) { float val_q = q[shift_q + d]; float val_k = k[shift_kv + d]; float val_bk = bk[shift_kv + d]; sc += val_q * val_k + val_q * val_bk + val_k * val_bk + gc[shift_q + d] * val_k + gp[shift_q + d] * val_bk; } sc = sc / koef; //--- max value //--- for(int cur_k = 0; cur_k < kunits; cur_k += ls) { if(k_id >= cur_k && k_id < (cur_k + ls)) { int shift_local = k_id % ls; temp[shift_local] = (cur_k == 0 ? sc : fmax(temp[shift_local], sc)); } BarrierLoc } uint count = min(ls, (uint)kunits); //--- do { count = (count + 1) / 2; if(k_id < ls) temp[k_id] = (k_id < count && (k_id + count) < kunits ? fmax(temp[k_id + count], temp[k_id]) : temp[k_id]); BarrierLoc } while(count > 1); sc = IsNaNOrInf(exp(fmax(sc - temp[0], -120)), 0); BarrierLoc //--- sum of exp //--- for(int cur_k = 0; cur_k < kunits; cur_k += ls) { if(k_id >= cur_k && k_id < (cur_k + ls)) { int shift_local = k_id % ls; temp[shift_local] = (cur_k == 0 ? 0 : temp[shift_local]) + sc; } BarrierLoc } //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k_id < ls) temp[k_id] += (k_id < count && (k_id + count) < kunits ? temp[k_id + count] : 0); if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- score float sum = IsNaNOrInf(temp[0], 1); if(fabs(sum) <= 1.2e-7f) sum = 1; sc /= sum; score[shift_s] = sc; BarrierLoc //--- out //--- for(int d = 0; d < dimension; d++) { float val_v = v[shift_kv + d]; float val_bv = bv[shift_kv + d]; float val = IsNaNOrInf(sc * (val_v + val_bv), 0); //--- sum of value for(int cur_v = 0; cur_v < kunits; cur_v += ls) { if(k_id >= cur_v && k_id < (cur_v + ls)) { int shift_local = k_id % ls; temp[shift_local] = (cur_v == 0 ? 0 : temp[shift_local]) + val; } BarrierLoc } //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k_id < count && (k_id + count) < kunits) temp[k_id] += temp[k_id + count]; if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- if(k_id == 0) out[shift_q + d] = IsNaNOrInf(temp[0], 0); BarrierLoc } } //+------------------------------------------------------------------+ //| Computes the mh relative attention inside gradients gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MHRelativeAttentionInsideGradients(__global const float *q, __global float *q_g, __global const float *k, __global float *k_g, __global const float *v, __global float *v_g, __global const float *bk, __global float *bk_g, __global const float *bv, __global float *bv_g, __global const float *gc, __global float *gc_g, __global const float *gp, __global float *gp_g, __global const float *scores, __global const float *gradient, const int kunits ) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (q_id * heads + h) * kunits; const int shift_g = h * dimension + d; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- Calculating Value's gradients int step_score = kunits * heads; //--- for(int v_id = q_id; v_id < kunits; v_id += qunits) { float grad = 0; int shift_score = h * kunits + v_id; for(int g = 0; g < qunits; g++) grad += gradient[shift_g + dimension * (g * heads)] * scores[shift_score + g * step_score]; int shift_v = dimension * (heads * v_id + h) + d; grad = IsNaNOrInf(grad, 0); v_g[shift_v] = grad; bv_g[shift_v] = grad; } //--- Calculating Query's gradients float grad_gc = 0; float grad_gp = 0; float out_g = gradient[shift_g + q_id * dimension]; int shift_val = h * dimension + d; int shift_key = h * dimension + d; //--- for(int k_id = 0; k_id < kunits; k_id++) { float sc_g = 0; float sc = scores[shift_s + k_id]; if(sc == 0) continue; for(int v_id = 0; v_id < kunits; v_id++) sc_g += scores[shift_s + v_id] * out_g * (v[shift_val + v_id * heads * dimension] + bv[shift_val + v_id * heads * dimension]) * ((float)(k_id == v_id) - sc); grad_gc += IsNaNOrInf(sc_g * k[shift_key + k_id * heads * dimension], 0); grad_gp += IsNaNOrInf(sc_g * bk[shift_key + k_id * heads * dimension], 0); } //--- q_g[shift_q] = (grad_gc + grad_gp) / koef; gc_g[shift_q] = grad_gc / koef; gp_g[shift_q] = grad_gp / koef; //--- Calculating Key's gradients //--- for(int k_id = q_id; k_id < kunits; k_id += qunits) { int shift_k = dimension * (heads * k_id + h) + d; float grad = 0; float grad_bk = 0; int shift_score = h * kunits + k_id; float val = (v[shift_k] + bv[shift_k]); for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k_id]; if(sc == 0) continue; for(int v_id = 0; v_id < kunits; v_id++) sc_g += scores[shift_sc + v_id] * gradient[shift_g + scr * dimension] * val * ((float)(k_id == v_id) - sc); float _q = q[shift_g + scr * heads * dimension]; grad += sc_g * (_q + bk[shift_k] + gc[shift_g + scr * heads * dimension]); grad_bk += sc_g * (_q + k[shift_k] + gp[shift_g + scr * heads * dimension]); } k_g[shift_k] = grad / koef; bk_g[shift_k] = grad_bk / koef; } } //+------------------------------------------------------------------+ //| Computes the calc alignment gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void CalcAlignmentGradient(__global const float *matrix_o1, __global const float *matrix_o2, __global float *matrix_g1, __global float *matrix_g2, const int activation, const int add) { int i = get_global_id(0); const float out1 = IsNaNOrInf(matrix_o1[i], 0); const float out2 = IsNaNOrInf(matrix_o2[i], 0); float grad1 = Deactivation(out2 - out1, out1, activation); float grad2 = Deactivation(out1 - out2, out2, activation); //--- if(add > 0) { matrix_g1[i] += grad1; matrix_g2[i] += grad2; } else { matrix_g1[i] = grad1; matrix_g2[i] = grad2; } } //+------------------------------------------------------------------+ //| Implements the feature smoothing kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void FeatureSmoothing(__global const float *feature, __global float *outputs, const int smoothing ) { const size_t pos = get_global_id(0); const size_t d = get_global_id(1); const size_t total = get_global_size(0); const size_t dimension = get_global_size(1); //--- const int shift_input = pos * dimension + d; const int shift_output = dimension * pos * smoothing + d; //--- float value = IsNaNOrInf(feature[shift_input], 0); outputs[shift_output] = value; //--- for(int s = 1; s <= smoothing; s++) { if(((int)pos - s) >= 0) value += IsNaNOrInf(feature[shift_input - s * dimension], 0); if(((int)pos + s) < total) value += IsNaNOrInf(feature[shift_input + s * dimension], 0); float factor = IsNaNOrInf(1.0f / (min((int)total, (int)(pos + s)) - max((int)(pos - s), 0) + 1), 0); float out = IsNaNOrInf(value * factor, 0); outputs[shift_output + s * dimension] = out; } } //+------------------------------------------------------------------+ //| Computes the feature smoothing gradient gradient contributions on| //| the GPU. | //+------------------------------------------------------------------+ __kernel void FeatureSmoothingGradient(__global float *feature_gr, __global const float *outputs_gr, const int smoothing ) { const size_t pos = get_global_id(0); const size_t d = get_global_id(1); const size_t total = get_global_size(0); const size_t dimension = get_global_size(1); //--- const int shift_input = pos * dimension + d; const int shift_output = dimension * pos * smoothing + d; const int step_output = dimension * smoothing; //--- float grad = IsNaNOrInf(outputs_gr[shift_output], 0); //--- for(int s = 1; s <= smoothing; s++) { int shift = shift_output + s * dimension; float factor = 1.0f / (min((int)total, (int)(pos + s)) - max((int)(pos - s), 0) + 1); //--- float value = IsNaNOrInf(outputs_gr[shift] * factor, 0); //--- if(((int)pos - s) >= 0) grad += IsNaNOrInf(outputs_gr[shift - s * step_output] * factor, 0); //--- if(((int)pos + s) < total) grad += IsNaNOrInf(outputs_gr[shift + s * step_output] * factor, 0); } //--- feature_gr[shift_input] = IsNaNOrInf(grad, 0); } //+------------------------------------------------------------------+ //| Runs the batch feed forward add noise feed-forward pass on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void BatchFeedForwardAddNoise(__global const float *inputs, __global float *options, __global const float *noise, __global float *output, const int batch, const int optimization, const int activation, const float alpha) { if(batch <= 1) return; int n = get_global_id(0); int shift = n * (optimization == 0 ? 7 : 9); //--- float inp = inputs[n]; float mean = (batch > 1 ? (IsNaNOrInf(options[shift], 0) * ((float)batch - 1.0f) + inp) / ((float)batch) : inp); float delt = inp - mean; float variance = IsNaNOrInf(options[shift + 1], 0) * ((float)batch - 1.0f) + delt * delt; if(batch > 0) variance /= (float)batch; float nx = (variance > 0 ? delt / sqrt(variance) : 0); float noisex = sqrt(alpha) * nx + sqrt(1 - alpha) * fabs(noise[n]) * sign(nx); //--- float gamma = IsNaNOrInf(options[shift + 3], 0); if(gamma == 0) { options[shift + 3] = 1; gamma = 1; } float betta = IsNaNOrInf(options[shift + 4], 0); //--- options[shift] = mean; options[shift + 1] = variance; options[shift + 2] = nx; output[n] = fActivation(gamma * noisex + betta, activation); } //+------------------------------------------------------------------+ //| Implements the hyper projection kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void HyperProjection(__global const float *inputs, __global float *outputs ) { const size_t pos = get_global_id(0); const size_t d = get_local_id(1); const size_t total = get_global_size(0); const size_t dimension = get_local_size(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((int)dimension, (int)LOCAL_ARRAY_SIZE); //--- const int shift_in = pos * dimension + d; const int shift_out = pos * (dimension + 1) + d + 1; //--- float v = IsNaNOrInf(inputs[shift_in], 0); //--- float v2 = IsNaNOrInf(v * v, 0); //--- if(d < ls) temp[d] = v2; BarrierLoc //--- for(int i = ls; i < (int)dimension; i += ls) { if(d >= i && d < (i + ls)) temp[d % ls] += v2; BarrierLoc } //--- int count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); //--- outputs[shift_out] = v; if(d == 0) { v = IsNaNOrInf(((float)pos) / ((float)total), 0); outputs[shift_out - 1] = sqrt(fmax(temp[0] - v * v, 1.2e-07f)); } } //+------------------------------------------------------------------+ //| Computes the hyper projection grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void HyperProjectionGrad(__global const float *inputs, __global float *inputs_gr, __global const float *outputs_gr ) { const size_t pos = get_global_id(0); const size_t d = get_global_id(1); const size_t total = get_global_size(0); const size_t dimension = get_global_size(1); //--- const int shift_in = pos * dimension + d; const int shift_start_out = pos * (dimension + 1); const int shift_out = shift_start_out + d + 1; //--- float v = IsNaNOrInf(inputs[shift_in], 0); float grad = IsNaNOrInf(outputs_gr[shift_out], 0); //--- v = IsNaNOrInf(v * outputs_gr[shift_start_out], 0); //--- inputs_gr[shift_in] = v + grad; } //+------------------------------------------------------------------+ //| Implements the log map kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void LogMap(__global const float *features, __global const float *centroids, __global const float *curvatures, __global float *outputs, __global float *product, __global float *distance, __global float *norma ) { //--- identify const size_t f = get_global_id(0); const size_t cent = get_global_id(1); const size_t d = get_local_id(2); const size_t total_f = get_global_size(0); const size_t total_cent = get_global_size(1); const size_t dimension = get_local_size(2); //--- create local array __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((int)dimension, (int)LOCAL_ARRAY_SIZE); //--- calc shifts const int shift_f = f * dimension + d; const int shift_out = (f * total_cent + cent) * dimension + d; const int shift_cent = cent * dimension + d; const int shift_temporal = f * total_cent + cent; //--- load inputs float feature = IsNaNOrInf(features[shift_f], 0); float centroid = IsNaNOrInf(centroids[shift_cent], 0); float curv = IsNaNOrInf(curvatures[cent], 1.2e-7f); //--- dot(features, centroids) float fc = IsNaNOrInf(feature * centroid, 0); //--- if(d < ls) temp[d] = (d > 0 ? fc : -fc); BarrierLoc //--- for(int i = ls; i < (int)dimension; i += ls) { if(d >= i && d < (i + ls)) temp[d % ls] += fc; BarrierLoc } //--- int count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); float prod = IsNaNOrInf(temp[0], 0); product[shift_temporal] = prod; //--- project float u = IsNaNOrInf(feature + prod * centroid * curv, 0); //--- norm(u) float u2 = IsNaNOrInf(u * u, 0); //--- if(d < ls) temp[d] = (d > 0 ? u2 : -u2); BarrierLoc //--- for(int i = ls; i < (int)dimension; i += ls) { if(d >= i && d < (i + ls)) temp[d % ls] += u2; BarrierLoc } //--- count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); float normu = IsNaNOrInf(temp[0], 0); if(normu <= 0) normu = 1.0e-7f; normu = sqrt(normu); norma[shift_temporal] = normu; //--- distance features to centroid float theta = IsNaNOrInf(-prod * curv, 0); theta = fmax(theta, 1.0f + 1.2e-07f); float acosh_theta = acosh(theta); float dist = IsNaNOrInf(sqrt(clamp((acosh_theta * acosh_theta) / curv, 0.0f, 50.0f)), 0); distance[shift_temporal] = dist; float proj_u = IsNaNOrInf(dist * u / normu, 0); //--- if(d < ls) temp[d] = (d > 0 ? proj_u * centroid : 0); BarrierLoc //--- for(int i = ls; i < (int)dimension; i += ls) { if(d >= i && d < (i + ls)) temp[d % ls] += proj_u * centroid; BarrierLoc } //--- count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); //--- if(d == 0) { proj_u = IsNaNOrInf(temp[0] / centroid, 0); proj_u = fmax(u, 1.2e-7f); } //--- outputs[shift_out] = proj_u; } //+------------------------------------------------------------------+ //| Computes the log map grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void LogMapGrad(__global const float *features, __global float *features_gr, __global const float *centroids, __global float *centroids_gr, __global const float *curvatures, __global float *curvatures_gr, __global const float *outputs, __global const float *outputs_gr, __global const float *product, __global const float *distance, __global const float *norma ) { //--- identify const size_t f = get_global_id(0); const size_t cent = get_global_id(1); const size_t d = get_local_id(2); const size_t total_f = get_global_size(0); const size_t total_cent = get_global_size(1); const size_t dimension = get_local_size(2); //--- create local array __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((int)dimension, (int)LOCAL_ARRAY_SIZE); //--- calc shifts const int shift_f = f * dimension + d; const int shift_out = (f * total_cent + cent) * dimension + d; const int shift_cent = cent * dimension + d; const int shift_temporal = f * total_cent + cent; //--- load inputs float feature = features[shift_f]; if(isinf(feature) || isnan(feature)) feature = 0; float centroid = centroids[shift_cent]; if(isinf(centroid) || isnan(centroid)) centroid = 0; float centroid0 = (d > 0 ? centroids[shift_cent - d] : centroid); if(isinf(centroid0) || isnan(centroid0) || centroid0 == 0) centroid0 = 1.2e-7f; float curv = curvatures[cent]; if(isinf(curv) || isnan(curv)) curv = 1.2e-7f; float prod = product[shift_temporal]; float dist = distance[shift_temporal]; float normu = norma[shift_temporal]; float u = feature + prod * centroid * curv; if(isinf(u) || isnan(u)) u = 0; //--- float grad = outputs_gr[shift_out]; if(isinf(grad) || isnan(grad)) grad = 0; float grad0 = (d > 0 ? outputs_gr[shift_out - d] : grad); if(isinf(grad0) || isnan(grad0)) grad0 = 0; //--- float feature_gr = 0; float centroid_gr = 0; float curv_gr = 0; float prod_gr = 0; float normu_gr = 0; float dist_gr = 0; //--- float proj_u_gr = (d > 0 ? grad + grad0 / centroid0 * centroid : 0); if(d == 0) centroid_gr += outputs[shift_out] / centroid * grad; else centroid_gr += grad0 / centroid0 * outputs[shift_out]; if(isnan(centroid_gr) || isinf(centroid_gr)) centroid_gr = 0; //--- dist_gr = u / normu * proj_u_gr; float u_gr = dist / normu * proj_u_gr; normu_gr = dist * u / (normu * normu) * proj_u_gr; //--- if(d < ls) temp[d] = dist_gr; BarrierLoc //--- for(int id = ls; id < (int)dimension; id += ls) { if(d >= id && d < (id + ls)) temp[d % ls] += dist_gr; BarrierLoc } //--- int count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); if(isinf(temp[0]) || isnan(temp[0])) temp[0] = 0; dist_gr = temp[0]; //--- if(d == 0) { float theta = -prod * curv; float theta_gr = 1.0f / sqrt(curv * (theta * theta - 1)) * dist_gr; if(isinf(theta_gr) || isnan(theta_gr)) theta_gr = 0; float acosh_theta = acosh(theta); curv_gr += -(acosh_theta * acosh_theta) / (2 * sqrt(curv * curv * curv)) * dist_gr; if(isinf(curv_gr) || isnan(curv_gr)) curv_gr = 0; temp[0] = -curv * theta_gr; if(isinf(temp[0]) || isnan(temp[0])) temp[0] = 0; curv_gr += -prod * theta_gr; if(isinf(curv_gr) || isnan(curv_gr)) curv_gr = 0; } BarrierLoc //--- prod_gr += temp[0]; BarrierLoc //--- if(d < ls) temp[d] = normu_gr; BarrierLoc //--- for(int id = ls; id < (int)dimension; id += ls) { if(d >= id && d < (id + ls)) temp[d % ls] += normu_gr; BarrierLoc } //--- count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); normu_gr = temp[0]; if(isinf(normu_gr) || isnan(normu_gr)) normu_gr = 1.2e-7f; u_gr += u / normu * normu_gr; if(isnan(u_gr) || isinf(u_gr)) u_gr = 0; //--- feature_gr += u_gr; centroid_gr += prod * curv * u_gr; BarrierLoc //--- dot (u_gr * centroid) if(d < ls) temp[d] = u_gr * centroid; BarrierLoc //--- for(int id = ls; id < (int)dimension; id += ls) { if(d >= id && d < (id + ls)) temp[d % ls] += u_gr * centroid; BarrierLoc } //--- count = min(ls, (int)dimension); //--- do { count = (count + 1) / 2; if(d < count) temp[d] += ((d + count) < dimension ? temp[d + count] : 0); if(d + count < dimension) temp[d + count] = 0; BarrierLoc } while(count > 1); if(d == 0) { if(isinf(temp[0]) || isnan(temp[0])) temp[0] = 0; prod_gr += temp[0] * curv; if(isinf(prod_gr) || isnan(prod_gr)) prod_gr = 0; curv_gr += temp[0] * prod; if(isinf(curv_gr) || isnan(curv_gr)) curv_gr = 0; temp[0] = prod_gr; } BarrierLoc //--- prod_gr = temp[0]; feature_gr += prod_gr * centroid * (d > 0 ? 1 : -1); centroid_gr += prod_gr * feature * (d > 0 ? 1 : -1); //--- result features_gr[shift_f] += feature_gr; centroids_gr[shift_cent] += centroid_gr; if(f == 0 && d == 0) curvatures_gr[cent] += curv; } //+------------------------------------------------------------------+ //| Calculates calc epsilon weights on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcEpsilonWeights(__global const float *matrix_w, __global const float *matrix_g, __global const float *matrix_i, __global float *matrix_epsw, const float rho ) { const size_t inp = get_local_id(0); const size_t inputs = get_local_size(0) - 1; const size_t out = get_global_id(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((int)inputs, (int)LOCAL_ARRAY_SIZE); //--- const int shift_w = out * (inputs + 1) + inp; const float w = IsNaNOrInf(matrix_w[shift_w], 0); float grad = fabs(w) * IsNaNOrInf(matrix_g[out], 0) * (inputs == inp ? 1.0f : IsNaNOrInf(matrix_i[inp], 0)); //--- const int local_shift = inp % ls; //--- for(int i = 0; i <= inputs; i += ls) { if(i <= inp && inp < (i + ls)) temp[local_shift] = (i == 0 ? 0 : temp[local_shift]) + IsNaNOrInf(grad * grad, 0); BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(inp < count) temp[inp] += ((inp + count) < inputs ? IsNaNOrInf(temp[inp + count], 0) : 0); if(inp + count < inputs) temp[inp + count] = 0; BarrierLoc } while(count > 1); //--- float norm = sqrt(IsNaNOrInf(temp[0], 0)); float epsw = IsNaNOrInf(w * w * grad * rho / (norm + 1.2e-7f), w); //--- matrix_epsw[shift_w] = epsw; } //+------------------------------------------------------------------+ //| Calculates calc epsilon weights conv on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcEpsilonWeightsConv(__global const float *matrix_w, __global const float *matrix_g, __global const float *matrix_i, __global float *matrix_epsw, const int inputs, const float rho, const int step ) { //--- const size_t inp = get_local_id(0); const size_t window_in = get_local_size(0) - 1; const size_t out = get_global_id(1); const size_t window_out = get_global_size(1); const size_t v = get_global_id(2); const size_t variables = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; const int ls = min((int)(window_in + 1), (int)LOCAL_ARRAY_SIZE); //--- const int shift_w = (out + v * window_out) * (window_in + 1) + inp; const int total = (inputs - window_in + step - 1) / step; const int shift_out = v * total * window_out + out; const int shift_in = v * inputs + inp; const float w = IsNaNOrInf(matrix_w[shift_w], 0); //--- float grad = 0; //--- for(int t = 0; t < total; t++) { if(inp != window_in && (inp + t * step) >= inputs) break; float g = IsNaNOrInf(matrix_g[t * window_out + shift_out], 0); float i = IsNaNOrInf(inp == window_in ? 1.0f : matrix_i[t * step + shift_in], 0); grad += IsNaNOrInf(g * i, 0); } grad *= fabs(w); //--- const int local_shift = inp % ls; //--- for(int i = 0; i <= inputs; i += ls) { if(i <= inp && inp < (i + ls)) temp[local_shift] = (i == 0 ? 0 : temp[local_shift]) + IsNaNOrInf(grad * grad, 0); BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(inp < count && (inp + count) < inputs) { temp[inp] += IsNaNOrInf(temp[inp + count], 0); temp[inp + count] = 0; } BarrierLoc } while(count > 1); //--- float norm = sqrt(IsNaNOrInf(temp[0], 0)); float epsw = IsNaNOrInf(w * w * grad * rho / (norm + 1.2e-7f), w); //--- matrix_epsw[shift_w] = epsw; } //+------------------------------------------------------------------+ //| Implements the plr multi agents kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PLRMultiAgents(__global const float *inputs, __global float *outputs, __global int *isttp, const int transpose, __global const float *min_step ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); const size_t a = get_global_id(2); const size_t agents = get_global_size(2); //--- constants const int shift_in = ((bool)transpose ? (i * variables + v) : (v * lenth + i)); const int step_in = ((bool)transpose ? variables : 1); const int shift_ag = a * lenth * variables; //--- look for ttp float value = IsNaNOrInf(inputs[shift_in], 0); bool bttp = false; if(i == 0 || i == lenth - 1) bttp = true; else { float prev = value; int prev_pos = i; float max_v = value; float max_pos = i; float min_v = value; float min_pos = i; while(fmax(fabs(prev - max_v), fabs(prev - min_v)) < min_step[a] && prev_pos > 0) { prev_pos--; prev = IsNaNOrInf(inputs[shift_in - (i - prev_pos) * step_in], 0); if(prev >= max_v && (prev - min_v) < min_step[a]) { max_v = prev; max_pos = prev_pos; } if(prev <= min_v && (max_v - prev) < min_step[a]) { min_v = prev; min_pos = prev_pos; } } //--- float next = value; int next_pos = i; while(fmax(fabs(next - max_v), fabs(next - min_v)) < min_step[a] && next_pos < (lenth - 1)) { next_pos++; next = IsNaNOrInf(inputs[shift_in + (next_pos - i) * step_in], 0); if(next > max_v && (next - min_v) < min_step[a]) { max_v = next; max_pos = next_pos; } if(next < min_v && (max_v - next) < min_step[a]) { min_v = next; min_pos = next_pos; } } //--- if( (value >= prev && value > next) || (value > prev && value == next) || (value <= prev && value < next) || (value < prev && value == next) ) if(max_pos == i || min_pos == i) bttp = true; } //--- isttp[shift_in + shift_ag] = (int)bttp; outputs[shift_in + shift_ag] = 0; BarrierLoc //--- calc position int pos = -1; int prev_in = 0; int prev_ttp = 0; if(bttp) { pos = 0; //--- for(int p = 0; p < i; p++) { int current_in = ((bool)transpose ? (p * variables + v) : (v * lenth + p)); if((bool)isttp[current_in + shift_ag]) { pos++; prev_ttp = p; prev_in = current_in; } } } //--- cacl tendency if(pos > 0 && pos < (lenth / 3)) { float sum_x = 0; float sum_y = 0; float sum_xy = 0; float sum_xx = 0; int dist = i - prev_ttp; //--- for(int p = 0; p < dist; p++) { float x = (float)(p); float y = IsNaNOrInf(inputs[prev_in + p * step_in], 0); sum_x += x; sum_y += y; sum_xy += x * y; sum_xx += x * x; } float slope = IsNaNOrInf((dist * sum_xy - sum_x * sum_y) / (dist > 1 ? (dist * sum_xx - sum_x * sum_x) : 1), 0); float intercept = IsNaNOrInf((sum_y - slope * sum_x) / dist, 0); int shift_out = ((bool)transpose ? ((pos - 1) * 3 * variables + v) : (v * lenth + (pos - 1) * 3)) + shift_ag; outputs[shift_out] = slope; outputs[shift_out + step_in] = intercept; outputs[shift_out + 2 * step_in] = ((float)dist) / lenth; } else { if(pos == (lenth / 3)) { float sum_x = 0; float sum_y = 0; float sum_xy = 0; float sum_xx = 0; int dist = lenth - prev_ttp; //--- for(int p = 0; p < dist; p++) { float x = (float)(p); float y = IsNaNOrInf(inputs[prev_in + p * step_in], 0); sum_x += x; sum_y += y; sum_xy += x * y; sum_xx += x * x; } float slope = IsNaNOrInf((dist * sum_xy - sum_x * sum_y) / (dist > 1 ? (dist * sum_xx - sum_x * sum_x) : 1), 0); float intercept = IsNaNOrInf((sum_y - slope * sum_x) / dist, 0); int shift_out = ((bool)transpose ? ((pos - 1) * 3 * variables + v) : (v * lenth + (pos - 1) * 3)) + shift_ag; outputs[shift_out] = slope; outputs[shift_out + step_in] = intercept; outputs[shift_out + 2 * step_in] = IsNaNOrInf((float)dist / lenth, 0); } } } //+------------------------------------------------------------------+ //| Computes the plr multi agents gradient gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void PLRMultiAgentsGradient(__global float *inputs_gr, __global const float *outputs, __global const float *outputs_gr, const int transpose, const int agents ) { const size_t i = get_global_id(0); const size_t lenth = get_global_size(0); const size_t v = get_global_id(1); const size_t variables = get_global_size(1); //--- constants const int shift_in = ((bool)transpose ? (i * variables + v) : (v * lenth + i)); const int step_in = ((bool)transpose ? variables : 1); const int shift_out = ((bool)transpose ? v : (v * lenth)); const int step_out = 3 * step_in; const int shift_ag = lenth * variables; //--- Sum gradient float grad = 0; //--- for(int a = 0; a < agents; a++) { //--- calc position int pos = -1; int prev_in = 0; int dist = 0; do { pos++; prev_in += dist; dist = (int)fmax(outputs[shift_out + pos * step_out + 2 * step_in + a * shift_ag] * lenth, 1); } while(!(prev_in <= i && (prev_in + dist) > i)); //--- calc constants float sum_x = 0; float sum_xx = 0; for(int p = 0; p < dist; p++) { float x = (float)(p); sum_x += x; sum_xx += x * x; } //--- get output gradient float grad_slope = IsNaNOrInf(outputs_gr[shift_out + pos * step_out + a * shift_ag], 0); float grad_intercept = IsNaNOrInf(outputs_gr[shift_out + pos * step_out + step_in + a * shift_ag], 0); //--- calc gradient grad_slope -= IsNaNOrInf(sum_x / dist * grad_intercept, 0); grad_slope /= fmax(IsNaNOrInf(dist * sum_xx - sum_x * sum_x, 0), 1); grad += IsNaNOrInf(grad_intercept / dist, 0); grad += IsNaNOrInf((dist * (i - prev_in) - sum_x) * grad_slope, 0); } grad = clamp(IsNaNOrInf(grad / agents, 0), -MAX_GRAD, MAX_GRAD); //--- save result inputs_gr[shift_in] = grad; } //+------------------------------------------------------------------+ //| Runs the feed forward mh conv feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardMHConv(__global float *matrix_w, __global float *matrix_i, __global float *matrix_o, const int inputs, const int step, const int window_in, const int window_out, const int activation ) { const size_t i = get_global_id(0); const size_t h = get_global_id(1); const size_t v = get_global_id(2); const size_t total = get_global_size(0); const size_t heads = get_global_size(1); //--- const int window_in_h = (window_in + heads - 1) / heads; const int window_out_h = (window_out + heads - 1) / heads; const int shift_out = window_out * i + window_out_h * h; const int shift_in = step * i + window_in_h * h; //--- const int shift_var_in = v * inputs; const int shift_var_out = v * window_out * total; const int shift_var_w = v * window_out * (window_in_h + 1); const int shift_w_h = h * window_out_h * (window_in_h + 1); //--- float sum = 0; float4 inp, weight; int stop = (window_in_h <= (inputs - shift_in) ? window_in_h : (inputs - shift_in)); stop = min(stop, (int)(window_in - h * window_in_h)); //--- //--- for(int out = 0; (out < window_out_h && (window_out_h * h + out) < window_out); out++) { int shift = (window_in_h + 1) * out + shift_w_h; for(int k = 0; k <= stop; k += 4) { switch(stop - k) { case 0: inp = (float4)(1, 0, 0, 0); weight = (float4)(matrix_w[shift_var_w + shift + window_in_h], 0, 0, 0); break; case 1: inp = (float4)(matrix_i[shift_var_in + shift_in + k], 1, 0, 0); weight = (float4)(matrix_w[shift_var_w + shift + k], matrix_w[shift_var_w + shift + window_in_h], 0, 0); break; case 2: inp = (float4)(matrix_i[shift_var_in + shift_in + k], matrix_i[shift_var_in + shift_in + k + 1], 1, 0); weight = (float4)(matrix_w[shift_var_w + shift + k], matrix_w[shift_var_w + shift + k + 1], matrix_w[shift_var_w + shift + window_in_h], 0); break; case 3: inp = (float4)(matrix_i[shift_var_in + shift_in + k], matrix_i[shift_var_in + shift_in + k + 1], matrix_i[shift_var_in + shift_in + k + 2], 1); weight = (float4)(matrix_w[shift_var_w + shift + k], matrix_w[shift_var_w + shift + k + 1], matrix_w[shift_var_w + shift + k + 2], matrix_w[shift_var_w + shift + shift_w_h]); break; default: inp = (float4)(matrix_i[shift_var_in + shift_in + k], matrix_i[shift_var_in + shift_in + k + 1], matrix_i[shift_var_in + shift_in + k + 2], matrix_i[shift_var_in + shift_in + k + 3]); weight = (float4)(matrix_w[shift_var_w + shift + k], matrix_w[shift_var_w + shift + k + 1], matrix_w[shift_var_w + shift + k + 2], matrix_w[shift_var_w + shift + k + 3]); break; } sum += IsNaNOrInf(dot(inp, weight), 0); } sum = IsNaNOrInf(sum, 0); //--- matrix_o[shift_var_out + out + shift_out] = fActivation(sum, activation);; } } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_gr /// Kernel of the Convolution neuron to transfer gradient //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientMHConv(__global float *matrix_w, __global float *matrix_g, __global float *matrix_o, __global float *matrix_ig, const int outputs, const int step, const int window_in, const int window_out, const int activation, const int shift_out, const int heads ) { const size_t i = get_global_id(0); const size_t inputs = get_global_size(0); const size_t v = get_global_id(1); //--- const int shift_var_in = v * inputs; const int shift_var_out = v * outputs; const int shift_var_w = v * window_out * (window_in + 1); const int window_in_h = (window_in + heads - 1) / heads; const int window_out_h = (window_out + heads - 1) / heads; //--- float sum = 0; float out = matrix_o[shift_var_in + i]; const int w_start = i % step; const int start = max((int)((i - window_in + step) / step), 0); int stop = (w_start + step - 1) / step; stop = min((int)((i + step - 1) / step + 1), stop) + start; if(stop > (outputs / window_out)) stop = outputs / window_out; //--- for(int k = start; k < stop; k++) { int head = (k % window_out) / window_out_h; for(int h = 0; h < window_out_h; h ++) { if((head * window_out_h + h) >= window_out) break; int shift_g = k * window_out + head * window_out_h + h; int shift_w = (stop - k - 1) * step + (i % step) / window_in_h + head * (window_in_h + 1) + h * (window_in_h + 1); if(shift_g >= outputs || shift_w >= (window_in_h + 1) * window_out) break; sum += IsNaNOrInf(matrix_g[shift_out + shift_g + shift_var_out] * matrix_w[shift_w + shift_var_w], 0); } } //--- matrix_ig[shift_var_in + i] = Deactivation(sum, out, activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_opt Convolution Neuron Adam optimization Updating /// Weights Calculation kernel /// Describes the process of Adam optimization weights for the Convolution /// Neuron (#CNeuronConvOCL). //+------------------------------------------------------------------+ __kernel void UpdateWeightsMHConvAdam(__global float *matrix_w, ///<[in,out] Weights matrix (m+1)*n, where m - ///< input window and n - output window __global const float *matrix_g, ///<[in] Tensor of gradients at current layer __global const float *matrix_i, ///<[in] Inputs tensor __global float *matrix_m, ///<[in] Matrix of first momentum __global float *matrix_v, ///<[in] Matrix of seconfd momentum const int inputs, ///< Number of inputs const float l, ///< Learning rates const float b1, ///< First momentum multiplier const float b2, ///< Second momentum multiplier const int window_in, ///< Size of input window const int window_out, ///< Size of output window const int step, ///< Step size const int heads ) { const size_t i = get_global_id(0); //--- const int window_in_h = (window_in + heads - 1) / heads; const int v = i / ((window_in_h + 1) * window_out); const int shift = i % window_out; const int shift_out = i / (window_in_h + 1) - v; const int total = (inputs - window_in + step - 1) / step; //--- const int shift_var_in = v * inputs; const int shift_var_out = v * total * window_out; //--- float grad = 0; //--- for(int t = 0; t < total; t++) { if(shift != window_in_h && (shift + t * window_in) >= inputs) break; grad += IsNaNOrInf(matrix_g[t * window_out + shift_out + shift_var_out] * (shift == window_in_h ? 1 : matrix_i[shift + t * step + shift_var_in]), 0); } float mt = IsNaNOrInf(clamp(b1 * matrix_m[i] + (1 - b1) * grad, -1.0e5f, 1.0e5f), 0); float vt = IsNaNOrInf(clamp(b2 * matrix_v[i] + (1 - b2) * (grad * grad), 1.0e-6f, 1.0e6f), 1.0e-6f); float weight = matrix_w[i] + l * mt / sqrt(vt); matrix_w[i] = weight; matrix_m[i] = mt; matrix_v[i] = vt; } //+------------------------------------------------------------------+ //| Implements the more less equal kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MoreLessEqual(__global const float *input, __global float *output) { const size_t i = get_global_id(0); const float value = IsNaNOrInf(input[i], 0); float result = 0; if(fabs(value) > 1.2e-7f) { if(value > 0) result = 1; else result = -1; } output[i] = result; } //+------------------------------------------------------------------+ //| Implements the multi scale relative attention out kernel on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void MultiScaleRelativeAttentionOut(__global const float *q, ///<[in] Matrix of Querys __global const float *k, ///<[in] Matrix of Keys __global const float *v, ///<[in] Matrix of Values __global const float *bk, ///<[in] Matrix of Positional Bias Keys __global const float *bv, ///<[in] Matrix of Positional Bias Values __global const float *gc, ///<[in] Global content bias vector __global const float *gp, ///<[in] Global positional bias vector __global float *score, ///<[out] Matrix of Scores __global float *out, ///<[out] Matrix of attention const int dimension ///< Dimension of Key ) { //--- init const uint q_id = get_global_id(0); const uint k_id = get_local_id(1); const uint h = get_global_id(2); const uint qunits = get_global_size(0); const uint kunits = get_local_size(1); const uint heads = get_global_size(2); const int shift_q = dimension * (q_id * heads + h); const int shift_kv = dimension * (heads * k_id + h); const int shift_gc = dimension * h; const int shift_s = kunits * (q_id * heads + h) + k_id; const int shift_pb = q_id * kunits + k_id; const uint ls = min((uint)get_local_size(1), (uint)LOCAL_ARRAY_SIZE); const uint window = max((uint)((kunits + h) / (h + 1)), min((uint)3, kunits)); float koef = sqrt((float)dimension); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- score float sc = 0; if(k_id < window) { //--- for(int d = 0; d < dimension; d++) { float val_q = q[shift_q + d]; float val_k = k[shift_kv + d]; float val_bk = bk[shift_kv + d]; sc += val_q * val_k + val_q * val_bk + val_k * val_bk + gc[shift_q + d] * val_k + gp[shift_q + d] * val_bk; } sc = sc / koef; } //--- max value //--- for(int cur_k = 0; cur_k < kunits; cur_k += ls) { if(k_id < window) if(k_id >= cur_k && k_id < (cur_k + ls)) { int shift_local = k_id % ls; temp[shift_local] = (cur_k == 0 ? sc : fmax(temp[shift_local], sc)); } BarrierLoc } uint count = min(ls, kunits); //--- //--- do { count = (count + 1) / 2; if(k_id < (window + 1) / 2) if(k_id < ls) temp[k_id] = (k_id < count && (k_id + count) < kunits ? fmax(temp[k_id + count], temp[k_id]) : temp[k_id]); BarrierLoc } while(count > 1); if(k_id < window) sc = IsNaNOrInf(exp(fmax(sc - temp[0], -120)), 0); BarrierLoc //--- sum of exp //--- for(int cur_k = 0; cur_k < kunits; cur_k += ls) { if(k_id >= cur_k && k_id < (cur_k + ls)) { int shift_local = k_id % ls; temp[shift_local] = (cur_k == 0 ? 0 : temp[shift_local]) + sc; } BarrierLoc } //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k_id < count && k_id < (window + 1) / 2) temp[k_id] += ((k_id + count) < kunits ? temp[k_id + count] : 0); if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- score float sum = IsNaNOrInf(temp[0], 1); if(sum <= 1.2e-7f) sum = 1; sc /= sum; score[shift_s] = sc; BarrierLoc //--- out int shift_local = k_id % ls; //--- for(int d = 0; d < dimension; d++) { float val_v = v[shift_kv + d]; float val_bv = bv[shift_kv + d]; float val = IsNaNOrInf(sc * (val_v + val_bv), 0); //--- sum of value for(int cur_v = 0; cur_v < kunits; cur_v += ls) { if(k_id >= cur_v && k_id < (cur_v + ls)) temp[shift_local] = (cur_v == 0 ? 0 : temp[shift_local]) + val; BarrierLoc } //--- count = min(ls, (uint)kunits); do { count = (count + 1) / 2; if(k_id < count && (k_id + count) < kunits) temp[k_id] += temp[k_id + count]; if(k_id + count < ls) temp[k_id + count] = 0; BarrierLoc } while(count > 1); //--- if(k_id == 0) out[shift_q + d] = IsNaNOrInf(temp[0], 0); BarrierLoc } } //+------------------------------------------------------------------+ //| Runs the ssm2 d feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void SSM2D_FeedForward(__global const float *ah, __global const float *b_time, __global const float *b_var, __global const float *px_time, __global const float *px_var, __global const float *c_time, __global const float *c_var, __global const float *delta_time, __global const float *delta_var, __global float *hidden, __global float *y ) { const size_t n = get_local_id(0); const size_t d = get_global_id(1); const size_t n_total = get_local_size(0); const size_t d_total = get_global_size(1); //--- Hidden state //--- for(int h = 0; h < 2; h++) { float new_h = ah[(2 * n + h) * d_total + d] + ah[(2 * n_total + 2 * n + h) * d_total + d]; if(h == 0) new_h += b_time[n] * px_time[n * d_total + d]; else new_h += b_var[n] * px_var[n * d_total + d]; hidden[(h * n_total + n)*d_total + d] = IsNaNOrInf(new_h, 0); } BarrierLoc //--- Output uint shift_c = n; uint shift_h1 = d; uint shift_h2 = shift_h1 + n_total * d_total; float value = 0; //--- for(int i = 0; i < n_total; i++) { value += IsNaNOrInf(c_time[shift_c] * delta_time[shift_c] * hidden[shift_h1], 0); value += IsNaNOrInf(c_var[shift_c] * delta_var[shift_c] * hidden[shift_h2], 0); shift_c += n_total; shift_h1 += d_total; shift_h2 += d_total; } //--- y[n * d_total + d] = IsNaNOrInf(value, 0); } //+------------------------------------------------------------------+ //| Computes the ssm2 d calc hidden gradient gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void SSM2D_CalcHiddenGradient(__global const float *ah, __global float *grad_ah, // Gradient with respect to ah __global const float *b_time, __global float *grad_b_time, // Gradient with respect to b_time __global const float *b_var, __global float *grad_b_var, // Gradient with respect to b_var __global const float *px_time, __global float *grad_px_time, // Gradient with respect to px_time __global const float *px_var, __global float *grad_px_var, // Gradient with respect to px_var __global const float *c_time, __global float *grad_c_time, // Gradient with respect to c_time __global const float *c_var, __global float *grad_c_var, // Gradient with respect to c_var __global const float *delta_time, __global float *grad_delta_time, // Gradient with respect to delta_time __global const float *delta_var, __global float *grad_delta_var, // Gradient with respect to delta_var __global const float *hidden, __global const float *grad_y // Gradient of loss with respect to y ) { //--- const size_t n = get_global_id(0); const size_t d = get_local_id(1); const size_t n_total = get_global_size(0); const size_t d_total = get_local_size(1); //--- Initialize indices for data access uint shift_c = n; uint shift_h1 = d; uint shift_h2 = shift_h1 + n_total * d_total; float grad_hidden1 = 0; float grad_hidden2 = 0; //--- Backpropagation: compute hidden gradients from y for(int i = 0; i < n_total; i++) { float grad = grad_y[i * d_total + d]; float c_t = c_time[shift_c]; float c_v = c_var[shift_c]; float delta_t = delta_time[shift_c]; float delta_v = delta_var[shift_c]; float h1 = hidden[shift_h1]; float h2 = hidden[shift_h2]; //-- Accumulate gradients for hidden states grad_hidden1 += IsNaNOrInf(grad * c_t * delta_t, 0); grad_hidden2 += IsNaNOrInf(grad * c_v * delta_v, 0); //--- Compute gradients for c_time, c_var, delta_time, delta_var grad_c_time[shift_c] += grad * delta_t * h1; grad_c_var[shift_c] += grad * delta_v * h2; grad_delta_time[shift_c] += grad * c_t * h1; grad_delta_var[shift_c] += grad * c_v * h2; //--- Update indices for the next element shift_c += n_total; shift_h1 += d_total; shift_h2 += d_total; } //--- Backpropagate through hidden -> ah, b_time, px_time for(int h = 0; h < 2; h++) { float grad_h = (h == 0) ? grad_hidden1 : grad_hidden2; //--- Store gradients in ah (considering its influence on two elements) grad_ah[(2 * n + h) * d_total + d] = grad_h; grad_ah[(2 * (n_total + n) + h) * d_total + d] = grad_h; } //--- Backpropagate through px_time and px_var (influenced by b_time and b_var) grad_px_time[n * d_total + d] = grad_hidden1 * b_time[n]; grad_px_var[n * d_total + d] = grad_hidden2 * b_var[n]; if(d == 0) { grad_b_time[n] = 0; grad_b_var[n] = 0; } BarrierLoc //--- Sum gradients over all d for b_time and b_var grad_b_time[n] += grad_hidden1 * px_time[n * d_total + d]; grad_b_var[n] += grad_hidden2 * px_var[n * d_total + d]; } //+------------------------------------------------------------------+ //| Implements the p scan kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PScan(__global const float* A, __global const float* X, __global const float* H, __global float* X_out) { const size_t idx = get_local_id(0); const size_t dim = get_global_id(1); const size_t L = get_local_size(0); const size_t D = get_global_size(1); const int num_steps = (int)log2((float)L); //--- __local float local_A[1024]; __local float local_X[1024]; __local float local_H[1024]; //--- Load data to local memory int offset = dim + idx * D; local_A[idx] = A[offset]; local_X[idx] = X[offset]; local_H[idx] = H[offset]; BarrierLoc //--- Scan //--- for(int step = 0; step < num_steps; step++) { int halfT = L >> (step + 1); if(idx < halfT) { int base = idx * 2; local_X[base + 1] += local_A[base + 1] * local_X[base]; local_X[base + 1] *= local_H[base + 1]; local_A[base + 1] *= local_A[base]; } BarrierLoc } //--- Save result X_out[offset] = local_X[idx]; } //+------------------------------------------------------------------+ //| Computes the p scan calc hidden gradient gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void PScan_CalcHiddenGradient(__global const float* A, __global float* grad_A, __global const float* X, __global float* grad_X, __global const float* H, __global float* grad_H, __global const float* grad_X_out) { const size_t idx = get_local_id(0); const size_t dim = get_global_id(1); const size_t L = get_local_size(0); const size_t D = get_global_size(1); const int num_steps = (int)log2((float)L); //--- __local float local_A[1024]; __local float local_X[1024]; __local float local_H[1024]; __local float local_grad_X[1024]; __local float local_grad_A[1024]; __local float local_grad_H[1024]; //--- Load data to local memory int offset = idx * D + dim; local_A[idx] = A[offset]; local_X[idx] = X[offset]; local_H[idx] = H[offset]; local_grad_X[idx] = grad_X_out[offset]; local_grad_A[idx] = 0.0f; local_grad_H[idx] = 0.0f; BarrierLoc //--- Reverse Scan (Backward) //--- for(int step = num_steps - 1; step >= 0; step--) { int halfT = L >> (step + 1); if(idx < halfT) { int base = idx * 2; // Compute gradients float grad_next = local_grad_X[base + 1] * local_H[base + 1]; local_grad_H[base + 1] = local_grad_X[base + 1] * local_X[base]; local_grad_A[base + 1] = local_grad_X[base + 1] * local_X[base]; local_grad_X[base] += local_A[base + 1] * grad_next; } BarrierLoc } //--- Save gradients grad_A[offset] = local_grad_A[idx]; grad_X[offset] = local_grad_X[idx]; grad_H[offset] = local_grad_H[idx]; } //+------------------------------------------------------------------+ //| Implements the diag mat mult kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DiagMatMult(__global const float *diag, __global const float *matr, __global float *result, int activation) { size_t row = get_global_id(0); size_t col = get_local_id(1); size_t var = get_global_id(2); size_t rows = get_global_size(0); size_t cols = get_local_size(1); //--- __local float local_diag[1]; if(cols == 0) local_diag[0] = diag[row + var * rows]; BarrierLoc //--- int shift = RCtoFlat(row, col, rows, cols, var); //--- float res = local_diag[0] * matr[shift]; //--- result[shift] = fActivation(res, activation); } //+------------------------------------------------------------------+ //| Computes the diag mat mult grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void DiagMatMultGrad(__global const float *diag, __global float *grad_diag, __global const float *matr, __global float *grad_matr, __global const float *grad_result) { size_t row = get_global_id(0); size_t col = get_local_id(1); size_t var = get_global_id(2); size_t rows = get_global_size(0); size_t cols = get_local_size(1); size_t vars = get_global_size(2); //--- __local float local_diag[LOCAL_ARRAY_SIZE]; if(cols == 0) local_diag[0] = diag[row + var * rows]; BarrierLoc //--- int shift = (row + var * rows) * cols + col; //--- float grad = grad_result[shift]; float inp = matr[shift]; //--- grad_matr[shift] = IsNaNOrInf(local_diag[0] * grad, 0); BarrierLoc //--- int loc = col % LOCAL_ARRAY_SIZE; //--- for(int c = 0; c < cols; c += LOCAL_ARRAY_SIZE) { if(c <= col && (c + LOCAL_ARRAY_SIZE) > col) { if(c == 0) local_diag[loc] = IsNaNOrInf(grad * inp, 0); else local_diag[loc] += IsNaNOrInf(grad * inp, 0); } BarrierLoc } //--- int count = min(LOCAL_ARRAY_SIZE, (int)cols); int ls = count; //--- do { count = (count + 1) / 2; if((col + count) < ls) { local_diag[col] += local_diag[col + count]; local_diag[col + count] = 0; } BarrierLoc } while(count > 1); //--- if(col == 0) grad_diag[row + var * rows] = IsNaNOrInf(local_diag[0], 0); } //+------------------------------------------------------------------+ //| Implements the top kgates kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void TopKgates(__global const float *inputs, __global const float *noises, __global float *gates, const uint k) { size_t idx = get_local_id(0); size_t var = get_global_id(1); size_t window = get_local_size(0); size_t vars = get_global_size(1); //--- const int shift_logit = var * 2 * window + idx; const int shift_std = shift_logit + window; const int shift_gate = var * window + idx; //--- float logit = IsNaNOrInf(inputs[shift_logit], MIN_VALUE); float noise = IsNaNOrInf(noises[shift_gate], 0); if(noise != 0) { noise *= fActivation(inputs[shift_std], 3); logit += IsNaNOrInf(noise, 0); } //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- const uint ls = min((uint)window, (uint)LOCAL_ARRAY_SIZE); uint bigger = 0; float max_logit = logit; //--- Top K for(int i = 0; i < window; i += ls) { if(idx >= i && idx < (i + ls)) temp[idx % ls] = logit; BarrierLoc for(int i1 = 0; (i1 < min((int)ls, (int)(window - i)) && bigger <= k); i1++) { if(temp[i1] > logit) bigger++; if(temp[i1] > max_logit) max_logit = temp[i1]; } BarrierLoc } //--- if(bigger <= k) gates[shift_gate] = logit - max_logit; else gates[shift_gate] = MIN_VALUE; } //+------------------------------------------------------------------+ //| Computes the top kgates grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void TopKgatesGrad(__global const float *inputs, __global float *grad_inputs, __global const float *noises, __global const float *gates, __global float *grad_gates) { size_t idx = get_global_id(0); size_t var = get_global_id(1); size_t window = get_global_size(0); size_t vars = get_global_size(1); //--- const int shift_logit = var * 2 * window + idx; const int shift_std = shift_logit + window; const int shift_gate = var * window + idx; //--- float grad = IsNaNOrInf(grad_gates[shift_gate], 0); grad_inputs[shift_logit] = grad; //--- float noise = IsNaNOrInf(noises[shift_gate], 0); if(noise == 0) { grad_inputs[shift_std] = 0; return; } //--- grad *= noise; grad_inputs[shift_std] = Deactivation(grad, fActivation(inputs[shift_std], 3), 3); } //+------------------------------------------------------------------+ //| Applies the mask by distance operation to the GPU buffers. | //+------------------------------------------------------------------+ __kernel void MaskByDistance(__global const float *buf_real, __global const float *buf_imag, __global float *mask, const int dimension ) { const size_t main = get_global_id(0); const size_t slave = get_local_id(1); const int total = (int)get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; int ls = min((int)total, (int)LOCAL_ARRAY_SIZE); //--- const int shift_main = main * dimension; const int shift_slave = slave * dimension; const int shift_mask = main * total + slave; //--- calc distance float dist = 0; if(main != slave) { //--- for(int d = 0; d < dimension; d++) { float delta = ComplexAbs((float2)(buf_real[shift_main + d], buf_imag[shift_main + d])) - ComplexAbs((float2)(buf_real[shift_slave + d], buf_imag[shift_slave + d])); dist += delta * delta; } dist = sqrt(dist); } //--- Look Max //--- for(int i = 0; i < total; i += ls) { if(i <= slave && (i + ls) > slave) Temp[slave % ls] = fmax((i == 0 ? 0 : Temp[slave % ls]), IsNaNOrInf(dist, 0)); BarrierLoc } //--- int count = ls; do { count = (count + 1) / 2; if(slave < count && (slave + count) < ls) { if(Temp[slave] < Temp[slave + count]) Temp[slave] = Temp[slave + count]; Temp[slave + count] = 0; } BarrierLoc } while(count > 1); //--- Normalize if(Temp[0] > 0) dist /= Temp[0]; //--- result mask[shift_mask] = 1 - IsNaNOrInf(dist, 1); } //+------------------------------------------------------------------+ //| Applies the mask attention operation to the GPU buffers. | //+------------------------------------------------------------------+ __kernel void MaskAttention(__global const float *q, ///<[in] Matrix of Querys __global const float *kv, ///<[in] Matrix of Keys __global float *scores, ///<[out] Matrix of Scores __global const float *masks, ///<[in] Mask Matrix __global float *out, ///<[out] Matrix of attention const int dimension, ///< Dimension of Key const int heads_kv ) { //--- init const int q_id = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_local_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_k = dimension * (2 * heads_kv * k + h_kv); const int shift_v = dimension * (2 * heads_kv * k + heads_kv + h_kv); const int shift_s = kunits * (q_id * heads + h) + k; const float mask = IsNaNOrInf(masks[q_id * kunits + k], 0); const uint ls = min((uint)kunits, (uint)LOCAL_ARRAY_SIZE); float koef = sqrt((float)dimension); if(koef < 1) koef = 1; __local float temp[LOCAL_ARRAY_SIZE]; //--- Score float score = 0; if(mask != 0) { for(int d = 0; d < dimension; d++) score += IsNaNOrInf(q[shift_q + d] * kv[shift_k + d], 0); score = IsNaNOrInf(exp(score / koef * mask), 0); } //--- sum of exp float sum = LocalSum(score, 1, temp); //--- score if(sum > 0) score /= sum; scores[shift_s] = score; //--- out for(int d = 0; d < dimension; d++) { float val = LocalSum(kv[shift_v + d] * score, 1, temp); if(k == 0) out[shift_q + d] = val; BarrierLoc } } //+------------------------------------------------------------------+ //| Computes the mask attention gradients gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void MaskAttentionGradients(__global const float *q, __global float *q_g, __global const float *kv, __global float *kv_g, __global const float *scores, __global const float *gradient, const int kunits, const int heads_kv ) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (q_id * heads + h) * kunits; const int shift_g = h * dimension + d; float koef = sqrt((float)dimension); if(koef < 1) koef = 1; //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v = q_id; v < kunits; v += qunits) { float grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v; for(int g = 0; g < qunits; g++) grad += gradient[shift_g + dimension * (hq - h + g * heads)] * scores[shift_score + g * step_score]; } int shift_v = dimension * (2 * heads_kv * v + heads_kv + h) + d; kv_g[shift_v] = grad; } } //--- Calculating Query's gradients float grad = 0; float out_g = IsNaNOrInf(gradient[shift_g + q_id * dimension], 0); int shift_val = (heads_kv + h_kv) * dimension + d; int shift_key = h_kv * dimension + d; //--- for(int k = 0; (k < kunits && out_g != 0); k++) { float sc_g = 0; float sc = scores[shift_s + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_s + v] * out_g * kv[shift_val + 2 * v * heads_kv * dimension] * ((float)(k == v) - sc); grad += sc_g * kv[shift_key + 2 * k * heads_kv * dimension]; } q_g[shift_q] = grad / koef; //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k = q_id; k < kunits; k += qunits) { int shift_k = dimension * (2 * heads_kv * k + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq++) { int shift_score = hq * kunits + k; float val = kv[shift_k + heads_kv * dimension]; for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_sc + v] * gradient[shift_g + scr * dimension] * val * ((float)(k == v) - sc); grad += sc_g * q[dimension * (h + scr * heads) + d]; } } kv_g[shift_k] = grad / koef; } } } //+------------------------------------------------------------------+ //| Runs the feed forward mult win conv feed-forward pass on the GPU.| //+------------------------------------------------------------------+ __kernel void FeedForwardMultWinConv(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_o, __global const int *windows_in, const int inputs, const int windows_total, const int window_out, const int activation ) { const size_t i = get_global_id(0); const size_t v = get_global_id(1); const size_t outputs = get_global_size(0); //--- const int block = window_out * windows_total; const int id = (int)(i % block); const int win_id = id / window_out; const int out_id = id % window_out; //--- int step = 0; int window_shift = 0; int shift_weight = 0; for(int w = 0; w < windows_total; w++) { int win = windows_in[w]; if(w < win_id) { window_shift += win; shift_weight += (win + 1) * window_out; } step += win; } int window_in = windows_in[win_id]; int steps = (int)(i / block); int shift_in = steps * step + window_shift; shift_weight += out_id * (window_in + 1); float sum = matrix_w[shift_weight + window_in]; float inp = 0.0f; //--- for(int w = 0; w < window_in; w++) if((shift_in + w) < inputs) { inp = IsNaNOrInf(matrix_i[v * inputs + shift_in + w], 0.0f); if(inp == 0.0f) continue; sum += IsNaNOrInf(inp * matrix_w[shift_weight + w], 0.0f); } //--- matrix_o[v * outputs + i] = fActivation(sum, activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_gr /// Kernel of the Convolution neuron to transfer gradient /// to previous layer (#CNeuronConvOCL) //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientMultWinConv(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_ig, __global const float *matrix_og, __global const int *windows_in, const int outputs, const int windows_total, const int window_out, const int activation ) { const size_t i = get_global_id(0); const size_t v = get_global_id(1); const size_t inputs = get_global_size(0); //--- int step = 0; //--- for(int w = 0; w < windows_total; w++) step += windows_in[w]; //--- int steps = (int)(i / step); int id = i % step; int window = 0; int before = 0; int window_in = 0; //--- for(int w = 0; w < windows_total; w++) { window_in = windows_in[w]; if(id < (before + window_in)) break; window = w + 1; before += window_in; } //--- int shift_weight = (before + window) * window_out + id - before; int shift_out = (steps * windows_total + window) * window_out + v * outputs; float sum = 0; //--- for(int w = 0; w < window_out; w++) { float grad = IsNaNOrInf(matrix_og[shift_out + w], 0.0f); if(grad == 0.0f) continue; sum += IsNaNOrInf(grad * matrix_w[shift_weight + w * (window_in + 1)], 0); } //--- matrix_ig[v * inputs + i] = Deactivation(sum, matrix_i[v * inputs + i], activation); } //+------------------------------------------------------------------+ ///\ingroup neuron_conv_opt Convolution Neuron Adam optimization Updating /// Weights Calculation kernel /// Describes the process of Adam optimization weights for the Convolution /// Neuron (#CNeuronConvOCL). //+------------------------------------------------------------------+ __kernel void UpdateWeightsMultWinConvAdam(__global float *matrix_w, __global const float *matrix_og, __global const float *matrix_i, __global float *matrix_m, __global float *matrix_v, __global const int *windows_in, const int windows_total, const int window_out, const int inputs, const int outputs, const float l, const float b1, const float b2 ) { const size_t i = get_global_id(0); // weight shift const size_t v = get_local_id(1); // variable const size_t variables = get_local_size(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- int step_out = window_out * windows_total; //--- int step_in = 0; for(int w = 0; w < windows_total; w++) step_in += windows_in[w]; //--- int window_shift = 0; int shift_in = 0; int shift_out = 0; int window = 0; int weight_shift = 0; int local_weight = 0; //--- for(int w = 0; w < windows_total; w++) { int win = windows_in[w]; int block = (win + 1) * window_out; if(i < weight_shift + block) { shift_in = window_shift; shift_out = w * window_out; window = win; local_weight = i - weight_shift; break; } window_shift += win; weight_shift += block; } bool bias = ((local_weight % (window + 1)) == window); int t = local_weight / (window + 1); int input_offset = local_weight % (window + 1); shift_out += t; shift_in += input_offset; //--- float grad = 0; int total = (inputs + step_in - 1) / step_in; //--- for(int t = 0; t < total; t++) { int sh_out = t * step_out + shift_out; if(sh_out >= outputs) break; if(bias) { grad += IsNaNOrInf(matrix_og[v * outputs + sh_out], 0); continue; } //--- int sh_in = t * step_in + shift_in; if(sh_in >= inputs) break; float grad_out = IsNaNOrInf(matrix_og[v * outputs + sh_out], 0.0f); if(grad_out == 0.0f) continue; float inp = IsNaNOrInf(matrix_i[v * inputs + sh_in], 0.0f); if(inp == 0.0f) continue; grad += IsNaNOrInf(grad_out * inp, 0); } //--- sum grad = LocalSum(grad, 1, temp); //--- if(v == 0) { float mt = IsNaNOrInf(clamp(b1 * matrix_m[i] + (1 - b1) * grad, -1.0e5f, 1.0e5f), 0); float vt = IsNaNOrInf(clamp(b2 * matrix_v[i] + (1 - b2) * (grad * grad), 1.0e-6f, 1.0e6f), 1.0e-6f); float weight = matrix_w[i] + IsNaNOrInf(l * mt / sqrt(vt), 0); matrix_w[i] = weight; matrix_m[i] = mt; matrix_v[i] = vt; } } //+------------------------------------------------------------------+ //| Applies the mask attention complex operation to the GPU buffers. | //+------------------------------------------------------------------+ __kernel void MaskAttentionComplex(__global const float2* __attribute__((aligned(8)))q, ///<[in] Matrix of Querys __global const float2* __attribute__((aligned(8)))kv, ///<[in] Matrix of Keys __global float *scores, ///<[out] Matrix of Scores __global const float *masks, ///<[in] Mask Matrix __global float2* __attribute__((aligned(8)))out, ///<[out] Matrix of attention const int dimension, ///< Dimension of Key const int heads_kv ) { //--- init const int q_id = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int kunits = get_local_size(1); const int heads = get_global_size(2); //--- const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h); const int shift_k = dimension * (2 * heads_kv * k + h_kv); const int shift_v = dimension * (2 * heads_kv * k + heads_kv + h_kv); const int shift_s = kunits * (q_id * heads + h) + k; const float mask = IsNaNOrInf(masks[shift_s], 0); const uint ls = min((uint)kunits, (uint)LOCAL_ARRAY_SIZE); float2 koef = (float2)(fmax((float)sqrt((float)dimension), (float)1), 0); __local float2 temp[LOCAL_ARRAY_SIZE]; //--- Score float score = 0; float2 score2 = (float2)0; if(ComplexAbs(mask) >= 0.01f) { for(int d = 0; d < dimension; d++) score2 += IsNaNOrInf2(ComplexMul(q[shift_q + d], kv[shift_k + d]), (float2)0); score = IsNaNOrInf(ComplexAbs(ComplexExp(ComplexDiv(score, koef))) * mask, 0); } //--- sum of exp //--- for(int i = 0; i < kunits; i += ls) { if(k >= i && k < (i + ls)) temp[k % ls].x = (i == 0 ? 0 : temp[k % ls].x) + score; BarrierLoc } //--- uint count = ls; //--- do { count = (count + 1) / 2; if(k < ls) temp[k].x += (k < count && (k + count) < kunits ? temp[k + count].x : 0); if(k + count < ls) temp[k + count].x = 0; BarrierLoc } while(count > 1); //--- score if(temp[0].x > 0) score = score / temp[0].x; scores[shift_s] = score; //--- out //--- for(int d = 0; d < dimension; d++) { float2 val = (score > 0 ? ComplexMul(kv[shift_v + d], (float2)(score, 0)) : (float2)0); //--- for(int i = 0; i < kunits; i += ls) { if(k >= i && k < (i + ls)) temp[k % ls] = (i == 0 ? (float2)0 : temp[k % ls]) + val; BarrierLoc } //--- uint count = ls; //--- do { count = (count + 1) / 2; if(k < ls) temp[k] += (k < count && (k + count) < kunits ? temp[k + count] : (float2)0); if((k + count) < ls) temp[k + count] = (float2)0; BarrierLoc } while(count > 1); //--- if(k == 0) out[shift_q + d] = temp[0]; BarrierLoc } } //+------------------------------------------------------------------+ //| Computes the mask attention gradients complex gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MaskAttentionGradientsComplex(__global const float2* __attribute__((aligned(8)))q, __global float2* __attribute__((aligned(8)))q_g, __global const float2* __attribute__((aligned(8)))kv, __global float2* __attribute__((aligned(8)))kv_g, __global const float *scores, __global const float *mask, __global float *mask_g, __global const float2* __attribute__((aligned(8)))gradient, const int kunits, const int heads_kv ) { //--- init const int q_id = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (q_id * heads + h) * kunits; const int shift_g = h * dimension + d; float2 koef = (float2)(fmax(sqrt((float)dimension), (float)1), 0); //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v = q_id; v < kunits; v += qunits) { float2 grad = (float2)0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v; for(int g = 0; g < qunits; g++) { float sc = IsNaNOrInf(scores[shift_score + g * step_score], 0); if(sc > 0) grad += ComplexMul(gradient[shift_g + dimension * (hq - h + g * heads)], (float2)(sc, 0)); } } int shift_v = dimension * (2 * heads_kv * v + heads_kv + h) + d; kv_g[shift_v] = grad; } } //--- Calculating Query's gradients float2 grad = 0; float2 out_g = IsNaNOrInf2(gradient[shift_g + q_id * dimension], (float2)0); int shift_val = (heads_kv + h_kv) * dimension + d; int shift_key = h_kv * dimension + d; //--- for(int k = 0; (k < kunits && ComplexAbs(out_g) != 0); k++) { float2 sc_g = 0; float2 sc = (float2)(scores[shift_s + k], 0); for(int v = 0; v < kunits; v++) sc_g += IsNaNOrInf2(ComplexMul( ComplexMul((float2)(scores[shift_s + v], 0), out_g * kv[shift_val + 2 * v * heads_kv * dimension]), ((float2)(k == v, 0) - sc)), (float2)0); float m = mask[shift_s + k]; mask_g[shift_s + k] = IsNaNOrInf(sc.x / m * sc_g.x + sc.y / m * sc_g.y, 0); grad += IsNaNOrInf2(ComplexMul(sc_g, kv[shift_key + 2 * k * heads_kv * dimension]), (float2)0); } q_g[shift_q] = IsNaNOrInf2(ComplexDiv(grad, koef), (float2)0); //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k = q_id; k < kunits; k += qunits) { int shift_k = dimension * (2 * heads_kv * k + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq++) { int shift_score = hq * kunits + k; float2 val = IsNaNOrInf2(kv[shift_k + heads_kv * dimension], (float2)0); for(int scr = 0; scr < qunits; scr++) { float2 sc_g = (float2)0; int shift_sc = scr * kunits * heads; float2 sc = (float2)(IsNaNOrInf(scores[shift_sc + k], 0), 0); if(ComplexAbs(sc) == 0) continue; for(int v = 0; v < kunits; v++) sc_g += IsNaNOrInf2( ComplexMul( ComplexMul((float2)(scores[shift_sc + v], 0), gradient[shift_g + scr * dimension]), ComplexMul(val, ((float2)(k == v, 0) - sc))), (float2)0); grad += IsNaNOrInf2(ComplexMul(sc_g, q[(h + scr * heads) * dimension + d]), (float2)0); } } kv_g[shift_k] = IsNaNOrInf2(ComplexDiv(grad, koef), (float2)0); } } } //+------------------------------------------------------------------+ //| Runs the cslstm feed forward feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void CSLSTM_FeedForward(__global const float4* __attribute__((aligned(16))) concatenated, __global float *memory, __global float *output) { uint id = (uint)get_global_id(0); uint total = (uint)get_global_size(0); // hidden size uint idv = (uint)get_global_id(1); uint total_v = (uint)get_global_size(1); // variables //--- uint shift = id + total * idv; float4 concat = concatenated[shift]; //--- float fg_s = fActivation(concat.s0, ActFunc_SIGMOID); float fg = 1 - fActivation(1 - 1 / (fg_s * fg_s), ActFunc_TANH); float ig = fActivation(fActivation(concat.s1, ActFunc_SIGMOID), ActFunc_TANH); float nc = fActivation(concat.s2, ActFunc_TANH); float og = fActivation(concat.s3, ActFunc_SIGMOID); float mem = IsNaNOrInf(memory[shift] * fg + ig * nc, 0); float out = IsNaNOrInf(og * fActivation(mem, ActFunc_TANH), 0); //--- memory[shift] = mem; output[shift] = out; } //+------------------------------------------------------------------+ //| Computes the cslstm calc hidden gradient gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void CSLSTM_CalcHiddenGradient(__global const float4* __attribute__((aligned(16))) concatenated, // Input from forward pass (W*x + U*h + b) __global float4* __attribute__((aligned(16))) grad_concat, // Output: gradients w.r.t. gate pre-activations __global const float* memory, // Updated memory (after forward pass) __global const float* grad_output // dL/dOutput from the next layer ) { uint id = get_global_id(0); // Index within sequences uint total = get_global_size(0); // Total size of sequences uint idv = get_global_id(1); // Index over independent univariate sequences (e.g., features or channels) in a multivariate time series uint shift = id + total * idv; // Flattened index //--- float4 concat = concatenated[shift]; // Pre-activation values for all 4 gates // --- Forward reconstruction of gates --- float fg_s = fActivation(concat.s0, ActFunc_SIGMOID); float fg = 1.0f - fActivation(1.0f - 1.0f / (fg_s * fg_s), ActFunc_TANH); // Forget gate (ft) float ig_s = fActivation(concat.s1, ActFunc_SIGMOID); float ig = fActivation(ig_s, ActFunc_TANH); // Input gate (it) float nc = fActivation(concat.s2, ActFunc_TANH); // Candidate (ct~) float og = fActivation(concat.s3, ActFunc_SIGMOID); // Output gate (ot) float mem = memory[shift]; // New memory state (ct) float mem_t = fActivation(mem, ActFunc_TANH); // tanh(ct) // --- Reconstruct previous memory state (t-1) --- float prev_mem = IsNaNOrInf((mem - ig * nc) / fg, 0); // --- Gradients computation --- float out_g = grad_output[shift]; float og_g = Deactivation(out_g * mem_t, og, ActFunc_SIGMOID); float mem_g = Deactivation(out_g * og, mem_t, ActFunc_TANH); float nc_g = Deactivation(mem_g * ig, nc, ActFunc_TANH); float ig_g = Deactivation(Deactivation(mem_g * nc, ig, ActFunc_TANH), ig_s, ActFunc_SIGMOID); // ∂L/∂fg = ∂L/∂ct * mem_(t-1) float fg_g = mem_g * prev_mem; // Derivative of the complex forget gate: // f(z) = 1 - tanh(1 - 1 / σ(z)^2) float fg_s_g = 2 / (fg_s * fg_s * fg_s) * Deactivation(-fg_g, fg, ActFunc_TANH); fg_g = Deactivation(fg_s_g, fg_s, ActFunc_SIGMOID); // --- Write back gradients --- grad_concat[shift] = (float4)(fg_g, ig_g, nc_g, og_g); } //+------------------------------------------------------------------+ //| Implements the prob attention qeury imp kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ProbAttentionQeuryImp(__global const float* querys, __global const float2* __attribute__((aligned(8))) keys_values, __global const float* index_keys, __global float* querys_imp, const int dimension ) { const size_t id_q = get_global_id(0); const size_t total_q = get_global_size(0); const size_t ind_k = get_local_id(1); const size_t total_ind = get_local_size(1); const size_t id_h = get_global_id(2); const size_t total_h = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE][2]; const int ls = min((int)total_ind, (int)LOCAL_ARRAY_SIZE); //--- const int shift_q = dimension * (id_q * total_h + id_h); const int id_k = index_keys[ind_k * total_h + id_h]; const int shift_k = dimension * (id_k * total_h + id_h); //--- float sum = 0; //--- for(int d = 0; d < dimension; d++) sum += IsNaNOrInf(querys[shift_q + d] * keys_values[shift_k + d].s0, 0); //--- int id_t = ind_k % ls; //--- for(int i = 0; i < total_ind; i += ls) { if(i <= ind_k || (i + ls) > ind_k) { temp[id_t][0] = IsNaNOrInf((i == 0 ? 0 : temp[id_t][0]) + sum, 0); temp[id_t][1] = (i == 0 ? IsNaNOrInf(sum, MIN_VALUE) : fmax(temp[id_t][1], IsNaNOrInf(sum, MIN_VALUE))); BarrierLoc } } int count = ls; //--- do { count = (count + 1) / 2; if(ind_k < count && (ind_k + count) < ls) { temp[ind_k][0] += temp[ind_k + count][0]; temp[ind_k + count][0] = 0; temp[ind_k][1] = fmax(temp[ind_k + count][1], temp[ind_k][1]); } BarrierLoc } while(count > 1); if(ind_k == 0) querys_imp[id_q * total_h + id_h] = IsNaNOrInf(temp[0][1] - temp[0][0] / total_ind, MIN_VALUE); } //+------------------------------------------------------------------+ //| Implements the top k importance to index kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void TopKImportanceToIndex(__global const float* importance, __global float* indexes, const int top_k ) { const size_t id_q = get_global_id(0); const size_t total_q = get_global_size(0); const size_t id_h = get_global_id(1); const size_t total_h = get_global_size(1); //--- float imp = importance[id_q * total_h + id_h]; int pos = 0; //--- for(int i = 0; i < total_q; i++) { if(i == id_q) continue; float val = importance[i * total_h + id_h]; if(val > imp || (i < id_q && val >= imp)) pos++; if(pos >= top_k) break; } //--- if(pos < top_k) indexes[pos * total_h + id_h] = (float)id_q; } //+------------------------------------------------------------------+ //| Implements the q index attention kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void QIndexAttention(__global const float *q, ///<[in] Matrix of Querys __global const float2* kv, ///<[in] Matrix of Keys __global float *scores, ///<[out] Matrix of Scores __global const float *indexes, ///<[in] Querys Indexes __global float *out, ///<[out] Matrix of attention const int dimension, ///< Dimension of Key const int heads_kv ) { //--- init const int ind_q = get_global_id(0); const int k = get_local_id(1); const int h = get_global_id(2); const int total_q = get_global_size(0); const int total_k = get_local_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int q_id = (int)(indexes[ind_q * heads + h] + 0.001f); const int shift_q = dimension * (q_id * heads + h); const int shift_kv = dimension * (heads_kv * k + h_kv); const int shift_s = total_k * (ind_q * heads + h) + k; //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Score float score = 0; if(q_id >= 0) { //--- for(int d = 0; d < dimension; d++) score += IsNaNOrInf(q[shift_q + d] * kv[shift_kv + d].s0, 0); } else score = MIN_VALUE; //--- norm score score = IsNaNOrInf(exp(score - LocalMax(score, 1, temp)), 0); score = IsNaNOrInf(score / LocalSum(score, 1, temp), 0); scores[shift_s] = score; BarrierLoc //--- out for(int d = 0; d < dimension; d++) { float val = LocalSum(kv[shift_kv + d].s1 * score, 1, temp); if(k == 0) out[dimension * (ind_q * heads + h) + d] = temp[0]; BarrierLoc } } //+------------------------------------------------------------------+ //| Computes the q index attention gradients gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void QIndexAttentionGradients(__global const float* q, __global float* q_g, __global const float2* kv, __global float2* kv_g, __global const float* indexes, __global const float* scores, __global const float* gradient, const int kunits, const int heads_kv ) { //--- init const int ind_q = get_global_id(0); const int d = get_global_id(1); const int h = get_global_id(2); const int qunits = get_global_size(0); const int dimension = get_global_size(1); const int heads = get_global_size(2); const int h_kv = h % heads_kv; const int q_id = (int)(indexes[ind_q * heads + h] + 0.001f); const int shift_q = dimension * (q_id * heads + h) + d; const int shift_s = (ind_q * heads + h) * kunits; const int shift_g = h * dimension + d; //--- Calculating Value's gradients int step_score = kunits * heads; if(h < heads_kv) { //--- for(int v = ind_q; v < kunits; v += qunits) { float grad = 0; for(int hq = h; hq < heads; hq += heads_kv) { int shift_score = hq * kunits + v; for(int g = 0; g < qunits; g++) grad += IsNaNOrInf(gradient[shift_g + dimension * (hq - h + g * heads)], 0) * scores[shift_score + g * step_score]; } int shift_v = dimension * (heads_kv * v + h) + d; kv_g[shift_v].s1 = IsNaNOrInf(grad, 0); } } //--- Calculating Query's gradients float grad = 0; float out_g = IsNaNOrInf(gradient[shift_g + ind_q * dimension], 0); int shift_kv = h_kv * dimension + d; //--- for(int k = 0; (k < kunits && out_g != 0); k++) { float sc_g = 0; float sc = scores[shift_s + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_s + v] * out_g * kv[shift_kv + v * heads_kv * dimension].s1 * ((float)(k == v) - sc); grad += sc_g * kv[shift_kv + k * heads_kv * dimension].s0; } q_g[shift_q] = grad; //--- Calculating Key's gradients if(h < heads_kv) { //--- for(int k = ind_q; k < kunits; k += qunits) { int shift_k = dimension * (heads_kv * k + h_kv) + d; grad = 0; for(int hq = h; hq < heads; hq++) { int shift_score = hq * kunits + k; float val = kv[shift_k + heads_kv * dimension].s1; for(int scr = 0; scr < qunits; scr++) { float sc_g = 0; int shift_sc = scr * kunits * heads; float sc = scores[shift_sc + k]; if(sc == 0) continue; for(int v = 0; v < kunits; v++) sc_g += scores[shift_sc + v] * gradient[shift_g + scr * dimension] * val * ((float)(k == v) - sc); grad += IsNaNOrInf(sc_g * q[(hq + (int)(indexes[scr * heads + hq] + 0.001f) * heads) * dimension + d], 0); } } kv_g[shift_k].s0 = IsNaNOrInf(grad, 0); } } } //+------------------------------------------------------------------+ //| Implements the ts positon encoder kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void TSPositonEncoder(__global const float2* __attribute__((aligned(8))) data, __global const float* time, __global float2* __attribute__((aligned(8))) output, __global const float* period ) { const int id = get_global_id(0); const int freq = get_global_id(1); const int p = get_global_id(2); const int total = get_global_size(0); const int freqs = get_global_size(1); const int periods = get_global_size(2); //--- const int shift = id * freqs + freq; const float2 d = data[shift * periods + p]; const float t = time[id] / period[p]; float val = M_PI_F * t * pow(2.0f, freq + 1); //--- output[shift * periods + p] = (float2)(d.s0 + sin(val), d.s1 + cos(val)); } //+------------------------------------------------------------------+ //| Runs the feed forward mult win conv w pad feed-forward pass on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardMultWinConvWPad(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_o, __global const int *windows_in, const int inputs, const int step, const int window_out, const int activation ) { const size_t id = get_global_id(0); const size_t id_w = get_global_id(1); const size_t v = get_global_id(2); const size_t outputs = get_global_size(0); const size_t windows_total = get_global_size(1); //--- int window_in = windows_in[id_w]; int mid_win = window_in / 2; int shift_in = id * step - mid_win; int shift_in_var = v * inputs; int shift_weight = 0; //--- for(int w = 0; w < id_w; w++) shift_weight += (windows_in[w] + 1) * window_out; //--- for(int w_out = 0; w_out < window_out; w_out++) { float sum = matrix_w[shift_weight + window_in]; //--- for(int w = 0; w < window_in; w++) if((shift_in + w) >= 0 && (shift_in + w) < inputs) sum += IsNaNOrInf(matrix_i[shift_in_var + shift_in + w] * matrix_w[shift_weight + w], 0); //--- int shift_out = (v * outputs + id) * window_out + w_out; matrix_o[shift_out] = fActivation(sum, activation); shift_weight += window_in + 1; } } //+------------------------------------------------------------------+ //| Computes the calc hidden gradient mult win conv w pad gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientMultWinConvWPad(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_ig, __global const float *matrix_og, __global const int *windows_in, const int outputs, const int step, const int window_out, const int filters, const int activation ) { const size_t id_x = get_global_id(0); const size_t loc = get_local_id(1); const size_t v = get_global_id(2); const size_t inputs = get_global_size(0); const size_t size_loc = get_local_size(1); const size_t windows_total = filters / window_out; //--- __local float temp[LOCAL_ARRAY_SIZE]; const uint ls = min((uint)size_loc, (uint)LOCAL_ARRAY_SIZE); //--- float grad = 0; for(int id_loc = loc; id_loc < filters; id_loc += size_loc) { const size_t id_win = id_loc / window_out; const size_t id_f = id_loc % window_out; int window_in = windows_in[id_win]; int shift_weight = id_f * (window_in + 1); for(int w = 0; w < id_win; w++) shift_weight += (windows_in[w] + 1) * window_out; //--- int shift_out = max((int)((id_x - window_in) / step), 0); //--- int mid_win = (window_in + 1) / 2; for(int out = shift_out; out < outputs; out++) { int shift_in = out * step - mid_win; if(shift_in > id_x) break; int shift_w = id_x - shift_in; if(shift_w >= window_in) continue; int shift_g = ((v * outputs + out) * windows_total + id_win) * window_out + id_f; grad += IsNaNOrInf(matrix_w[shift_w + shift_weight] * matrix_og[shift_g], 0); } } //--- grad = LocalSum(grad, 1, temp); //--- if(loc == 0) matrix_ig[v * inputs + id_x] = Deactivation(grad, matrix_i[v * inputs + id_x], activation); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights mult win conv| //| adam w pad step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsMultWinConvAdamWPad(__global float *matrix_w, __global const float *matrix_og, __global const float *matrix_i, __global float *matrix_m, __global float *matrix_v, __global const int *windows_in, const int windows_total, const int window_out, const int inputs, const int step, const int outputs, const float l, const float b1, const float b2 ) { const size_t i = get_global_id(0); // weight shift const size_t v = get_local_id(1); // variable const size_t variables = get_local_size(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; const uint ls = min((uint)variables, (uint)LOCAL_ARRAY_SIZE); //--- int step_out = window_out * windows_total; //--- int shift_before = 0; int window = 0; int number_w = 0; for(int w = 0; w < windows_total; w++) { int win = windows_in[w]; if(shift_before <= i && (win + 1)*window_out > (i - shift_before)) { window = win; number_w = w; } else shift_before += (win + 1) * window_out; } //--- int shift_in = (i - shift_before) % (window + 1); int shift_in_var = v * inputs; bool bias = (shift_in == window); int mid_win = (window + 1) / 2; int id_f = (i - shift_before) / (window + 1); int shift_out = number_w * window_out + id_f; int shift_out_var = v * outputs * step_out; //--- float grad = 0; if(!bias) { for(int out = 0; out < outputs; out++) { int in = out * step - mid_win + shift_in; if(in >= inputs) break; if(in < 0) continue; //--- grad += IsNaNOrInf(matrix_og[shift_out_var + shift_out + out * step_out] * matrix_i[shift_in_var + in], 0); } } else { for(int out = 0; out < outputs; out++) grad += IsNaNOrInf(matrix_og[shift_out_var + shift_out + out * step_out], 0); } //--- sum for(int s = 0; s < (int)variables; s += ls) { if(v >= s && v < (s + ls)) temp[v % ls] = (s == 0 ? 0 : temp[v % ls]) + grad; BarrierLoc } //--- uint count = ls; do { count = (count + 1) / 2; if(v < count && (v + count) < ls) { temp[v] += temp[v + count]; temp[v + count] = 0; } BarrierLoc } while(count > 1); //--- if(v == 0) { grad = temp[0]; float mt = IsNaNOrInf(clamp(b1 * matrix_m[i] + (1 - b1) * grad, -1.0e5f, 1.0e5f), 0); float vt = IsNaNOrInf(clamp(b2 * matrix_v[i] + (1 - b2) * (grad * grad), 1.0e-6f, 1.0e6f), 1.0e-6f); float weight = matrix_w[i] + IsNaNOrInf(l * mt / sqrt(vt), 0); matrix_w[i] = weight; matrix_m[i] = mt; matrix_v[i] = vt; } } //+------------------------------------------------------------------+ //| Performs the concat diff data-layout operation on the GPU. | //+------------------------------------------------------------------+ __kernel void ConcatDiff(__global const float* data, __global float* output, const int step) { const size_t i = get_global_id(0); const size_t v = get_local_id(1); const size_t inputs = get_local_size(0); const size_t variables = get_local_size(1); //--- const int shift = i * variables; const float d = data[shift + v]; float diff = 0; if(step > 0 && (i + step) < inputs) diff = IsNaNOrInf(d - data[shift + step * variables + v], 0); //--- output[2 * shift + v] = d; output[2 * shift + v + variables] = diff; } //+------------------------------------------------------------------+ //| Runs the feed forward mask mult win conv feed-forward pass on the| //| GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardMaskMultWinConv(__global const float *matrix_w, __global const float *matrix_i, __global const float *masks, __global float *matrix_o, const int inputs, const int window_in, const int windows_total, const int activation ) { const size_t u = get_global_id(0); const size_t w = get_global_id(1); const size_t v = get_global_id(2); const size_t units = get_global_size(0); const size_t window_out = get_global_size(1); const size_t variables = get_global_size(2); //--- const int shift_in = u * window_in * windows_total; const int shift_in_var = v * units * window_in * windows_total; const int shift_out = (u + v * units) * window_out + w; const int shift_mask = (u + v * units) * windows_total; const int shift_weight = (v * window_out * windows_total + w) * (window_in + 1); const int step_weight = window_out * (window_in + 1); //--- float sum = 0; for(int w_in = 0; w_in < windows_total; w_in++) { float m = IsNaNOrInf(masks[shift_mask + w_in], 0); if(m < FLT_EPSILON) continue; const int shift_in_loc = shift_in + w_in * window_in; const int shift_weight_loc = shift_weight + w_in * step_weight; for(int i = 0; i < window_in; i++) if((shift_in_loc + i) < (inputs / variables)) sum += IsNaNOrInf(matrix_i[shift_in_var + shift_in_loc + i], 0) * matrix_w[shift_weight_loc + i] * m; sum += matrix_w[shift_weight_loc + window_in] * m; } //--- matrix_o[shift_out] = fActivation(sum, activation); } //+------------------------------------------------------------------+ //| Computes the calc hidden gradient mask mult win conv gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientMaskMultWinConv(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_ig, __global const float *matrix_og, __global const float *masks, __global float *masks_g, const int outputs, const int window_in, const int window_out, const int activation ) { const size_t u = get_global_id(0); const size_t w_in = get_global_id(1); const size_t v = get_global_id(2); const size_t units = get_global_size(0); const size_t windows_total = get_global_size(1); const size_t variables = get_global_size(2); //--- const int shift_in = (u + v * units) * window_in * windows_total + w_in * window_in; const int shift_out = u * window_out; const int shift_out_var = v * units * window_out; const int shift_mask = (u + v * units) * windows_total + w_in; const int shift_weight = (v * window_out * windows_total + w_in * window_out) * (window_in + 1); //--- const float m = IsNaNOrInf(masks[shift_mask], 0); for(int i = 0; i < window_in; i++) { float sum = 0; if(m >= FLT_EPSILON) { for(int out = 0; out < window_out; out++) { if((shift_out + out) >= (outputs / variables)) continue; sum += IsNaNOrInf(matrix_og[shift_out_var + shift_out + out] * matrix_w[shift_weight + out * (window_in + 1) + i] * m, 0); } } matrix_ig[shift_in + i] = Deactivation(sum, matrix_i[shift_in + i], activation); } //--- float sum = 0; for(int out = 0; out < window_out; out++) { int shift_weight_loc = out * (window_in + 1) + shift_weight; float temp = matrix_w[shift_weight_loc + window_in]; for(int i = 0; i < window_in; i++) temp += IsNaNOrInf(matrix_i[shift_in + i], 0) * matrix_w[shift_weight_loc + i]; sum += IsNaNOrInf(temp * matrix_og[shift_out_var + shift_out + out], 0); } masks_g[shift_mask] = IsNaNOrInf(sum, 0); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights mask mult win| //| conv adam step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsMaskMultWinConvAdam(__global float *matrix_w, __global const float *matrix_og, __global const float *matrix_i, __global const float *masks, __global float *matrix_m, __global float *matrix_v, const int windows_total, const int inputs, const int outputs, const float l, const float b1, const float b2 ) { const size_t id_in = get_global_id(0); // input shift const size_t id_out = get_global_id(1); // filter shift const size_t id_v = get_global_id(2); // variable const size_t window_in = get_global_size(0) / windows_total - 1; const size_t window_out = get_global_size(1); const size_t variables = get_global_size(2); //--- const int w_id = id_in / (window_in + 1); const int shift_in = id_in - w_id; const int step_in = window_in * windows_total; const int units = outputs / window_out; const int shift_in_var = id_v * inputs; const int shift_out_var = id_v * outputs; const int shift_mask_var = id_v * units * windows_total; const int shift_weight = ((id_v * windows_total + w_id) * window_out + id_out) * (window_in + 1) + id_in % (window_in + 1); const bool bias = (id_in % (window_in + 1) == window_in); //--- float grad = 0; for(int u = 0; u < units; u++) { const int shift_in_loc = shift_in + u * step_in; if(shift_in < inputs) continue; float m = IsNaNOrInf(masks[shift_mask_var + u * windows_total + w_id], 0); if(m < FLT_EPSILON) continue; float inp = (bias ? 1 : IsNaNOrInf(matrix_i[shift_in_var + shift_in_loc], 0)); grad += IsNaNOrInf(inp * m * matrix_og[shift_out_var + u * window_out + id_out], 0); } float mt = IsNaNOrInf(clamp(b1 * matrix_m[shift_weight] + (1 - b1) * grad, -1.0e5f, 1.0e5f), 0); float vt = IsNaNOrInf(clamp(b2 * matrix_v[shift_weight] + (1 - b2) * (grad * grad), 1.0e-6f, 1.0e6f), 1.0e-6f); float weight = matrix_w[shift_weight] + IsNaNOrInf(l * mt / sqrt(vt), 0); matrix_w[shift_weight] = weight; matrix_m[shift_weight] = mt; matrix_v[shift_weight] = vt; } //+------------------------------------------------------------------+ //| Implements the main freq kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MainFreq(__global const float* freq_r, __global const float* freq_im, __global float *main_freq, int dimension ) { if(dimension <= 0) return; //--- size_t n = get_global_id(0); const int shift = n * dimension; //--- float max_f = 0; float max_id = 0; float energy; //--- for(int i = 1; i < dimension; i++) { float2 freq = (float2)(freq_r[shift + i], freq_im[shift + i]); energy = ComplexAbs(freq); if(max_f < energy) { max_f = energy; max_id = i + 1; } } main_freq[n] = max_id; } //+------------------------------------------------------------------+ //| Runs the feed forward adapt conv feed-forward pass on the GPU. | //+------------------------------------------------------------------+ __kernel void FeedForwardAdaptConv(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_o, __global const float *main_freq, const int inputs, const int window_in, const int activation ) { const size_t u = get_global_id(0); const size_t f = get_global_id(1); const size_t v = get_global_id(2); const size_t units = get_global_size(0); const size_t filters = get_global_size(1); const size_t variables = get_global_size(2); //--- const int freq = main_freq[v]; int window = (inputs / variables + freq - 1) / freq; const int step = (int)(inputs / variables + units + 1) / (units + 2); if(window < step) window = (int)((step + window - 1) / window) * window; if(window > window_in) window = window_in; //--- const int shift_in = (u < (units - 1) ? u * step : inputs / variables - window); const int shift_in_var = v * inputs / variables; const int shift_out = (u + v * units) * filters + f; const int shift_weight = (v * filters + f) * (window_in + 1); //--- float sum = matrix_w[shift_weight + window_in]; for(int i = 0; i < window; i++) if((shift_in + i) < (inputs / variables)) sum += IsNaNOrInf(matrix_i[shift_in_var + shift_in + i], 0) * matrix_w[shift_weight + i]; //--- matrix_o[shift_out] = fActivation(sum, activation); } //+------------------------------------------------------------------+ //| Computes the calc hidden gradient adapt conv gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradientAdaptConv(__global const float *matrix_w, __global const float *matrix_i, __global float *matrix_ig, __global const float *matrix_og, __global const float *main_freq, const int outputs, const int window_in, const int window_out, const int activation ) { const size_t inp = get_global_id(0); const size_t v = get_global_id(1); const size_t inputs = get_global_size(0); const size_t variables = get_global_size(1); //--- const int units = outputs / (window_out * variables); const int freq = main_freq[v]; int window = (inputs / variables + freq - 1) / freq; const int step = (int)(inputs + units + 1) / (units + 2); if(window < step) window = (int)((step + window - 1) / window) * window; if(window > window_in) window = window_in; //--- const int shift_in = v * inputs + inp; int u = inp / step; int shift_out_var = v * (outputs / variables); int shift_weight_var = (v * window_out) * (window_in + 1); //--- float sum = 0; while(u * step <= inp && u < (units - 1)) { int pos = inp - u * step; if(pos >= window) { u++; continue; } int shift_out = u * window_out; int shift_weight = pos + shift_weight_var; for(int out = 0; out < window_out; out++) { if((shift_out + out) >= (outputs / variables)) continue; sum += IsNaNOrInf(matrix_og[shift_out_var + shift_out + out] * matrix_w[shift_weight + out * (window_in + 1)], 0); } u++; } if(inp >= (inputs - window)) { int pos = inp + window - inputs; int shift_out = (units - 1) * window_out; int shift_weight = pos + shift_weight_var; for(int out = 0; out < window_out; out++) { if((shift_out + out) >= (outputs / variables)) continue; sum += IsNaNOrInf(matrix_og[shift_out_var + shift_out + out] * matrix_w[shift_weight + out * (window_in + 1)], 0); } } matrix_ig[shift_in] = Deactivation(sum, matrix_i[shift_in], activation); } //+------------------------------------------------------------------+ //| Updates the trainable weights of the update weights adapt conv | //| adam step using the optimizer rule. | //+------------------------------------------------------------------+ __kernel void UpdateWeightsAdaptConvAdam(__global float *matrix_w, __global const float *matrix_og, __global const float *matrix_i, __global float *matrix_m, __global float *matrix_v, __global float *main_freq, const int inputs, const int outputs, const float l, const float b1, const float b2 ) { const size_t id_in = get_global_id(0); // input shift const size_t id_out = get_global_id(1); // filter shift const size_t id_v = get_global_id(2); // variable const size_t window_in = get_global_size(0) - 1; const size_t window_out = get_global_size(1); const size_t variables = get_global_size(2); //--- const int units = outputs / (window_out * variables); const int freq = main_freq[id_v]; int window = (inputs / variables + freq - 1) / freq; const int step = (int)(inputs / variables + units + 1) / (units + 2); if(window < step) window = (int)((step + window - 1) / window) * window; if(window > window_in) window = window_in; //--- if(id_in != window_in && id_in >= window) return; //--- const int shift_in_var = id_v * inputs / variables; const int shift_out_var = id_v * outputs / variables; const int shift_weight = (id_v * window_out + id_out) * (window_in + 1) + id_in; const bool bias = (id_in == window_in); //--- float grad = 0; for(int u = 0; u < (units - 1); u++) { const int shift_in_loc = id_in + u * step; if(shift_in_loc >= (inputs / variables)) continue; float inp = (bias ? 1 : IsNaNOrInf(matrix_i[shift_in_var + shift_in_loc], 0)); grad += IsNaNOrInf(inp * matrix_og[shift_out_var + u * window_out + id_out], 0); } { const int shift_in_loc = id_in + inputs / variables - window; if(shift_in_loc < (inputs / variables)) { float inp = (bias ? 1 : IsNaNOrInf(matrix_i[shift_in_var + shift_in_loc], 0)); grad += IsNaNOrInf(inp * matrix_og[shift_out_var + (units - 1) * window_out + id_out], 0); } } float mt = IsNaNOrInf(clamp(b1 * matrix_m[shift_weight] + (1 - b1) * grad, -1.0e5f, 1.0e5f), 0); float vt = IsNaNOrInf(clamp(b2 * matrix_v[shift_weight] + (1 - b2) * (grad * grad), 1.0e-6f, 1.0e6f), 1.0e-6f); float weight = matrix_w[shift_weight] + IsNaNOrInf(l * mt / sqrt(vt), 0); matrix_w[shift_weight] = weight; matrix_m[shift_weight] = mt; matrix_v[shift_weight] = vt; } //+------------------------------------------------------------------+ //| Implements the ro pe kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RoPE(__global const float2* __attribute__((aligned(8))) inputs, __global const float2* __attribute__((aligned(8))) position_emb, __global float2* __attribute__((aligned(8))) outputs ) { const size_t id_d = get_global_id(0); // dimension const size_t id_u = get_global_id(1); // unit const size_t id_v = get_global_id(2); // variable const size_t dimension = get_global_size(0); const size_t units = get_global_size(1); const size_t variables = get_global_size(2); //--- const int shift_in = (id_v * units + id_u) * dimension + id_d; const int shift_pos = id_u * dimension + id_d; const float2 inp = inputs[shift_in]; const float2 pe = position_emb[shift_pos]; //--- float2 result = 0; result.s0 = inp.s0 * pe.s0 - inp.s1 * pe.s1; result.s1 = inp.s0 * pe.s1 + inp.s1 * pe.s0; //--- outputs[shift_in] = result; } //+------------------------------------------------------------------+ //| Calculates calc hidden grad ro pe on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcHiddenGradRoPE(__global float2* __attribute__((aligned(8))) inputs_gr, __global const float2* __attribute__((aligned(8))) position_emb, __global const float2* __attribute__((aligned(8))) outputs_gr ) { const size_t id_d = get_global_id(0); // dimension const size_t id_u = get_global_id(1); // unit const size_t id_v = get_global_id(2); // variable const size_t dimension = get_global_size(0); const size_t units = get_global_size(1); const size_t variables = get_global_size(2); //--- const int shift_in = (id_v * units + id_u) * dimension + id_d; const int shift_pos = id_u * dimension + id_d; const float2 grad = outputs_gr[shift_in]; const float2 pe = position_emb[shift_pos]; //--- float2 grad_x; grad_x.s0 = grad.s0 * pe.s0 + grad.s1 * pe.s1; grad_x.s1 = grad.s1 * pe.s0 - grad.s0 * pe.s1; //--- inputs_gr[shift_in] = grad_x; } //+------------------------------------------------------------------+ //| Implements the dif matrix kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DifMatrix(__global const float *matrix1, ///<[in] First matrix __global const float *matrix2, ///<[in] Second matrix __global float *matrix_out, ///<[out] Output matrix const float multiplyer, ///< Multiplyer for output const int shift_in1, ///< Shift for input 1 const int shift_in2, ///< Shift for input 2 const int shift_out ///< Shift for output ) { const int i = get_global_id(0); const int d = get_global_id(1); const int step = get_global_size(0); const int dimension = get_global_size(1); //--- int index = i * dimension + d; matrix_out[i * shift_out + index] = IsNaNOrInf((matrix1[i * shift_in1 + index] - matrix2[i * shift_in2 + index]) * multiplyer, 0); } //+------------------------------------------------------------------+ //| Computes the dif matrix grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void DifMatrixGrad(__global float *matrix1, ///<[in] First matrix __global float *matrix2, ///<[in] Second matrix __global const float *matrix_out, ///<[out] Output matrix const float multiplyer, ///< Multiplyer for output const int shift_in1, ///< Shift for input 1 const int shift_in2, ///< Shift for input 2 const int shift_out ///< Shift for output ) { const int i = get_global_id(0); const int d = get_global_id(1); const int step = get_global_size(0); const int dimension = get_global_size(1); //--- int index = i * dimension + d; float grad = IsNaNOrInf(matrix_out[i * shift_out + index] * multiplyer, 0); matrix1[i * shift_in1 + index] = grad; matrix2[i * shift_in2 + index] = -grad; } //+------------------------------------------------------------------+ //| Implements the identity sum matrix kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void IdentitySumMatrix(__global const float *matrix_in, __global float *matrix_out, const float multiplyer, const int shift_in, const int shift_out ) { const int i = get_global_id(0); const int d = get_global_id(1); const int step = get_global_size(0); const int dimension = get_global_size(1); //--- int index = i * dimension + d; matrix_out[i * shift_out + index] = IsNaNOrInf(((int)(i == d) + matrix_in[i * shift_in + index]) * multiplyer, 0); } //+------------------------------------------------------------------+ //| Implements the identity dif matrix kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void IdentityDifMatrix(__global const float *matrix_in, __global float *matrix_out, const float multiplyer, const int shift_in, const int shift_out ) { const int i = get_global_id(0); const int d = get_global_id(1); const int step = get_global_size(0); const int dimension = get_global_size(1); //--- int index = i * dimension + d; matrix_out[i * shift_out + index] = IsNaNOrInf(((int)(i == d) - matrix_in[i * shift_in + index]) * multiplyer, 0); } //+------------------------------------------------------------------+ //| Computes the identity dif matrix grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void IdentityDifMatrixGrad(__global float *matrix_in, __global const float *matrix_out, const float multiplyer, const int shift_in, const int shift_out ) { const int i = get_global_id(0); const int d = get_global_id(1); const int step = get_global_size(0); const int dimension = get_global_size(1); //--- int index = i * dimension + d; matrix_in[i * shift_in + index] = IsNaNOrInf(-multiplyer * matrix_out[i * shift_out + index], 0); } //+------------------------------------------------------------------+ //| Implements the sum vec matrix kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SumVecMatrix(__global const float *vector_in, __global const float *matrix_in, __global float *matrix_out, const float multiplyer, ///< Multiplyer for output const int shift_in1, ///< Shift for input 1 const int shift_in2, ///< Shift for input 2 const int shift_out, ///< Shift for output __global float *numeric_flags, const int flags_enabled ) { const int r = get_global_id(0); const int c = get_global_id(1); const int v = get_global_id(2); const int rows = get_global_size(0); const int cols = get_global_size(1); const int variables = get_global_size(2); //--- int flat_in = shift_in2 + RCtoFlat(r, c, rows, cols, v); int flat_out = shift_out + RCtoFlat(r, c, rows, cols, v); int flat_v = shift_in1 + RCtoFlat(0, c, 1, cols, v); if(flags_enabled) { const float vector_value = vector_in[flat_v]; const float matrix_value = matrix_in[flat_in]; const float raw_sum = vector_value + matrix_value; const float raw_result = raw_sum * multiplyer; const int invalid = isnan(vector_value) || isinf(vector_value) || isnan(matrix_value) || isinf(matrix_value) || isnan(raw_sum) || isinf(raw_sum) || isnan(raw_result) || isinf(raw_result); numeric_flags[RCtoFlat(r, c, rows, cols, v)] = (invalid ? 1.0f : 0.0f); } matrix_out[flat_out] = IsNaNOrInf((vector_in[flat_v] + matrix_in[flat_in]) * multiplyer, 0); } //+------------------------------------------------------------------+ //| Computes the sum vec matrix grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SumVecMatrixGrad(__global float *vector_in, __global float *matrix_in, __global const float *matrix_out, const float multiplyer, ///< Multiplyer for output const int shift_in1, ///< Shift for input 1 const int shift_in2, ///< Shift for input 2 const int shift_out ///< Shift for output ) { const int r = get_global_id(0); const int c = get_global_id(1); const int v = get_global_id(2); const int rows = get_global_size(0); const int cols = get_global_size(1); const int variables = get_global_size(2); //--- int flat_in = shift_in2 + RCtoFlat(r, c, rows, cols, v); int flat_out = shift_out + RCtoFlat(r, c, rows, cols, v); int flat_v = shift_in1 + RCtoFlat(0, c, 1, cols, v); //--- float grad = IsNaNOrInf(matrix_out[flat_out] * multiplyer, 0); matrix_in[flat_in] = grad; //--- if(r == 0) { for(int i = 1; i < rows; i++) { flat_out += cols; grad += IsNaNOrInf(matrix_out[flat_out] * multiplyer, 0); } vector_in[flat_v] = IsNaNOrInf(grad / rows, 0); } } //+------------------------------------------------------------------+ //| Implements the interpolation attention kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void InterpolationAttention(__global const float* matrix_in, __global const float* W, __global const float* A, __global const float* GL, __global float* Adj, __global float* H, __global float* Atten, __global float* matrix_out, const int dimension ) { const size_t i = get_global_id(0); const size_t j = get_local_id(1); const size_t total = get_global_size(0); const size_t total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift_i = i * dimension; const int shift_j = j * dimension; const int shift_adj = i * total_loc + j; //--- float adj = 0; for(int d = 0; d < dimension; d++) adj += IsNaNOrInf(GL[shift_i + d] * GL[shift_j + d], 0); adj = max(IsNaNOrInf(adj, 0), 0.0f); adj = LocalSoftMax(adj, 1, Temp); Adj[shift_adj] = adj; adj += (float)(i == j); //--- for(int id_h = 0; id_h < dimension; id_h += total_loc) { if(j >= (dimension - id_h)) break; float h = 0; for(int w = 0; w < dimension; w++) h += IsNaNOrInf(matrix_in[shift_i + w] * W[(id_h + j) * dimension + w], 0); H[shift_i + id_h + j] = h; BarrierLoc } float e = 1e-12f; if(adj > 0) { e = 0; for(int a = 0; a < dimension; a++) e += IsNaNOrInf(H[shift_i + a] * A[a], 0) + IsNaNOrInf(H[shift_j + a] * A[dimension + a], 0); } e = LocalSoftMax(e, 1, Temp); Atten[shift_adj] = e; //--- Scale output by attention for(int d = 0; d < dimension; d += total_loc) { if(j >= (dimension - d)) break; float out = 0; int shift_h = d + j; int shift_att = i * total_loc; int shift_out = i * dimension + shift_h; for(int n = 0; n < total_loc; n++) out += IsNaNOrInf(H[shift_h + n * dimension] * Atten[shift_att + n], 0); matrix_out[shift_out] = fActivation(out, ActFunc_LReLU); } } //+------------------------------------------------------------------+ //| Computes the interpolation attention grad gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void InterpolationAttentionGrad(__global const float* matrix_in, __global float* matrix_in_gr, __global const float* W, __global float* W_gr, __global const float* A, __global float* A_gr, __global const float* GL, __global float* GL_gr, __global float* Adj, __global float* H, __global float* H_gr, __global float* Atten, __global float* matrix_out_gr, const int dimension ) { const size_t i = get_global_id(0); const size_t j = get_local_id(1); const size_t total = get_global_size(0); const size_t total_loc = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift_i = i * dimension; const int shift_j = j * dimension; const int shift_adj = i * total_loc + j; //--- H Gradient for(int d = 0; d < dimension; d += total_loc) { if(j >= (dimension - d)) break; float h_grad = 0; int shift_h = shift_i + d + j; int shift_att = i; int shift_out = d + j; for(int n = 0; n < total_loc; n++) { float gr = matrix_out_gr[shift_out + n * dimension]; h_grad += IsNaNOrInf( Deactivation(gr, gr, ActFunc_LReLU) * Atten[shift_att + n * total_loc], 0); } H_gr[shift_h] = h_grad; BarrierLoc } //--- Attention Gradient float att_grad = 0; for(int d = 0; d < dimension; d++) { float gr = matrix_out_gr[shift_i + d]; gr = Deactivation(gr, gr, ActFunc_LReLU); att_grad += IsNaNOrInf(gr * H[shift_j + d], 0); } att_grad = LocalSoftMaxGrad(Atten[shift_adj], att_grad, 1, Temp); //--- Add H Gradient for(int d = 0; d < dimension; d++) { float h_grad = att_grad * A[d]; h_grad = LocalSum(h_grad, 1, Temp); if(j == 0) H_gr[shift_i + d] += h_grad; h_grad = att_grad * A[dimension + d]; h_grad = LocalSum(h_grad, 1, Temp); if(j == 0) H_gr[shift_j + d] += h_grad; float a_grad = att_grad * H[shift_i + d]; a_grad = LocalSum(a_grad, 1, Temp); A_gr[d] += a_grad; a_grad = att_grad * H[shift_j + d]; a_grad = LocalSum(a_grad, 1, Temp); A_gr[dimension + d] += a_grad; } //--- Inputs' Gradient for(int d = 0; d < dimension; d += total_loc) { if(j >= (dimension + d)) break; float grad = 0; for(int w = 0; w < dimension; w++) grad += IsNaNOrInf(H_gr[shift_i + w] * W[(d + j) + dimension * w], 0); matrix_in_gr[shift_i + d + j] = grad; BarrierLoc } //--- Adj Gradient float grad = LocalSoftMaxGrad(Adj[shift_adj], att_grad, 1, Temp); for(int d = 0; d < dimension; d++) { GL_gr[shift_i + d] += IsNaNOrInf(grad * GL[shift_j + d], 0); GL_gr[shift_j + d] += IsNaNOrInf(grad * GL[shift_i + d], 0); } } //+------------------------------------------------------------------+ //| Implements the period norm kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PeriodNorm(__global const float* inputs, __global float2* mean_stdevs, __global float* outputs, const int total_inputs ) { const size_t i = get_global_id(0); const size_t p = get_local_id(1); const size_t v = get_global_id(2); const size_t windows = get_global_size(0); const size_t period = get_local_size(1); const size_t variable = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift_i = i * period + p; const int shift_v = v * total_inputs; const int shift_ms = v * windows + i; //--- float val = 0; if((shift_i) < total_inputs) val = IsNaNOrInf(inputs[shift_v + shift_i], 0); float mean = IsNaNOrInf(LocalSum(val, 1, Temp) / period, 0); val -= mean; BarrierLoc float stdev = LocalSum(val * val, 1, Temp) / period; stdev = IsNaNOrInf(sqrt(stdev), 1); //--- mean_stdevs[shift_ms] = (float2)(mean, stdev); if((shift_i) < total_inputs) outputs[shift_v + shift_i] = IsNaNOrInf(val / stdev, 0); } //+------------------------------------------------------------------+ //| Computes the period norm grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void PeriodNormGrad(__global const float* inputs, __global float* inputs_gr, __global const float2* mean_stdevs, __global const float2* mean_stdevs_gr, __global const float* outputs, __global const float* outputs_gr, const int total_inputs ) { const size_t i = get_global_id(0); const size_t p = get_local_id(1); const size_t v = get_global_id(2); const size_t windows = get_global_size(0); const size_t period = get_local_size(1); const size_t variable = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift_i = i * period + p; const int shift_v = v * total_inputs; const int shift_ms = v * windows + i; //--- float inp = 0; float inp_gr = 0; float out = 0; float out_gr = 0; const float2 mean_stdev = mean_stdevs[shift_ms]; const float2 mean_stdev_gr = mean_stdevs_gr[shift_ms]; if((shift_i) < total_inputs) { inp = IsNaNOrInf(inputs[shift_v + shift_i], 0); out = IsNaNOrInf(outputs[shift_v + shift_i], 0); out_gr = IsNaNOrInf(outputs_gr[shift_v + shift_i], 0); } float mean_gr = LocalSum(out_gr, 1, Temp) / period + IsNaNOrInf(mean_stdev.x, 0); BarrierLoc float stdev_gr = out * LocalSum(IsNaNOrInf(out * out_gr, 0), 1, Temp) / period + IsNaNOrInf(mean_stdev.y, 0); inp_gr = (out_gr - mean_gr - stdev_gr) / IsNaNOrInf(mean_stdev.y, 1); //--- if((shift_i) < total_inputs) inputs_gr[shift_v + shift_i] = IsNaNOrInf(inp_gr, 0); } //+------------------------------------------------------------------+ //| Implements the adapt spatial norm kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void AdaptSpatialNorm(__global const float* inputs, __global const float* attention, __global float2* mean_stdevs, __global float* outputs ) { const size_t i = get_global_id(0); const size_t a = get_local_id(1); const size_t v = get_global_id(2); const size_t total_inputs = get_global_size(0); const size_t total_local = get_local_size(1); const size_t variables = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift_v = v * total_inputs; const int shift_out = shift_v + i; //--- float mean = 0, stdev = 0; for(uint l = 0; l < variables; l += total_local) { const int shift_at = v * variables + (a + l); float val = IsNaNOrInf(inputs[(a + l) * total_inputs + i], 0); float att = IsNaNOrInf(attention[shift_at], 0); mean += val * att; stdev += val * val * att; } mean = LocalSum(mean, 1, Temp); BarrierLoc stdev = LocalSum(stdev, 1, Temp); //--- if(a == 0) { stdev -= mean * mean; stdev = IsNaNOrInf(sqrt(stdev), 1); if(stdev <= 0) stdev = 1; mean_stdevs[shift_out] = (float2)(mean, stdev); outputs[shift_out] = IsNaNOrInf((inputs[shift_out] - mean) / stdev, 0); } } //+------------------------------------------------------------------+ //| Computes the adapt spatial norm grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void AdaptSpatialNormGrad(__global const float* inputs, __global float* inputs_gr, __global const float* attention, __global float* attention_gr, __global const float2* mean_stdevs, __global const float2* mean_stdevs_gr, __global const float* outputs_gr, const uint total_inputs ) { const size_t i = get_global_id(0); // main const size_t loc = get_local_id(1); // local to sum const size_t v = get_global_id(2); // variable const size_t total_main = get_global_size(0); // total const size_t total_loc = get_local_size(1); // local dimension const size_t variables = get_global_size(2); // total variables //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- Inputs gradient { int shift_in = v * total_inputs + i; float grad = 0; if(i < total_inputs) { float x = IsNaNOrInf(inputs[shift_in], 0); for(int l = 0; l < variables; l += total_loc) { if((l + loc) >= variables) break; int shift_out = i + (l + loc) * total_inputs; float att = IsNaNOrInf(attention[(l + loc) * variables + v], 0); float out_gr = IsNaNOrInf(outputs_gr[shift_out], 0); float2 ms = mean_stdevs[shift_out]; float2 ms_gr = mean_stdevs_gr[shift_out]; float dy = (1 - att) * (1 / ms.y - (x - ms.x) * att * x / (ms.y * ms.y * ms.y)); float dmean = IsNaNOrInf(ms_gr.x * att, 0); float dstd = IsNaNOrInf(ms_gr.y * x * (att - att * att) / ms.y, 0); grad += IsNaNOrInf(dy * out_gr + dmean + dstd, 0); } } grad = LocalSum(grad, 1, Temp); if(loc == 0 && i < total_inputs) inputs_gr[shift_in] = grad; BarrierLoc } //--- Attention gradient { int shift_att = v * variables + i; float grad = 0; if(i < variables) { float att = IsNaNOrInf(attention[shift_att], 0); for(int l = 0; l < total_inputs; l += total_loc) { if((l + loc) >= total_inputs) break; int shift_out = (l + loc) + v * total_inputs; int shift_in = (l + loc) + i * total_inputs; float x = IsNaNOrInf(inputs[shift_in], 0); float out_gr = IsNaNOrInf(outputs_gr[shift_out], 0); float2 ms = mean_stdevs[shift_out]; float2 ms_gr = mean_stdevs_gr[shift_out]; float dy = -x / ms.y - (x - ms.x) * x * x * (1 - 2 * att) / (2 * ms.y * ms.y * ms.y); float dmean = IsNaNOrInf(ms_gr.x * x, 0); float dstd = IsNaNOrInf(ms_gr.y * x * x * (1 - 2 * att) / (2 * ms.y), 0); grad += IsNaNOrInf(dy * out_gr + dmean + dstd, 0); } } grad = LocalSum(grad, 1, Temp); if(loc == 0 && i < variables) attention_gr[shift_att] = grad; } } //+------------------------------------------------------------------+ //| Implements the attent norm kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void AttentNorm(__global const float* inputs, __global const float* attention, __global float* means, __global float* stdevs, __global float* outputs, const int total_inputs, const int segment_size ) { const size_t s = get_global_id(0); const size_t i = get_local_id(1); const size_t v = get_global_id(2); const size_t total_segments = get_global_size(0); const size_t total_local = get_local_size(1); const size_t variables = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int shift = v * total_inputs + s * segment_size + i; //--- float mean = 0, stdev = 0; float val = 0; for(uint l = 0; l < segment_size; l += total_local) { if((l + i) >= segment_size || (s * segment_size + l + i) >= total_inputs) break; float val_l = IsNaNOrInf(inputs[shift + l], 0); if(l == 0) val = val_l; float att = IsNaNOrInf(attention[v * segment_size + l + i], 0); mean += val_l * att; stdev += val_l * val_l * att; } mean = LocalSum(mean, 1, Temp); BarrierLoc stdev = LocalSum(stdev, 1, Temp); //--- stdev -= mean * mean; stdev = IsNaNOrInf(sqrt(stdev), 1); if(stdev <= 0) stdev = 1; //--- if(i == 0) { int shift_ms = v * total_segments + s; means[shift_ms] = mean; stdevs[shift_ms] = stdev; } for(uint l = 0; l < segment_size; l += total_local) { if((l + i) >= segment_size || (s * segment_size + l + i) >= total_inputs) break; if(l > 0) val = inputs[shift + l]; outputs[shift + l] = IsNaNOrInf((val - mean) / stdev, 0); } } //+------------------------------------------------------------------+ //| Computes the attent norm grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void AttentNormGrad(__global const float* inputs, __global float* inputs_gr, __global const float* attention, __global float* attention_gr, __global const float* means, __global const float* stdevs, __global const float* means_gr, __global const float* outputs_gr, const int total_inputs, const int segment_size ) { const size_t i = get_global_id(0); // main const size_t loc = get_local_id(1); // local to sum const size_t v = get_global_id(2); // variable const size_t total_main = get_global_size(0); // total const size_t total_loc = get_local_size(1); // local dimension const size_t variables = get_global_size(2); // total variables //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- Inputs gradient { const int s = i / segment_size; const int shift_in = v * total_inputs + i; const int shift_ms = v * segment_size + s; float grad = 0; if(loc == 0 && i < total_inputs) { Temp[0] = IsNaNOrInf(inputs[shift_in], 0); Temp[1] = IsNaNOrInf(means[shift_ms], 0); Temp[2] = IsNaNOrInf(stdevs[shift_ms], 1); Temp[3] = IsNaNOrInf(means_gr[shift_ms], 0); Temp[4] = IsNaNOrInf(attention[(v - s) * segment_size + i], 0); } BarrierLoc if(i < total_inputs) { float x = Temp[0]; float mean = Temp[1]; float stdev = Temp[2]; float mean_gr = Temp[3]; float att = Temp[4]; for(int l = 0; l < segment_size; l += total_loc) { if((l + loc) >= segment_size || (i * segment_size + loc + l) >= total_inputs) break; float out_gr = IsNaNOrInf(outputs_gr[v * total_inputs + s * segment_size + loc + l], 0); bool same = (i - s * segment_size) == (loc + l); float xl = x; if(!same) xl = IsNaNOrInf(inputs[v * total_inputs + s * segment_size + loc + l], 0); float dy = ((int)same - att) * (1 / stdev - (xl - mean) * att * x / (stdev * stdev * stdev)); float dmean = (same ? IsNaNOrInf(mean_gr * att, 0) : 0); grad += IsNaNOrInf(dy * out_gr + dmean, 0); } } grad = LocalSum(grad, 1, Temp); if(loc == 0 && i < total_inputs) inputs_gr[shift_in] = grad; BarrierLoc } //--- Attention gradient { float grad = 0; int shift_att = v * segment_size + i; if(i < segment_size) { float att = IsNaNOrInf(attention[shift_att], 0); for(int l = 0; l < total_inputs; l += total_loc) { if((l + loc) >= total_inputs) break; int shift_out = (l + loc) + v * total_inputs; int s = (l + loc) / segment_size; int shift_in = v * total_inputs + s * segment_size + i; float x = IsNaNOrInf(inputs[shift_in], 0); float out_gr = IsNaNOrInf(outputs_gr[shift_out], 0); float mean = means[v * segment_size + s]; float stdev = stdevs[v * segment_size + s]; float mean_gr = means_gr[v * segment_size + s]; bool same = (i - s * segment_size) == (loc + l); float xl = x; if(!same) xl = IsNaNOrInf(inputs[shift_out], 0); float dy = -x / stdev - (xl - mean) * x * x * (1 - 2 * att) / (2 * stdev * stdev * stdev); float dmean = IsNaNOrInf(mean_gr * x, 0); grad += IsNaNOrInf(dy * out_gr + dmean, 0); } } grad = LocalSum(grad, 1, Temp); if(loc == 0 && i < segment_size) attention_gr[shift_att] = grad; } } //+------------------------------------------------------------------+ //| Implements the cheb step kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ChebStep(__global const float* support, __global float* outputs, const int step ) { const size_t l = get_local_id(0); const size_t r = get_global_id(1); const size_t c = get_global_id(2); const size_t total_l = get_local_size(0); const size_t total_r = get_global_size(1); const size_t total_c = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- if(step <= 0 || total_r != total_c) return; //--- if(step <= 3) { const float diag = (r == c ? 1.0f : 0.0f); if(l == 0) outputs[RCtoFlat(r, c, total_r, total_c, 0)] = diag; if(step < 2) return; if(l == 0) { const float s = IsNaNOrInf(support[RCtoFlat(r, c, total_r, total_c, 0)], 0); outputs[RCtoFlat(r, c, total_r, total_c, 1)] = s; } if(step < 3) return; float out = 0; for(int t = 0; t < total_c; t += total_l) { const float s1 = IsNaNOrInf(support[RCtoFlat(r, t + l, total_r, total_c, 0)], 0); const float s2 = IsNaNOrInf(support[RCtoFlat(t + l, c, total_r, total_c, 0)], 0); out += IsNaNOrInf(s1 * s2, 0); } out = 2 * LocalSum(out, 0, Temp); if(l == 0) { out -= diag; outputs[RCtoFlat(r, c, total_r, total_c, 2)] = IsNaNOrInf(out, 0); } return; } //--- float out = 0; for(int t = 0; t < total_c; t += total_l) { if((t + l) >= total_c) continue; const float s1 = IsNaNOrInf(support[RCtoFlat(r, t + l, total_r, total_c, 0)], 0); const float s2 = IsNaNOrInf(outputs[RCtoFlat(t + l, c, total_r, total_c, step - 2)], 0); out += IsNaNOrInf(s1 * s2, 0); } out = 2 * LocalSum(out, 0, Temp); if(l == 0) { out -= IsNaNOrInf(outputs[RCtoFlat(r, c, total_r, total_c, step - 3)], 0); outputs[RCtoFlat(r, c, total_r, total_c, step - 1)] = IsNaNOrInf(out, 0); } return; } //+------------------------------------------------------------------+ //| Computes the cheb step grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void ChebStepGrad(__global const float* support, __global float* support_g, __global const float* outputs, __global float* outputs_g, const int step ) { const size_t l = get_local_id(0); const size_t r = get_global_id(1); const size_t c = get_global_id(2); const size_t total_l = get_local_size(0); const size_t total_r = get_global_size(1); const size_t total_c = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- if(step < 1 || total_r != total_c) return; //--- if(step >= 2) { float grad = IsNaNOrInf(outputs_g[RCtoFlat(r, c, total_r, total_c, step)], 0); if(l == 0) outputs_g[RCtoFlat(r, c, total_r, total_c, step - 2)] -= grad; //--- support grad grad = 0; for(int t = 0; t < total_c; t += total_l) { if((t + l) >= total_c) continue; const float s2 = IsNaNOrInf(outputs[RCtoFlat(c, t + l, total_r, total_c, step - 2)], 0); grad += IsNaNOrInf(outputs_g[RCtoFlat(r, t + l, total_r, total_c, step)] * s2, 0); } grad = LocalSum(grad, 0, Temp); if(l == 0) outputs_g[RCtoFlat(r, c, total_r, total_c, 1)] += grad; BarrierLoc //--- T(k-1) grad grad = 0; for(int t = 0; t < total_c; t += total_l) { if((t + l) >= total_c) continue; const float s2 = IsNaNOrInf(support[RCtoFlat(t + l, r, total_r, total_c, 0)], 0); grad += IsNaNOrInf(outputs_g[RCtoFlat(t + l, c, total_r, total_c, step)] * s2, 0); } grad = LocalSum(grad, 0, Temp); if(l == 0) outputs_g[RCtoFlat(r, c, total_r, total_c, step - 1)] += grad; } //--- if(step <= 2) { if(l == 0) support_g[RCtoFlat(r, c, total_r, total_c, 0)] = outputs_g[RCtoFlat(r, c, total_r, total_c, 1)]; return; } } //+------------------------------------------------------------------+ //| Implements the significant neighbors sampling kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SignificantNeighborsSampling(__global const float *data, __global const float *candidates, __global const float *random_cands, __global float *neighbors, const int dimension ) { const size_t main = get_global_id(0); const size_t slave = get_local_id(1); const int total_main = (int)get_global_size(0); const int total_slave = (int)get_local_size(1); //--- __local int Idx[LOCAL_ARRAY_SIZE]; __local float Temp[LOCAL_ARRAY_SIZE]; const int ls = min(total_slave, (int)LOCAL_ARRAY_SIZE); //--- const int shift_main = RCtoFlat(main, 0, total_main, dimension, 0); int cand = (int)candidates[slave]; int rand_cand = (int)random_cands[slave]; //--- duplicate check if(rand_cand == cand) rand_cand = -1; //--- Look in candidates for(int l = 0; l < total_slave; l += ls) { if(slave >= l && slave < (l + ls)) Idx[slave - l] = cand; BarrierLoc for(int i = 0; i < ls; i++) { if(i >= (slave - l)) continue; if(cand == Idx[i]) cand = -1; if(rand_cand == Idx[i]) rand_cand = -1; } BarrierLoc } //--- Look in random candidates for(int l = 0; l < total_slave; l += ls) { if(slave >= l && slave < (l + ls)) Idx[slave - l] = rand_cand; BarrierLoc for(int i = 0; i < ls; i++) { if(i >= (slave - l)) continue; if(cand == Idx[i]) cand = -1; if(rand_cand == Idx[i]) rand_cand = -1; } BarrierLoc } //--- const int shift_cand = RCtoFlat(cand, 0, total_main, dimension, 0); const int shift_rand_cand = RCtoFlat(rand_cand, 0, total_main, dimension, 0); //--- calc distance float dist_cand = 0; float dist_rand_cand = 0; for(int d = 0; d < dimension; d++) { float value = IsNaNOrInf(data[shift_main + d], 0); if(main != cand && cand >= 0) { float delta = value - IsNaNOrInf(data[shift_cand + d], 0); dist_cand += delta * delta; } if(main != rand_cand && rand_cand >= 0) { float delta = value - IsNaNOrInf(data[shift_rand_cand + d], 0); dist_rand_cand += delta * delta; } } //--- calc position int cand_position = 0; int rand_position = (int)(dist_cand >= dist_rand_cand); //--- by candidates for(int l = 0; l < total_slave; l += ls) { if(slave >= l && slave < (l + ls)) Temp[slave - l] = (cand >= 0 ? IsNaNOrInf(dist_cand, -1) : -1); BarrierLoc for(int i = 0; i < ls; i++) { if(i == (slave - l)) continue; if(Temp[i] < 0) continue; if(cand >= 0) { if(Temp[i] < dist_cand) cand_position++; else if(Temp[i] < dist_cand && i < (slave - l)) cand_position++; } if(rand_cand >= 0) { if(Temp[i] < dist_rand_cand) rand_position++; else if(Temp[i] < dist_rand_cand && i < (slave - l)) rand_position++; } } BarrierLoc } //--- by random candidates for(int l = 0; l < total_slave; l += ls) { if(slave >= l && slave < (l + ls)) Temp[slave - l] = (rand_cand >= 0 ? IsNaNOrInf(dist_rand_cand, -1) : -1); BarrierLoc for(int i = 0; i < ls; i++) { if(i == (slave - l)) continue; if(Temp[i] < 0) continue; if(cand >= 0) { if(Temp[i] < dist_cand) cand_position++; else if(Temp[i] < dist_cand && i < (slave - l)) cand_position++; } if(rand_cand >= 0) { if(Temp[i] < dist_rand_cand) rand_position++; else if(Temp[i] < dist_rand_cand && i < (slave - l)) rand_position++; } } BarrierLoc } //--- result if(cand >= 0 && cand_position < total_slave) { const int shift_dist_cand = RCtoFlat(main, cand_position, total_main, total_slave, 0); neighbors[shift_dist_cand] = cand; } if(rand_cand >= 0 && rand_position < total_slave) { const int shift_dist_cand = RCtoFlat(main, rand_position, total_main, total_slave, 0); neighbors[shift_dist_cand] = rand_cand; } } //+------------------------------------------------------------------+ //| Implements the sparse mh scores kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SparseMHScores(__global const float* data, __global const float* indexes, __global float* scores, const float sparse ///< [0.0 .. 1.0) coefficient of sparse ) { const int main = (int)get_global_id(0); const int slave = (int)get_local_id(1); const int head = (int)get_global_id(2); const int total_mains = (int)get_global_size(0); const int total_slaves = (int)get_local_size(1); const int total_heads = (int)get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- float value = IsNaNOrInf(data[RCtoFlat(main, head, total_mains, 2 * total_heads, 0)], 0); int slave_id = (int)indexes[RCtoFlat(main, slave, total_mains, total_slaves, 0)]; if(slave_id < total_mains && slave_id >= 0) value += IsNaNOrInf(data[RCtoFlat(slave_id, head + total_heads, total_mains, 2 * total_heads, 0)], 0); //--- const float max_value = LocalMax(value, 1, Temp); const float min_value = LocalMin(value, 1, Temp); const float threshold = (max_value - min_value) * sparse + min_value; value = (threshold <= value ? IsNaNOrInf(exp(value - max_value), 0) : 0); const float sum = LocalSum(value, 1, Temp); value = IsNaNOrInf(value / sum, 0); //--- scores[RCtoFlat(slave, head, total_slaves, total_heads, main)] = value; } //+------------------------------------------------------------------+ //| Computes the sparse mh scores grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SparseMHScoresGrad(__global float* data_gr, __global const float* indexes, __global const float* scores, __global const float* scores_gr ) { const int main = (int)get_global_id(0); const int slave = (int)get_local_id(1); const int head = (int)get_global_id(2); const int total_mains = (int)get_global_size(0); const int total_slaves = (int)get_local_size(1); const int total_heads = (int)get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const uint ls = min((uint)total_slaves, (uint)LOCAL_ARRAY_SIZE); //--- Calc grad by main { float value = IsNaNOrInf(scores[RCtoFlat(slave, head, total_slaves, total_heads, main)], 0); int slave_id = (int)indexes[RCtoFlat(main, slave, total_mains, total_slaves, 0)]; const float sc_gr = IsNaNOrInf(scores_gr[RCtoFlat(slave, head, total_slaves, total_heads, main)], 0); //--- float grad = 0; for(uint d = 0; d < total_slaves; d += ls) { if(slave >= d && slave < (d + ls)) Temp[slave - d] = IsNaNOrInf(sc_gr, 0); BarrierLoc for(uint l = 0; l < min(ls, (uint)(total_slaves - d)); l++) grad += IsNaNOrInf(Temp[l] * ((float)((d + l) == slave && slave_id == main) - value), 0); BarrierLoc } grad = LocalSum(grad, 1, Temp); if(slave == 0) data_gr[RCtoFlat(main, head, total_mains, 2 * total_heads, 0)] = grad; } //--- Calc grad by slave { float grad = 0; for(uint d = 0; d < total_mains; d++) { float value = IsNaNOrInf(scores[RCtoFlat(slave, head, total_slaves, total_heads, d)], 0); const float sc_gr = IsNaNOrInf(scores_gr[RCtoFlat(slave, head, total_slaves, total_heads, d)], 0); int slave_id = (int)indexes[RCtoFlat(d, slave, total_mains, total_slaves, 0)]; //--- float gr = IsNaNOrInf(sc_gr * ((float)(slave_id == d) - value), 0); gr = LocalSum(gr, 1, Temp); if(slave == 0) grad += gr; } if(slave == 0) data_gr[RCtoFlat(main, head + total_heads, total_mains, 2 * total_heads, 0)] = IsNaNOrInf(grad, 0); } } //+------------------------------------------------------------------+ //| Implements the sparse mat mult kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SparseMatMult(__global const float *sparse_index, __global const float *sparse_data, __global const float *full, __global float *result, const int full_rows ) { const size_t sparse_row = get_global_id(0); const size_t sparse_col = get_local_id(1); const size_t full_col = get_global_id(2); const size_t sparse_rows = get_global_size(0); const size_t sparse_cols = get_local_size(1); const size_t full_cols = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- const int shift_sparse = RCtoFlat(sparse_row, sparse_col, sparse_rows, sparse_cols, 0); const int full_row = sparse_index[shift_sparse]; const int shift_full = RCtoFlat(full_row, full_col, full_rows, full_cols, 0); //--- float res = (full_row >= 0 && full_row < full_rows ? IsNaNOrInf(sparse_data[shift_sparse] * full[shift_full], 0) : 0); res = LocalSum(res, 1, Temp); //--- if(sparse_col == 0) { const int shift_result = RCtoFlat(sparse_row, full_col, sparse_rows, full_cols, 0); result[shift_result] = res; } } //+------------------------------------------------------------------+ //| Computes the sparse mat mult grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SparseMatMultGrad(__global const float *sparse_index, __global const float *sparse_data, __global float *sparse_gr, __global const float *full, __global float *full_gr, __global const float *result_gr, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols ) { const size_t row_id = get_global_id(0); const size_t local_id = get_local_id(1); const size_t col_id = get_global_id(2); const size_t total_rows = get_global_size(0); const size_t total_local = get_local_size(1); const size_t total_cols = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- Calce sparse gradient if(row_id < sparse_rows && col_id < sparse_cols) { float grad = 0; int shift_sparse = 0; if(local_id == 0) { shift_sparse = RCtoFlat(row_id, col_id, sparse_rows, sparse_cols, 0); Temp[0] = sparse_index[shift_sparse]; } BarrierLoc uint full_row = (uint)Temp[0]; if(full_row < (uint)full_rows) for(int i = local_id; i < full_cols; i += total_local) { int shift_result = RCtoFlat(row_id, i, sparse_rows, full_cols, 0); int shift_full = RCtoFlat(full_row, i, full_rows, full_cols, 0); grad += IsNaNOrInf(result_gr[shift_result] * full[shift_full], 0); } grad = LocalSum(grad, 1, Temp); if(local_id == 0) sparse_gr[shift_sparse] = grad; } //--- Calce full gradient if(row_id < full_rows && col_id < full_cols) { float grad = 0; for(int r = 0; r < sparse_rows; r ++) { float s = 0; for(int c = local_id; c < sparse_cols; c += total_local) { int shift_sparse = RCtoFlat(r, c, sparse_rows, sparse_cols, 0); if((int)sparse_index[shift_sparse] == (int)row_id) { s = sparse_data[shift_sparse]; break; } } s = LocalSum(s, 1, Temp); if(s != 0 && local_id == 0) { int shift_result = RCtoFlat(r, col_id, sparse_rows, full_cols, 0); grad += IsNaNOrInf(s * result_gr[shift_result], 0); } } if(local_id == 0) { int shift_full = RCtoFlat(row_id, col_id, full_rows, full_cols, 0); full_gr[shift_full] = grad; } } } //+------------------------------------------------------------------+ //| Implements the random walk kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RandomWalk(__global const float *data, __global float *inv_diag, __global float *norm, const int total_cols ) { const size_t row_id = get_global_id(0); const size_t local_id = get_local_id(1); const size_t total_rows = get_global_size(0); const size_t total_local = get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- float d = 0; for(int c = local_id; c < total_cols; c += total_local) { int shift = RCtoFlat(row_id, c, total_rows, total_cols, 0); d += IsNaNOrInf(data[shift], 0); } d = IsNaNOrInf(1.0f / (LocalSum(d, 1, Temp) + 1.0f), 1.0f); if(local_id == 0) inv_diag[row_id] = d; //--- for(int c = local_id; c < total_cols; c += total_local) { int shift = RCtoFlat(row_id, c, total_rows, total_cols, 0); norm[shift] = IsNaNOrInf(data[shift] * d, 0); } } //+------------------------------------------------------------------+ //| Performs the concat by label data-layout operation on the GPU. | //+------------------------------------------------------------------+ __kernel void ConcatByLabel(__global const float* data, __global const float* label, __global const float* embedding1, __global const float* embedding2, __global float *output, const int dimension_data, const int dimension_emb1, const int dimension_emb2, const int frame1, const int frame2, const int period1, const int period2 ) { const size_t row_id = get_global_id(0); const size_t col_id = get_global_id(1); const size_t buffer_id = get_global_id(2); const size_t total_rows = get_global_size(0); const size_t total_cols = get_global_size(1); const size_t total_buffers = get_global_size(2); //--- __global const float *buffer; int dimension_in, dimension_out; int shift_in, shift_out; //--- switch(total_buffers) { case 1: dimension_out = dimension_data; break; case 2: dimension_out = dimension_data + dimension_emb1; break; case 3: dimension_out = dimension_data + dimension_emb1 + dimension_emb2; break; default: return; } //--- switch(buffer_id) { case 0: buffer = data; dimension_in = dimension_data; shift_in = RCtoFlat(row_id, col_id, total_rows, dimension_in, 0); shift_out = RCtoFlat(row_id, col_id, total_rows, dimension_out, 0); break; case 1: buffer = embedding1; dimension_in = dimension_emb1; shift_in = ((int)IsNaNOrInf(label[row_id] / frame1, 0)) % period1; shift_in = RCtoFlat(shift_in, col_id, period1, dimension_in, 0); shift_out = RCtoFlat(row_id, dimension_data + col_id, total_rows, dimension_out, 0); break; case 2: buffer = embedding2; dimension_in = dimension_emb2; shift_in = ((int)IsNaNOrInf(label[row_id] / frame2, 0)) % period2; shift_in = RCtoFlat(shift_in, col_id, period2, dimension_in, 0); shift_out = RCtoFlat(row_id, dimension_data + dimension_emb1 + col_id, total_rows, dimension_out, 0); break; } //--- if(col_id < dimension_in) output[shift_out] = IsNaNOrInf(buffer[shift_in], 0); } //+------------------------------------------------------------------+ //| Computes the concat by label grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void ConcatByLabelGrad(__global float* data_gr, __global const float* label, __global float* embedding1_gr, __global float* embedding2_gr, __global float *output_gr, const int dimension_data, const int dimension_emb1, const int dimension_emb2, const int frame1, const int frame2, const int period1, const int period2, const int units ) { const size_t row_id = get_global_id(0); const size_t col_id = get_global_id(1); const size_t buffer_id = get_global_id(2); const size_t total_rows = get_global_size(0); const size_t total_cols = get_global_size(1); const size_t total_buffers = get_global_size(2); //--- __global float *buffer; int dimension_in, dimension_out; int shift_in, shift_out, shift_col; int period, frame, rows; //--- switch(total_buffers) { case 1: dimension_out = dimension_data; break; case 2: dimension_out = dimension_data + dimension_emb1; break; case 3: dimension_out = dimension_data + dimension_emb1 + dimension_emb2; break; default: return; } //--- switch(buffer_id) { case 0: if(col_id < dimension_data && row_id < units) { shift_in = RCtoFlat(row_id, col_id, total_rows, dimension_in, 0); shift_out = RCtoFlat(row_id, col_id, total_rows, dimension_out, 0); data_gr[shift_in] = IsNaNOrInf(output_gr[shift_out], 0); } return; case 1: rows = period1; buffer = embedding1_gr; dimension_in = dimension_emb1; shift_in = RCtoFlat(row_id, col_id, period1, dimension_in, 0); shift_col = dimension_data; period = period1; frame = frame1; break; case 2: rows = period2; buffer = embedding2_gr; dimension_in = dimension_emb2; shift_in = RCtoFlat(row_id, col_id, period2, dimension_in, 0); shift_col = dimension_data + dimension_emb1; period = period2; frame = frame2; break; } //--- if(row_id >= rows || col_id >= dimension_in) return; float grad = 0; for(uint r = 0; r < total_rows; r ++) { int row = ((int)IsNaNOrInf(label[r] / frame, 0)) % period; if(row != row_id) continue; shift_out = RCtoFlat(r, shift_col + col_id, total_rows, dimension_out, 0); grad += IsNaNOrInf(output_gr[shift_out], 0); } buffer[shift_in] = IsNaNOrInf(grad, 0); } //+------------------------------------------------------------------+ //| Implements the global local attention kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void GlobalLocalAttention(__global const float *q, __global const float2* kv, __global float *scores, __global const float* mask, __global const float* label, __global float *out, const int dimension, const int total_kv, const int total_mask ) { //--- init const int q_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_local = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Score int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); if(h_id % 2 == 0) { const int shift_kv = RCtoFlat(h_id, 0, total_heads, dimension, local_id); const int shift_s = RCtoFlat(h_id / 2, local_id, total_heads / 2, total_kv + total_mask, q_id); float score = 0; if(local_id < total_kv) { for(int d = 0; d < dimension; d++) score += IsNaNOrInf(q[shift_q + d] * kv[shift_kv + d].s0, 0); } else score = MIN_VALUE; //--- norm score score = LocalSoftMax(score, 1, temp); if(local_id < total_kv) scores[shift_s] = score; //--- out for(int d = 0; d < dimension; d++) { float val = (local_id < total_kv ? kv[shift_kv + d].s1 * score : 0); val = LocalSum(val, 1, temp); if(local_id == 0) out[shift_q + d] = val; } } else { int kv_id = -1; float score = 0; int shift_kv = -1; float m = 0; const int shift_s = RCtoFlat(h_id / 2, total_kv + local_id, total_heads / 2, total_kv + total_mask, q_id); if(local_id < total_mask) { const int l = RCtoFlat(q_id, local_id, total_q, total_mask, 0); kv_id = IsNaNOrInf(label[l], -1); m = IsNaNOrInf(mask[l], 0); shift_kv = RCtoFlat(h_id, 0, total_heads, dimension, kv_id); if(kv_id >= 0) for(int d = 0; d < dimension; d++) score += IsNaNOrInf(q[shift_q + d] * kv[shift_kv + d].s0, 0); else score = MIN_VALUE; } else score = MIN_VALUE; //--- norm score score = LocalSoftMax(score * m, 1, temp); if(local_id < total_mask) scores[shift_s] = score; //--- out for(int d = 0; d < dimension; d++) { float val = (kv_id >= 0 ? IsNaNOrInf(kv[shift_kv + d].s1, 0) * score : 0); val = LocalSum(val, 1, temp); if(local_id == 0) out[shift_q + d] = val; } } } //+------------------------------------------------------------------+ //| Computes the global local attention grad gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void GlobalLocalAttentionGrad(__global const float *q, __global float *q_gr, __global const float *kv, __global float *kv_gr, __global float *scores, __global const float *mask, __global float *mask_gr, __global const float *label, __global float *out_gr, const int dimension, const int total_q, const int total_kv, const int total_mask ) { //--- init const int global_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_global = get_global_size(0); const int total_local = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- if(h_id % 2 == 0) { //--- Value Gradient global_id -> v_id, local_id -> q_id for(int d = 0; d < dimension; d++) { const int shift_v = RCtoFlat(h_id, 2 * d + 1, total_heads, 2 * dimension, global_id); float grad = 0; for(int q_id = local_id; q_id < total_q; q_id += total_local) { int shift_s = RCtoFlat(h_id / 2, global_id, total_heads / 2, total_kv + total_mask, q_id); int shift_q = RCtoFlat(h_id, d, total_heads, dimension, q_id); grad += IsNaNOrInf(scores[shift_s] * out_gr[shift_q], 0); } grad = LocalSum(grad, 1, temp); kv_gr[shift_v] = grad; } //--- Query Gradient global_id -> q_id, local_id -> k_id/v_id if(global_id < total_q) { //--- 1. Score grad float grad_s = 0; const int shift_v = RCtoFlat(h_id, 1, total_heads, 2 * dimension, local_id); const int shift_s = RCtoFlat(h_id / 2, local_id, total_heads / 2, total_kv + total_mask, global_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, global_id); if(local_id < total_kv) for(int d = 0; d < dimension; d++) grad_s += IsNaNOrInf(kv[shift_v + 2 * d] * out_gr[shift_q + d], 0); //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); //--- 3. Query grad const int shift_k = shift_v - 1; for(int d = 0; d < dimension; d++) { float grad = 0; if(local_id < total_kv) grad = kv[shift_k + 2 * d] * grad_s; grad = LocalSum(grad, 1, temp); if(local_id == 0) q_gr[shift_q + d] = grad; } } //--- Key Gradient global_id -> k_id, local_id -> score_id/v_id/dimension if(global_id < total_kv) { float grad = 0; for(int q_id = 0; q_id < total_q; q_id++) { //--- 1. Score grad local_id -> score_id/v_id float grad_s = 0; const int shift_v = RCtoFlat(h_id, 1, total_heads, 2 * dimension, local_id); const int shift_s = RCtoFlat(h_id / 2, local_id, total_heads / 2, total_kv + total_mask, q_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); if(local_id < total_kv) for(int d = 0; d < dimension; d++) grad_s += IsNaNOrInf(kv[shift_v + 2 * d] * out_gr[shift_q + d], 0); //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); BarrierLoc if(global_id == local_id) temp[0] = grad_s; BarrierLoc grad_s = temp[0]; //--- 3. Key grad local_id -> dimension shift_q = RCtoFlat(h_id, local_id, total_heads, dimension, q_id); if(local_id < dimension) grad += IsNaNOrInf(q[shift_q] * grad_s, 0); } const int shift_k = RCtoFlat(h_id, 2 * local_id, total_heads, 2 * dimension, global_id); if(local_id < dimension) kv_gr[shift_k] = IsNaNOrInf(grad, 0); } } else { //--- Value Gradient global_id -> v_id, local_id -> mask_index/dimension if(global_id < total_kv) { float grad = 0; for(int q_id = 0; q_id < total_q; q_id++) { //--- 1. kv_id int kv_id = -1; float m = 0; const int l = RCtoFlat(q_id, local_id, total_q, total_mask, 0); const int shift_s = RCtoFlat(h_id / 2, total_kv + local_id, total_heads / 2, total_kv + total_mask, q_id); //--- Check for use current Value if(local_id < total_mask) kv_id = (int)label[l]; if(local_id == 0) temp[0] = 0; BarrierLoc if(kv_id == global_id) temp[0] = scores[shift_s]; BarrierLoc if(temp[0] == 0) continue; //--- Value grad int shift_q = RCtoFlat(h_id, local_id, total_heads, dimension, q_id); if(local_id < dimension) grad += IsNaNOrInf(temp[0] * out_gr[shift_q], 0); } const int shift_v = RCtoFlat(h_id, 2 * local_id + 1, total_heads, 2 * dimension, global_id); if(local_id < dimension) kv_gr[shift_v] = IsNaNOrInf(grad, 0); } //--- Query Gradient global_id -> q_id, local_id -> mask label if(global_id < total_q) { //--- 1. kv_id; int kv_id = -1; float m = 0; const int l = RCtoFlat(global_id, local_id, total_q, total_mask, 0); if(local_id < total_mask) { kv_id = (int)IsNaNOrInf(label[l], -1); m = IsNaNOrInf(mask[l], 0); } //--- 2. Score grad float grad_s = 0; const int shift_v = RCtoFlat(h_id, 1, total_heads, 2 * dimension, kv_id); const int shift_s = RCtoFlat(h_id / 2, total_kv + local_id, total_heads / 2, total_kv + total_mask, global_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, global_id); if(local_id < total_mask) for(int d = 0; d < dimension; d++) grad_s += IsNaNOrInf(kv[shift_v + 2 * d] * out_gr[shift_q + d], 0); //--- 3. SoftMax grad float score = IsNaNOrInf(scores[shift_s], 0); grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); mask_gr[l] = IsNaNOrInf(grad_s * score, 0); grad_s *= m; //--- 4. Query grad const int shift_k = shift_v - 1; for(int d = 0; d < dimension; d++) { float grad = 0; if(local_id < total_mask) grad = kv[shift_k + 2 * d] * grad_s; grad = LocalSum(grad, 1, temp); if(local_id == 0) q_gr[shift_q + d] = grad; } } //--- Key Gradient global_id -> k_id, local_id -> score_id/v_id/dimension if(global_id < total_kv) { float grad = 0; for(int q_id = 0; q_id < total_q; q_id++) { //--- 1. kv_id; int kv_id = -1; float m = 0; const int l = RCtoFlat(global_id, local_id, total_q, total_mask, 0); if(local_id < total_mask) { kv_id = (int)label[l]; if(kv_id == global_id) m = mask[l]; } m = LocalSum(m, 1, temp); if(m == 0) continue; //--- 2. Score grad local_id -> score_id/v_id float grad_s = 0; const int shift_v = RCtoFlat(h_id, 1, total_heads, 2 * dimension, kv_id); const int shift_s = RCtoFlat(h_id / 2, total_kv + local_id, total_heads / 2, total_kv + total_mask, q_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); if(local_id < total_mask) for(int d = 0; d < dimension; d++) grad_s += IsNaNOrInf(kv[shift_v + 2 * d] * out_gr[shift_q + d], 0); //--- 3. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); BarrierLoc if(global_id == local_id) temp[0] = grad_s * m; BarrierLoc grad_s = temp[0]; //--- 4. Key grad local_id -> dimension shift_q = RCtoFlat(h_id, local_id, total_heads, dimension, q_id); if(local_id < dimension) grad += IsNaNOrInf(q[shift_q] * grad_s, 0); } const int shift_k = RCtoFlat(h_id, 2 * local_id, total_heads, 2 * dimension, global_id); if(local_id < dimension) kv_gr[shift_k] = IsNaNOrInf(grad, 0); } } } //+------------------------------------------------------------------+ //| Applies the sparse soft max normalization to the input scores. | //+------------------------------------------------------------------+ __kernel void SparseSoftMax(__global const float *data, __global float *outputs, __global float *indexes, const int out_dimension ) { const size_t row = get_global_id(0); const size_t col_in = get_local_id(1); const int total_rows = (int)get_global_size(0); const int total_cols_in = (int)get_local_size(1); //--- __local float Temp[LOCAL_ARRAY_SIZE]; const int ls = min(total_cols_in, (int)LOCAL_ARRAY_SIZE); //--- const int shift_in = RCtoFlat(row, col_in, total_rows, total_cols_in, 0); //--- calc position float value = IsNaNOrInf(data[shift_in], MIN_VALUE); int position = 0; for(int l = 0; l < total_cols_in; l += ls) { if(col_in >= l && col_in < (l + ls)) Temp[col_in - l] = value; BarrierLoc for(int i = 0; i < ls; i++) { if(i == (col_in - l)) continue; if(Temp[i] > value) position++; else if(Temp[i] == value && i < (col_in - l)) position++; } BarrierLoc } //--- SoftMax if(position >= out_dimension) value = MIN_VALUE; value = LocalSoftMax(value, 1, Temp); //--- result const int shift_out = RCtoFlat(row, position, total_rows, out_dimension, 0); if(position < out_dimension) { outputs[shift_out] = value; indexes[shift_out] = (float)col_in; } } //+------------------------------------------------------------------+ //| Computes the sparse soft max grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void SparseSoftMaxGrad(__global float *data_gr, __global const float *outputs, __global const float *outputs_gr, __global const float *indexes, const int out_dimension ) { const size_t row = get_global_id(0); const size_t col_in = get_local_id(1); const int total_rows = (int)get_global_size(0); const int total_cols_in = (int)get_local_size(1); //--- __local int Ind[LOCAL_ARRAY_SIZE]; __local float Temp[LOCAL_ARRAY_SIZE]; const int ls = min(total_cols_in, (int)LOCAL_ARRAY_SIZE); //--- look position float value = 0; float grad = 0; int position = -1; int idx = -1; const int shift_idx = RCtoFlat(row, col_in, total_rows, out_dimension, 0); if(col_in < out_dimension) idx = (int)IsNaNOrInf(indexes[shift_idx], -1.0f); for(int l = 0; l < out_dimension; l += ls) { if(col_in >= l && col_in < (l + ls)) Ind[col_in - l] = idx; BarrierLoc for(int i = 0; (i < ls && position < 0); i++) { if(Ind[i] == col_in) position = l + i; } BarrierLoc } //--- SoftMax Grad if(position < out_dimension && position >= 0) { const int shift_out = RCtoFlat(row, position, total_rows, out_dimension, 0); value = IsNaNOrInf(outputs[shift_out], 0); grad = IsNaNOrInf(outputs_gr[shift_out], 0); } grad = LocalSoftMaxGrad(value, grad, 1, Temp); //--- result const int shift_in = RCtoFlat(row, col_in, total_rows, total_cols_in, 0); data_gr[shift_in] = grad; } //+------------------------------------------------------------------+ //| Implements the float to spike kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void FloatToSpike(__global float* values, __global const float* levels, __global float* outputs ) { const size_t id = get_global_id(0); float val = IsNaNOrInf(values[id], 0.0f); if(val == 0.0f) outputs[id] = 0.0f; else { const float lev = IsNaNOrInf(levels[id], 0.0f); if(fabs(val) < lev) outputs[id] = 0.0f; else { outputs[id] = (float)sign(val); values[id] = IsNaNOrInf(sign(val) * (fabs(val) - lev), 0.0f); } } } //+------------------------------------------------------------------+ //| Computes the float to spike grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void FloatToSpikeGrad(__global const float* values, __global float* values_gr, __global float* levels_gr, __global const float* gradients ) { const size_t id = get_global_id(0); const float grad = IsNaNOrInf(gradients[id], 0.0f); values_gr[id] = grad; if(fabs(grad) > 0.0f) { float val = IsNaNOrInf(values[id], 0.0f); levels_gr[id] = (float)(-sign(val) * grad); } else levels_gr[id] = 0.0f; } //+------------------------------------------------------------------+ //| Implements the spike mh attention kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SpikeMHAttention(__global const float *qkv, __global const float *diag_bias, __global float *scores, __global float *out, const int dimension, const int mask_future ) { //--- init const int q_id = get_global_id(0); const int k_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Shifts const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, 3 * q_id); const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 3 * k_id + 1); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 3 * k_id + 2); const int shift_s = RCtoFlat(h_id, k_id, total_heads, total_q, q_id); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, q_id); //--- Score float score = 0; if(mask_future == 0 || q_id <= k_id) { for(int d = 0; d < dimension; d++) { float q = IsNaNOrInf(qkv[shift_q + d], 0); if(q == 0) continue; float k = IsNaNOrInf(qkv[shift_k + d], 0); if(k == 0) continue; score += q * k; } } else score = MIN_VALUE; if(q_id == k_id) score += IsNaNOrInf(diag_bias[q_id], 0); //--- norm score score = LocalSoftMax(score, 1, temp); scores[shift_s] = score; //--- out for(int d = 0; d < dimension; d++) { float val = 0; if(score > 0) { float v = IsNaNOrInf(qkv[shift_v + d], 0); if(v != 0) val = v * score; } val = LocalSum(val, 1, temp); if(k_id == 0) out[shift_out + d] = val; } } //+------------------------------------------------------------------+ //| Computes the spike mh attention grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void SpikeMHAttentionGrad(__global const float *qkv, __global float *qkv_gr, __global const float *diag_bias, __global float *diag_bias_gr, __global const float *scores, __global const float *gradients, const int dimension, const int mask_future ) { //--- init const int global_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_global = get_global_size(0); const int total_local = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Value Gradient global_id -> v_id, local_id -> q_id { //--- Shifts const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 3 * global_id + 2); const int shift_s = RCtoFlat(h_id, global_id, total_heads, total_global, local_id); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, local_id); for(int d = 0; d < dimension; d++) { float grad = 0; if(mask_future == 0 || local_id <= global_id) { float score = IsNaNOrInf(scores[shift_s], 0); if(score > 0) grad = IsNaNOrInf(score * gradients[shift_out + d], 0); } grad = LocalSum(grad, 1, temp); if(local_id == 0) qkv_gr[shift_v + d] = grad; } } //--- Query Gradient global_id -> q_id, local_id -> k_id/v_id { //--- Shifts const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, 3 * global_id); const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 3 * local_id + 1); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 3 * local_id + 2); const int shift_s = RCtoFlat(h_id, local_id, total_heads, total_local, global_id); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, global_id); //--- 1. Score grad float grad_s = 0; if(mask_future == 0 || global_id <= local_id) for(int d = 0; d < dimension; d++) { float val = IsNaNOrInf(qkv[shift_v + d], 0); if(val == 0) continue; grad_s += IsNaNOrInf(qkv[shift_v + d] * gradients[shift_out + d], 0); } //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); if(global_id == local_id) diag_bias_gr[global_id] = grad_s; //--- 3. Query grad for(int d = 0; d < dimension; d++) { float grad = 0; if(mask_future == 0 || global_id <= local_id) { float key = IsNaNOrInf(qkv[shift_k + d], 0); if(key != 0) grad = key * grad_s; } grad = LocalSum(grad, 1, temp); if(local_id == 0) qkv_gr[shift_q + d] = grad; } } //--- Key Gradient global_id -> k_id, local_id -> score_id/v_id/dimension { //--- Shifts const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 3 * global_id + 1); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 3 * local_id + 2); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, local_id); float grad = 0; for(int q_id = 0; q_id < total_local; q_id++) { //--- 1. Score grad local_id -> score_id/v_id float grad_s = 0; const int shift_s = RCtoFlat(h_id, local_id, total_heads, total_local, q_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, 3 * q_id); if(mask_future == 0 || q_id <= local_id) for(int d = 0; d < dimension; d++) { float val = IsNaNOrInf(qkv[shift_v + d], 0); if(val == 0) continue; grad_s += IsNaNOrInf(val * gradients[shift_q + d], 0); } //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); BarrierLoc if(global_id == local_id) temp[0] = grad_s; BarrierLoc grad_s = temp[0]; //--- 3. Key grad local_id -> dimension if(local_id < dimension) { float query = IsNaNOrInf(qkv[shift_q + local_id], 0); if(query != 0) grad += IsNaNOrInf(query * grad_s, 0); } } if(local_id < dimension) qkv_gr[shift_k + local_id] = IsNaNOrInf(grad, 0); } } //+------------------------------------------------------------------+ //| Implements the stfs kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void STFS(__global const float* inputs, __global const float* mask_time, __global const float* mask_spatial, __global float* outputs ) { const size_t time_id = get_global_id(0); const size_t spat_id = get_global_id(1); const size_t head = get_local_id(2); const size_t total_times = get_global_size(0); const size_t total_spats = get_global_size(1); const size_t total_heads = get_local_size(2); //--- __local float temp[3]; //--- const int shift_in = RCtoFlat(time_id, spat_id, total_times, total_spats, 1); const int shift_out = RCtoFlat(time_id, spat_id, total_times, total_spats, head); //--- switch(head) { case 0: temp[0] = IsNaNOrInf(inputs[shift_in], 0); break; case 1: temp[1] = IsNaNOrInf(mask_time[time_id], 0); break; case 2: temp[2] = IsNaNOrInf(mask_spatial[spat_id], 0); break; } BarrierLoc float out = temp[0]; if(out != 0) switch(head) { case 1: out *= temp[1]; break; case 2: out *= (1 - temp[1]); break; case 3: out *= temp[2]; break; case 4: out *= (1 - temp[2]); break; } //--- outputs[shift_out] = IsNaNOrInf(out, 0); } //+------------------------------------------------------------------+ //| Computes the stfs grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void STFSGrad(__global float* inputs_gr, __global const float* mask_time, __global const float* mask_spatial, __global const float* outputs_gr ) { const size_t time_id = get_global_id(0); const size_t spat_id = get_global_id(1); const size_t head = get_local_id(2); const size_t total_times = get_global_size(0); const size_t total_spats = get_global_size(1); const size_t total_heads = get_local_size(2); //--- __local float temp[5]; //--- const int shift_in = RCtoFlat(time_id, spat_id, total_times, total_spats, 1); const int shift_out = RCtoFlat(time_id, spat_id, total_times, total_spats, head); //--- switch(head) { case 0: temp[1] = IsNaNOrInf(mask_time[time_id], 0); break; case 1: temp[2] = IsNaNOrInf(mask_spatial[spat_id], 0); break; } BarrierLoc float grad = IsNaNOrInf(outputs_gr[shift_out], 0); if(grad != 0) switch(head) { case 1: grad *= temp[1]; break; case 2: grad *= (1 - temp[1]); break; case 3: grad *= temp[2]; break; case 4: grad *= (1 - temp[2]); break; } //--- grad = LocalSum(grad, 2, temp); BarrierLoc if(head == 0) inputs_gr[shift_in] = grad; } //+------------------------------------------------------------------+ //| Applies the add to stack operation to the GPU buffers. | //+------------------------------------------------------------------+ __kernel void AddToStack(__global const float* inputs, __global float* stack, const int stack_size) { const size_t id = get_global_id(0); const size_t loc_id = get_local_id(1); const size_t var = get_global_id(2); const size_t dimension = get_global_size(0); const size_t total_loc = get_local_size(1); const size_t variables = get_global_size(2); //--- const int total = (stack_size + total_loc - 1) / total_loc; for(int i = total; i >= 0; i--) { float inp = 0; if(i == 0 && loc_id == 0) inp = IsNaNOrInf(inputs[RCtoFlat(var, id, variables, dimension, 0)], 0); else if((i * total_loc + loc_id) < stack_size) { int shift = RCtoFlat(i * total_loc + loc_id - 1, id, stack_size, dimension, var); inp = IsNaNOrInf(stack[shift], 0); } BarrierLoc if((i * total_loc + loc_id) < stack_size) { int shift = RCtoFlat(i * total_loc + loc_id, id, stack_size, dimension, var); stack[shift] = inp; } } } //+------------------------------------------------------------------+ //| Implements the aggregation by time kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void AggregationByTime(__global const float* inputs, __global const float* stack, __global float* outputs, const int stack_size, const int levels ) { const size_t id = get_global_id(0); const size_t var = get_global_id(1); const size_t dimension = get_global_size(0); const size_t variables = get_global_size(1); //--- float val = IsNaNOrInf(inputs[RCtoFlat(var, id, variables, dimension, 0)], 0); outputs[RCtoFlat(var, id, variables, dimension, 0)] = val; for(int l = 1; l < levels; l++) { int total = 1 << l; int start = total - 1; val /= total; for(int s = 0; s < total; s++) { if(s + start >= stack_size) continue; val += IsNaNOrInf(stack[RCtoFlat(var, id, variables * levels, dimension, start + s)] / total, 0); } outputs[RCtoFlat(var, id, variables, dimension, l)] = IsNaNOrInf(val, 0); } } //+------------------------------------------------------------------+ //| Computes the aggregation by time grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void AggregationByTimeGrad(__global float* inputs_gr, __global const float* outputs_gr, const int levels ) { const size_t id = get_global_id(0); const size_t var = get_global_id(1); const size_t dimension = get_global_size(0); const size_t variables = get_global_size(1); //--- float grad = 0; for(int l = 0; l < levels; l++) { int total = 1 << l; grad += IsNaNOrInf(outputs_gr[RCtoFlat(var, id, variables, dimension, l)] / total, 0); } inputs_gr[RCtoFlat(var, id, variables, dimension, 0)] = IsNaNOrInf(grad, 0); } //+------------------------------------------------------------------+ //| Runs the gru kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void GRU(__global const float* XH, __global const float* prev_state, __global float* outputs ) { const size_t id = get_global_id(0); const size_t d = get_global_id(1); const size_t units = get_global_size(0); const size_t dimension = get_global_size(1); //--- const float xz = IsNaNOrInf(XH[RCtoFlat(0, d, 6, dimension, id)], 0); const float xr = IsNaNOrInf(XH[RCtoFlat(1, d, 6, dimension, id)], 0); const float xh = IsNaNOrInf(XH[RCtoFlat(2, d, 6, dimension, id)], 0); const float hz = IsNaNOrInf(XH[RCtoFlat(3, d, 6, dimension, id)], 0); const float hr = IsNaNOrInf(XH[RCtoFlat(4, d, 6, dimension, id)], 0); const float hh = IsNaNOrInf(XH[RCtoFlat(5, d, 6, dimension, id)], 0); const float prev = IsNaNOrInf(prev_state[RCtoFlat(id, d, units, dimension, 0)], 0); //--- float r = fActivation(xr + hr, ActFunc_SIGMOID); float z = fActivation(xz + hz, ActFunc_SIGMOID); float ht = fActivation(r * hh + xh, ActFunc_TANH); float out = (1 - z) * prev + z * ht; //--- outputs[RCtoFlat(id, d, units, dimension, 0)] = IsNaNOrInf(out, 0); } //+------------------------------------------------------------------+ //| Computes the gru grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void GRU_Grad(__global const float* XH, __global float * XH_gr, __global const float* prev_state, __global const float* outputs_gr ) { const size_t id = get_global_id(0); const size_t d = get_global_id(1); const size_t units = get_global_size(0); const size_t dimension = get_global_size(1); //--- const float xz = IsNaNOrInf(XH[RCtoFlat(0, d, 6, dimension, id)], 0); const float xr = IsNaNOrInf(XH[RCtoFlat(1, d, 6, dimension, id)], 0); const float xh = IsNaNOrInf(XH[RCtoFlat(2, d, 6, dimension, id)], 0); const float hz = IsNaNOrInf(XH[RCtoFlat(3, d, 6, dimension, id)], 0); const float hr = IsNaNOrInf(XH[RCtoFlat(4, d, 6, dimension, id)], 0); const float hh = IsNaNOrInf(XH[RCtoFlat(5, d, 6, dimension, id)], 0); const float prev = IsNaNOrInf(prev_state[RCtoFlat(id, d, units, dimension, 0)], 0); const float grad = IsNaNOrInf(outputs_gr[RCtoFlat(id, d, units, dimension, 0)], 0); //--- float r = fActivation(xr + hr, ActFunc_SIGMOID); float z = fActivation(xz + hz, ActFunc_SIGMOID); float ht = fActivation(r * hh + xh, ActFunc_TANH); //--- float ht_grad = IsNaNOrInf(grad * z, 0); float z_grad = IsNaNOrInf(grad * (ht - prev), 0); float xh_grad = Deactivation(ht_grad, ht, ActFunc_TANH); float hh_grad = IsNaNOrInf(xh_grad * r, 0); float r_grad = IsNaNOrInf(xh_grad * hh, 0); float xz_grad = Deactivation(z_grad, z, ActFunc_SIGMOID); float hz_grad = xz_grad; float xr_grad = Deactivation(r_grad, r, ActFunc_SIGMOID); float hr_grad = xr_grad; //--- XH_gr[RCtoFlat(0, d, 6, dimension, id)] = IsNaNOrInf(xz_grad, 0); XH_gr[RCtoFlat(1, d, 6, dimension, id)] = IsNaNOrInf(xr_grad, 0); XH_gr[RCtoFlat(2, d, 6, dimension, id)] = IsNaNOrInf(xh_grad, 0); XH_gr[RCtoFlat(3, d, 6, dimension, id)] = IsNaNOrInf(hz_grad, 0); XH_gr[RCtoFlat(4, d, 6, dimension, id)] = IsNaNOrInf(hr_grad, 0); XH_gr[RCtoFlat(5, d, 6, dimension, id)] = IsNaNOrInf(hh_grad, 0); } //+------------------------------------------------------------------+ //| Implements the scalar to vector kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void ScalarToVector(__global const float* scalar, __global const float* vector_in, __global float* vector_out ) { const size_t vec = get_global_id(0); const size_t d = get_global_id(1); const size_t vectors = get_global_size(0); const size_t dimension = get_global_size(1); //--- float sc = IsNaNOrInf(scalar[vec], 0.0f); int shift = RCtoFlat(vec, d, vectors, dimension, 0); float v = IsNaNOrInf(vector_in[shift], 0.0f); //--- vector_out[shift] = IsNaNOrInf(sc * v, 0.0f); } //+------------------------------------------------------------------+ //| Computes the scalar to vector grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void ScalarToVectorGrad(__global const float* scalar, __global float* scalar_gr, __global const float* vector_in, __global float* vector_in_gr, __global float* vector_out_gr, const int dimension ) { const size_t vec = get_global_id(0); const size_t loc = get_local_id(1); const size_t vectors = get_global_size(0); const size_t total_loc = get_local_size(1); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- if(loc == 0) temp[0] = IsNaNOrInf(scalar[vec], 0.0f); BarrierLoc float sc = temp[0]; float sc_gr = 0; for(int d = loc; d < dimension; d += total_loc) { int shift = RCtoFlat(vec, d, vectors, dimension, 0); float v = IsNaNOrInf(vector_in[shift], 0.0f); float grad = IsNaNOrInf(vector_out_gr[shift], 0.0f); vector_in_gr[shift] = IsNaNOrInf(grad * sc, 0.0f); sc_gr += IsNaNOrInf(v * grad, 0.0f); } //--- sc_gr = LocalSum(sc_gr, 1, temp); if(loc == 0) scalar_gr[vec] = sc_gr; } //+------------------------------------------------------------------+ //| Calculates calc flow on the GPU. | //+------------------------------------------------------------------+ __kernel void CalcFlow(__global const float* value, __global float* prev_value, __global float* flow ) { const size_t id = get_global_id(0); const size_t total = get_global_size(0); //--- const float v = IsNaNOrInf(value[id], 0); const float p = IsNaNOrInf(prev_value[id], 0); flow[id] = IsNaNOrInf(v - p, 0); prev_value[id] = v; } //+------------------------------------------------------------------+ //| Implements the dilated correlation kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DilatedCorrelation(__global const float* feature, __global const int* shifts, __global float* correlations, const int dimension ) { const size_t main = get_global_id(0); const size_t loc = get_local_id(1); const size_t sh = get_global_id(2); const size_t units = get_global_size(0); const size_t total_loc = get_local_size(1); const size_t total_corr = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- const int slave = main + shifts[sh >> 1] * ((sh & 1) ? -1 : 1); if(slave < 0 || slave >= units) { if(loc == 0) correlations[RCtoFlat(main, sh, units, total_corr, 0)] = 0; return; } //--- float result = 0.0f; for(int d = loc; d < dimension; d += total_loc) { float value_main = IsNaNOrInf(feature[RCtoFlat(main, d, units, dimension, 0)], 0); float value_slave = IsNaNOrInf(feature[RCtoFlat(slave, d, units, dimension, 0)], 0); result += IsNaNOrInf(value_main * value_slave, 0); } result = LocalSum(result, 1, temp); //--- if(loc == 0) correlations[RCtoFlat(main, sh, units, total_corr, 0)] = result; } //+------------------------------------------------------------------+ //| Computes the dilated correlation grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void DilatedCorrelationGrad(__global const float* feature, __global float* feature_gr, __global const int* shifts, __global const float* corr_gr, const int total_corr ) { const size_t id = get_global_id(0); const size_t loc = get_local_id(1); const size_t d = get_global_id(2); const size_t units = get_global_size(0); const size_t total_loc = get_local_size(1); const size_t dimension = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- float result = 0.0f; for(int sh = loc; sh < total_corr; sh += total_loc) { const int offset = shifts[sh >> 1]; const int sign = (sh & 1) ? -1 : +1; // id — main int slave = id + sign * offset; if(slave >= 0 && slave < units) { float g = corr_gr[RCtoFlat(id, sh, units, total_corr, 0)]; result += IsNaNOrInf(g * feature[RCtoFlat(slave, d, units, dimension, 0)], 0.0f); } // id — slave int main = id - sign * offset; if(main >= 0 && main < units) { float g = corr_gr[RCtoFlat(main, sh, units, total_corr, 0)]; result += IsNaNOrInf(g * feature[RCtoFlat(main, d, units, dimension, 0)], 0.0f); } } result = LocalSum(result, 1, temp); if(loc == 0) feature_gr[RCtoFlat(id, d, units, dimension, 0)] = result; } //+------------------------------------------------------------------+ //| Implements the dilated difference kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void DilatedDifference(__global const float* feature, __global const int* shifts, __global float* differences ) { const size_t main = get_global_id(0); const size_t sh = get_global_id(1); const size_t d = get_global_id(2); const size_t units = get_global_size(0); const size_t total_shifts = get_global_size(1); const size_t dimension = get_global_size(2); //--- const int slave = main + shifts[sh]; if(slave < 0 || slave >= units) { differences[RCtoFlat(main, d, units, dimension, sh)] = 0; return; } //--- float value_main = IsNaNOrInf(feature[RCtoFlat(main, d, units, dimension, 0)], 0); float value_slave = IsNaNOrInf(feature[RCtoFlat(slave, d, units, dimension, 0)], 0); float result = value_main - value_slave; //--- differences[RCtoFlat(main, d, units, dimension, sh)] = result; } //+------------------------------------------------------------------+ //| Computes the dilated difference grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void DilatedDifferenceGrad(__global const float* feature, __global float* feature_gr, __global const int* shifts, __global const float* differences_gr, const int total_shifts ) { const size_t id = get_global_id(0); const size_t loc = get_local_id(1); const size_t d = get_global_id(2); const size_t units = get_global_size(0); const size_t total_loc = get_local_size(1); const size_t dimension = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- float result = 0.0f; for(int sh = loc; sh < total_shifts; sh += total_loc) { const int offset = shifts[sh]; int slave = id + offset; if(slave >= 0 && slave < units) { // id — main result += IsNaNOrInf(differences_gr[RCtoFlat(id, d, units, dimension, sh)], 0.0f); // id — slave result -= IsNaNOrInf(differences_gr[RCtoFlat(slave, d, units, dimension, sh)], 0.0f); } } result = LocalSum(result, 1, temp); if(loc == 0) feature_gr[RCtoFlat(id, d, units, dimension, 0)] = result; } //+------------------------------------------------------------------+ //| Implements the perturbed matrix kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void PerturbedMatrix(__global const float* inputs, __global const float* perturb, __global float* output, const float perturb_mult) { const size_t id = get_global_id(0); const size_t var = get_global_id(1); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); //--- int shift = RCtoFlat(var, id, variables, total, 0); output[shift] = IsNaNOrInf(inputs[shift] + perturb[shift] * perturb_mult, 0.0f); } //+------------------------------------------------------------------+ //| Computes the perturbed matrix grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void PerturbedMatrixGrad(__global float* inputs_gr, __global float* perturb_gr, __global const float* output_gr, const float perturb_mult) { const size_t id = get_global_id(0); const size_t var = get_global_id(1); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); //--- int shift = RCtoFlat(var, id, variables, total, 0); float grad = IsNaNOrInf(output_gr[shift], 0.0f); inputs_gr[shift] = grad; perturb_gr[shift] = IsNaNOrInf(perturb_mult * grad, 0.0f); } //+------------------------------------------------------------------+ //| Implements the linear upsample kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void LinearUpsample(__global const float* data, __global float* upsample) { const size_t id_ltr = get_global_id(0); const size_t var = get_global_id(1); const size_t id_htr = get_global_id(2); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); const size_t dimension_htr = get_global_size(2); //--- const float ltr = IsNaNOrInf(data[RCtoFlat(id_ltr, var, total, variables, 0)], 0.0f); const float prev_ltr = (id_ltr > 0 ? IsNaNOrInf(data[RCtoFlat(id_ltr - 1, var, total, variables, 0)], 0.0f) : 0.0f); const float htr = (id_htr < (dimension_htr - 1) ? (float)id_htr / (float)(dimension_htr - 1) * (ltr - prev_ltr) + prev_ltr : ltr); //--- upsample[RCtoFlat(id_htr, var, dimension_htr, variables, id_ltr)] = IsNaNOrInf(htr, 0.0f); } //+------------------------------------------------------------------+ //| Computes the linear upsample grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void LinearUpsampleGrad(__global float* data_gr, __global const float* upsample_gr, const int dimension_htr) { const size_t id_ltr = get_global_id(0); const size_t var = get_global_id(1); const size_t id_loc = get_local_id(2); const size_t total = get_global_size(0); const size_t variables = get_global_size(1); const size_t total_loc = get_local_size(2); float grad = 0.0f; //--- __local float temp[LOCAL_ARRAY_SIZE]; // --- main ltr { for(int id_htr = id_loc; id_htr < dimension_htr; id_htr += total_loc) { const float g = upsample_gr[RCtoFlat(id_htr, var, dimension_htr, variables, id_ltr)]; if(id_htr < dimension_htr - 1) { const float t = (float)id_htr / (float)(dimension_htr - 1); grad += g * t; } else grad += g; } } // --- prev ltr if(id_ltr + 1 < total) for(int id_htr = id_loc; id_htr < dimension_htr; id_htr += total_loc) if(id_htr < dimension_htr - 1) { const float g = upsample_gr[RCtoFlat(id_htr, var, dimension_htr, variables, id_ltr + 1)]; const float t = (float)id_htr / (float)(dimension_htr - 1); grad += g * (1.0f - t); } // --- grad = LocalSum(grad, 2, temp); if(id_loc == 0) data_gr[RCtoFlat(id_ltr, var, total, variables, 0)] = grad; } //+------------------------------------------------------------------+ //| Implements the mix experts predict kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MixExpertsPredict(__global const float4* __attribute__((aligned(16))) experts, __global float* outputs ) { const size_t id = get_global_id(0); //--- float4 expert = experts[id]; float mu = IsNaNOrInf(expert.s0, 0.0f); float alpha = fActivation(expert.s1, ActFunc_SoftPlus); float sigma = fActivation(expert.s2, ActFunc_SoftPlus); float txi = fActivation(expert.s3, ActFunc_TANH); float out = mu + alpha * sigma * txi; //--- outputs[id] = IsNaNOrInf(out, 0.0f); } //+------------------------------------------------------------------+ //| Computes the mix experts predict grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void MixExpertsPredictGrad(__global const float4* __attribute__((aligned(16))) experts, __global float4* __attribute__((aligned(16))) experts_gr, __global const float* outputs_gr ) { const size_t id = get_global_id(0); //--- float4 expert = experts[id]; float grad = IsNaNOrInf(outputs_gr[id], 0.0f); float4 expert_gr = (float4)0.0f; //--- float alpha = fActivation(expert.s1, ActFunc_SoftPlus); float sigma = fActivation(expert.s2, ActFunc_SoftPlus); float txi = fActivation(expert.s3, ActFunc_TANH); //--- float mu_grad = grad; float alpha_grad = Deactivation(grad * sigma * txi, alpha, ActFunc_SoftPlus); float sigma_grad = Deactivation(grad * alpha * txi, sigma, ActFunc_SoftPlus); float txi_grad = Deactivation(grad * sigma * alpha, txi, ActFunc_TANH); //--- experts_gr[id] = (float4)(mu_grad, alpha_grad, sigma_grad, txi_grad); } //+------------------------------------------------------------------+ //| Implements the mhfat kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHFAT(__global const float *q, __global const float *kv, __global const float *scale, __global float *scores, __global float *out, const int dimension, const int mask_future ) { //--- init const int q_id = get_global_id(0); const int k_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_k = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Shifts const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 2 * k_id); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 2 * k_id + 1); const int shift_s = RCtoFlat(h_id, k_id, total_heads, total_k, q_id); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, q_id); //--- Score float score = 0; if(mask_future == 0 || q_id <= k_id) { float sc = IsNaNOrInf(scale[shift_s], 0.0f); if(sc != 0) { for(int d = 0; d < dimension; d++) { float q_ = IsNaNOrInf(q[shift_q + d], 0.0f); if(q_ == 0) continue; float k = IsNaNOrInf(kv[shift_k + d], 0.0f); if(k == 0) continue; score += q_ * k; } score *= sc; } else score = MIN_VALUE; } else score = MIN_VALUE; //--- norm score score = LocalSoftMax(score, 1, temp); scores[shift_s] = score; //--- out for(int d = 0; d < dimension; d++) { float val = 0; if(score > 0) { float v = IsNaNOrInf(kv[shift_v + d], 0); if(v != 0) val = v * score; } val = LocalSum(val, 1, temp); if(k_id == 0) out[shift_out + d] = val; } } //+------------------------------------------------------------------+ //| Computes the mhfat grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void MHFATGrad(__global const float *q, __global float *q_gr, __global const float *kv, __global float *kv_gr, __global const float *scale, __global float *scale_gr, __global const float *scores, __global const float *gradients, const int dimension, const int total_k, const int mask_future ) { //--- init const int global_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_global = get_global_size(0); const int total_local = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- Value Gradient global_id -> v_id, local_id -> q_id for(int d = 0; d < dimension; d++) { for(int v_id = global_id; v_id < total_k; v_id += total_global) { float grad = 0; //--- Shifts const int shift_v = RCtoFlat(h_id, d, total_heads, dimension, 2 * v_id + 1); for(int q_id = 0; q_id < total_global; q_id += total_local) { const int shift_s = RCtoFlat(h_id, v_id, total_heads, total_k, q_id + local_id); const int shift_out = RCtoFlat(h_id, d, total_heads, dimension, q_id + local_id); if((q_id + local_id) < total_global) if(mask_future == 0 || (q_id + local_id) <= v_id) { float score = IsNaNOrInf(scores[shift_s], 0.0f); if(score > 0) grad += IsNaNOrInf(score * gradients[shift_out], 0.0f); } } grad = LocalSum(grad, 1, temp); if(local_id == 0) kv_gr[shift_v] = grad; } } //--- Query Gradient global_id -> q_id, local_id -> k_id/v_id for(int d_q = 0; d_q < dimension; d_q++) { //--- Shifts const int shift_q = RCtoFlat(h_id, d_q, total_heads, dimension, global_id); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, global_id); float grad = 0; for(int id = 0; id < total_k; id += total_local) { int k_id = id + local_id; const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 2 * k_id); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 2 * k_id + 1); const int shift_s = RCtoFlat(h_id, k_id, total_heads, total_k, global_id); //--- 1. Score grad float grad_s = 0; float score = 0; float sc = 0; if(k_id < total_k) { if(mask_future == 0 || global_id <= local_id) for(int d = 0; d < dimension; d++) { float val = IsNaNOrInf(kv[shift_v + d], 0); if(val == 0.0f) continue; grad_s += IsNaNOrInf(kv[shift_v + d] * gradients[shift_out + d], 0); } score = scores[shift_s]; sc = IsNaNOrInf(scale[shift_s], 0.0f); } //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(score, grad_s, 1, temp); float grad_sc = LocalSum(score * grad_s, 1, temp); if(local_id == 0 && k_id < total_k) { if(sc != 0.0f) scale_gr[shift_s] = grad_sc * log(scores[shift_s]) / sc; else scale_gr[shift_s] = grad_sc; } grad_s *= sc; //--- 3. Query grad if(grad_s != 0.0f) if(mask_future == 0 || global_id <= k_id) { float key = IsNaNOrInf(kv[shift_k + d_q], 0.0f); if(key != 0.0f) grad += key * grad_s; } } grad = LocalSum(grad, 1, temp); if(local_id == 0) q_gr[shift_q] = grad; } //--- Key Gradient global_id -> k_id, local_id -> score_id/v_id/dimension for(int k_id = global_id; k_id < total_k; k_id += total_global) { //--- Shifts const int shift_k = RCtoFlat(h_id, 0, total_heads, dimension, 2 * k_id); const int shift_v = RCtoFlat(h_id, 0, total_heads, dimension, 2 * local_id + 1); const int shift_out = RCtoFlat(h_id, 0, total_heads, dimension, local_id); float grad = 0; for(int q_id = 0; q_id < total_global; q_id++) { //--- 1. Score grad local_id -> score_id/v_id float grad_s = 0; const int shift_s = RCtoFlat(h_id, local_id, total_heads, total_k, q_id); int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); float score = 0; float sc = 0; if(local_id < total_k) { if(mask_future == 0 || q_id <= local_id) for(int d = 0; d < dimension; d++) { float val = IsNaNOrInf(kv[shift_v + d], 0); if(val == 0) continue; grad_s += IsNaNOrInf(val * gradients[shift_q + d], 0); } score = scores[shift_s]; sc = IsNaNOrInf(scale[shift_s], 0.0f); } //--- 2. SoftMax grad grad_s = LocalSoftMaxGrad(scores[shift_s], grad_s, 1, temp); grad_s *= sc; //--- 3. Key grad local_id -> dimension if(local_id < dimension) { float query = IsNaNOrInf(q[shift_q + local_id], 0); if(query != 0) grad += IsNaNOrInf(query * grad_s, 0); } } if(local_id < dimension) kv_gr[shift_k + local_id] = IsNaNOrInf(grad, 0); } } //+------------------------------------------------------------------+ //| Implements the sparse concatenate kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SparseConcatenate(__global const float *sparse_index, __global const float *sparse_data, __global const float *full, __global float *result, const int full_rows ) { const size_t sparse_row = get_global_id(0); const size_t sparse_col = get_global_id(1); const size_t full_col = get_global_id(2); const size_t sparse_rows = get_global_size(0); const size_t sparse_cols = get_global_size(1); const size_t full_cols = get_global_size(2); //--- const int shift_sparse = RCtoFlat(sparse_row, sparse_col, sparse_rows, sparse_cols, 0); const int full_row = sparse_index[shift_sparse]; const int shift_full = RCtoFlat(full_row, full_col, full_rows, full_cols, 0); const int shift_out = RCtoFlat(sparse_col, full_col, sparse_cols, full_cols, sparse_row); //--- float res = (full_row >= 0 && full_row < full_rows ? IsNaNOrInf(sparse_data[shift_sparse] * full[shift_full], 0) : 0.0f); result[shift_out] = res; } //+------------------------------------------------------------------+ //| Computes the sparse concatenate grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void SparseConcatenateGrad(__global const float *sparse_index, __global const float *sparse_data, __global float *sparse_gr, __global const float *full, __global float *full_gr, __global const float *result_gr, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols ) { const size_t row_id = get_global_id(0); const size_t local_id = get_local_id(1); const size_t col_id = get_global_id(2); const size_t total_rows = get_global_size(0); const size_t total_local = get_local_size(1); const size_t total_cols = get_global_size(2); //--- __local float Temp[LOCAL_ARRAY_SIZE]; //--- Calce sparse gradient if(row_id < sparse_rows && col_id < sparse_cols) { float grad = 0; int shift_sparse = 0; if(local_id == 0) { shift_sparse = RCtoFlat(row_id, col_id, sparse_rows, sparse_cols, 0); Temp[0] = sparse_index[shift_sparse]; } BarrierLoc uint full_row = (uint)Temp[0]; if(full_row < (uint)full_rows) for(int i = local_id; i < full_cols; i += total_local) { int shift_out = RCtoFlat(col_id, i, sparse_cols, full_cols, row_id); int shift_full = RCtoFlat(full_row, i, full_rows, full_cols, 0); grad += IsNaNOrInf(result_gr[shift_out] * full[shift_full], 0.0f); } grad = LocalSum(grad, 1, Temp); if(local_id == 0) sparse_gr[shift_sparse] = grad; } //--- Calce full gradient if(row_id < full_rows && col_id < full_cols) { float grad = 0; for(int r = 0; r < sparse_rows; r ++) { float s = 0; for(int c = local_id; c < sparse_cols; c += total_local) { int shift_sparse = RCtoFlat(r, c, sparse_rows, sparse_cols, 0); if((uint)sparse_index[shift_sparse] == (uint)row_id) { s = sparse_data[shift_sparse]; int shift_out = RCtoFlat(c, col_id, sparse_cols, full_cols, r); grad += IsNaNOrInf(s * result_gr[shift_out], 0.0f); break; } } } grad = LocalSum(grad, 1, Temp); if(local_id == 0) { int shift_full = RCtoFlat(row_id, col_id, full_rows, full_cols, 0); full_gr[shift_full] = grad; } } } //+------------------------------------------------------------------+ //| Implements the mh flash attention kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHFlashAttention(__global const float *query, __global const float *key_value, __global float *logsumexp, __global float *output, const int dimension, const int total_kv, const int mask_future ) { //--- init const int q_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_loc = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; __local float4 temp4[LOCAL_ARRAY_SIZE]; //--- const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); float prev_max = MIN_VALUE; float sumexp = 0; float out = 0; for(int id = 0; id < total_kv; id += total_loc) { int k_id = id + local_id; const int shift_k = RCtoFlat(h_id, 0, 2 * total_heads, dimension, k_id); const int shift_v = RCtoFlat(h_id + total_heads, 0, 2 * total_heads, dimension, k_id); //--- Score float score = 0; if(k_id < total_kv && (mask_future == 0 || q_id <= k_id)) { for(int d = 0; d < dimension; d += 4) { float4 q = IsNaNOrInf4((float4)( (d < dimension ? query[shift_q + d] : 0.0f), ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k = IsNaNOrInf4((float4)( (d < dimension ? key_value[shift_k + d] : 0.0f), ((d + 1) < dimension ? key_value[shift_k + d + 1] : 0.0f), ((d + 2) < dimension ? key_value[shift_k + d + 2] : 0.0f), ((d + 3) < dimension ? key_value[shift_k + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q, k), 0.0f); } score /= sqrt((float)dimension); } else score = MIN_VALUE; //--- norm score float max = fmax(prev_max, LocalMax(score, 1, temp)); if(score > MIN_VALUE) score = exp(score - max); else score = 0.0f; if(sumexp == 0.0f) sumexp = LocalSum(score, 1, temp); else sumexp = IsNaNOrInf(exp(prev_max - max) * sumexp + LocalSum(score, 1, temp), 0.0f); for(int d = 0; d < dimension; d += 4) { float4 val = (float4)0.0f; if(score > 0.0f && k_id < total_kv) { float4 v = (float4)( (d < dimension ? key_value[shift_v + d] : 0.0f), ((d + 1) < dimension ? key_value[shift_v + d + 1] : 0.0f), ((d + 2) < dimension ? key_value[shift_v + d + 2] : 0.0f), ((d + 3) < dimension ? key_value[shift_v + d + 3] : 0.0f) ); val = IsNaNOrInf4(v * score, 0.0f); } val = LocalSum4(val, 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) { if(out != 0.0f) out = IsNaNOrInf(exp(prev_max - max) * out + vget4(val, idx), 0.0f); else out = vget4(val, idx); } } prev_max = max; } if(local_id < dimension) { if(sumexp > 0.0f) output[shift_q + local_id] = IsNaNOrInf(out / sumexp, 0.0f); else output[shift_q + local_id] = 0.0f; } if(local_id == 0) { int shift_logse = RCtoFlat(q_id, h_id, total_q, total_heads, 0); if(sumexp > 0.0f) logsumexp[shift_logse] = IsNaNOrInf(prev_max + log(sumexp), 0.0f); else logsumexp[shift_logse] = 0.0f; } } //+------------------------------------------------------------------+ //| Computes the mh flash attention grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void MHFlashAttentionGrad(__global const float *query, __global float *query_gr, __global const float *key_value, __global float *key_value_gr, __global const float *logsumexp, __global const float *output, __global const float *output_gr, const int dimension, const int total_q, const int total_kv, const int mask_future ) { const int id = get_global_id(0); const int d_id = get_local_id(1); const int h_id = get_global_id(2); const int total_heads = get_global_size(2); __local float temp[LOCAL_ARRAY_SIZE]; //--- Query gradient: dQ[q, d] if(id < total_q) { const int q_id = id; const int shift_q = RCtoFlat(h_id, d_id, total_heads, dimension, q_id); const int shift_lse = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float lse = IsNaNOrInf(logsumexp[shift_lse], 0.0f); const float q_d = IsNaNOrInf(query[shift_q], 0.0f); const float g_d = IsNaNOrInf(output_gr[shift_q], 0.0f); const float o_d = IsNaNOrInf(output[shift_q], 0.0f); const float D = LocalSum(IsNaNOrInf(g_d * o_d, 0.0f), 1, temp); float grad_q = 0.0f; for(int k_id = 0; k_id < total_kv; k_id++) { if(mask_future != 0 && q_id > k_id) continue; const int shift_k = RCtoFlat(h_id, d_id, 2 * total_heads, dimension, k_id); const int shift_v = RCtoFlat(h_id + total_heads, d_id, 2 * total_heads, dimension, k_id); const float k_d = IsNaNOrInf(key_value[shift_k], 0.0f); const float v_d = IsNaNOrInf(key_value[shift_v], 0.0f); const float s = LocalSum(IsNaNOrInf(q_d * k_d, 0.0f), 1, temp) / sqrt((float)dimension); const float p = IsNaNOrInf(exp(clamp(s - lse, -120.0f, 0.0f)), 0.0f); if(p == 0.0f) continue; const float dp = LocalSum(IsNaNOrInf(g_d * v_d, 0.0f), 1, temp); const float ds = IsNaNOrInf(p * (dp - D), 0.0f); grad_q += IsNaNOrInf(k_d * ds, 0.0f); } query_gr[shift_q] = IsNaNOrInf(grad_q, 0.0f); } //--- Key & Value gradients: dK[k, d], dV[k, d] if(id < total_kv) { const int k_id = id; const int shift_k = RCtoFlat(h_id, d_id, 2 * total_heads, dimension, k_id); const int shift_v = RCtoFlat(h_id + total_heads, d_id, 2 * total_heads, dimension, k_id); const float k_d = IsNaNOrInf(key_value[shift_k], 0.0f); const float v_d = IsNaNOrInf(key_value[shift_v], 0.0f); float grad_k = 0.0f; float grad_v = 0.0f; for(int q_id = 0; q_id < total_q; q_id++) { if(mask_future != 0 && q_id > k_id) continue; const int shift_q = RCtoFlat(h_id, d_id, total_heads, dimension, q_id); const int shift_lse = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float lse = IsNaNOrInf(logsumexp[shift_lse], 0.0f); const float q_d = IsNaNOrInf(query[shift_q], 0.0f); const float g_d = IsNaNOrInf(output_gr[shift_q], 0.0f); const float o_d = IsNaNOrInf(output[shift_q], 0.0f); const float D = LocalSum(IsNaNOrInf(g_d * o_d, 0.0f), 1, temp); const float s = LocalSum(IsNaNOrInf(q_d * k_d, 0.0f), 1, temp) / sqrt((float)dimension); const float p = IsNaNOrInf(exp(clamp(s - lse, -120.0f, 0.0f)), 0.0f); if(p == 0.0f) continue; const float dp = LocalSum(IsNaNOrInf(g_d * v_d, 0.0f), 1, temp); const float ds = IsNaNOrInf(p * (dp - D), 0.0f); grad_k += IsNaNOrInf(q_d * ds, 0.0f); grad_v += IsNaNOrInf(p * g_d, 0.0f); } key_value_gr[shift_k] = IsNaNOrInf(grad_k, 0.0f); key_value_gr[shift_v] = IsNaNOrInf(grad_v, 0.0f); } } //+------------------------------------------------------------------+ //| Implements the mh flash stca kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHFlashSTCA(__global const float *query, __global const float *X, __global float *logsumexp, __global float *output, const int dimension, const int total_X, const int mask_future ) { //--- init const int q_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_loc = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; __local float4 temp4[LOCAL_ARRAY_SIZE]; //--- const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); float prev_max = MIN_VALUE; float sumexp = 0; float out = 0; for(int id = 0; id < total_X; id += total_loc) { int x_id = id + local_id; const int shift_x = RCtoFlat(x_id, 0, total_X, dimension, 0); //--- Score float score = 0; if(x_id < total_X && (mask_future == 0 || q_id <= x_id)) { for(int d = 0; d < dimension; d += 4) { float4 q = IsNaNOrInf4((float4)( (d < dimension ? query[shift_q + d] : 0.0f), ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k = IsNaNOrInf4((float4)( (d < dimension ? X[shift_x + d] : 0.0f), ((d + 1) < dimension ? X[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? X[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? X[shift_x + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q, k), 0.0f); } score /= sqrt((float)dimension); } else score = MIN_VALUE; //--- norm score float max = fmax(prev_max, LocalMax(score, 1, temp)); if(score > MIN_VALUE) score = IsNaNOrInf(exp(score - max), 0.0f); else score = 0.0f; if(sumexp == 0.0f) sumexp = LocalSum(score, 1, temp); else sumexp = IsNaNOrInf(exp(prev_max - max) * sumexp, 0.0f) + LocalSum(score, 1, temp); for(int d = 0; d < dimension; d += 4) { float4 val = (float4)0.0f; if(score > 0.0f && x_id < total_X) { float4 v = IsNaNOrInf4((float4)( (d < dimension ? X[shift_x + d] : 0.0f), ((d + 1) < dimension ? X[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? X[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? X[shift_x + d + 3] : 0.0f) ), 0.0f); val = IsNaNOrInf4(v * score, 0.0f); } val = LocalSum4(val, 1, temp4); float add = 0.0f; int idx = local_id - d; if(idx >= 0 && idx < 4) { if(out != 0.0f) out = IsNaNOrInf(exp(prev_max - max) * out + vget4(val, idx), 0.0f); else out = vget4(val, idx); } } prev_max = max; } if(local_id < dimension) { if(sumexp > 0.0f) output[shift_q + local_id] = IsNaNOrInf(out / sumexp, 0.0f); else output[shift_q + local_id] = 0.0f; } if(local_id == 0) { int shift_logse = RCtoFlat(q_id, h_id, total_q, total_heads, 0); if(sumexp > 0.0f) logsumexp[shift_logse] = IsNaNOrInf(prev_max + log(sumexp), 0.0f); else logsumexp[shift_logse] = 0.0f; } } //+------------------------------------------------------------------+ //| Computes the mh flash stca grad gradient contributions on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void MHFlashSTCAGrad(__global const float *query, __global float *query_gr, __global const float *X, __global float *X_gr, __global const float *logsumexp, __global const float *output, __global const float *output_gr, const int dimension, const int total_q, const int total_X, const int mask_future ) { const int id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_loc = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; __local float4 temp4[LOCAL_ARRAY_SIZE]; //--- Query gradient: dQ[q, d] if(id < total_q) { float grad_q = 0.0f; const int q_id = id; const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); const int shift_lse = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float lse = IsNaNOrInf(logsumexp[shift_lse], 0.0f); float D = 0; for(int d = 0; d < dimension; d += 4) { float4 g_d = IsNaNOrInf4((float4)( (d < dimension ? output_gr[shift_q + d] : 0.0f), ((d + 1) < dimension ? output_gr[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output_gr[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output_gr[shift_q + d + 3] : 0.0f) ), 0.0f); float4 o_d = IsNaNOrInf4((float4)( (d < dimension ? output[shift_q + d] : 0.0f), ((d + 1) < dimension ? output[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output[shift_q + d + 3] : 0.0f) ), 0.0f); D += IsNaNOrInf(dot(g_d, o_d), 0.0f); } for(int l_id = 0; l_id < total_X; l_id += total_loc) { int x_id = l_id + local_id; float ds = 0; if(x_id < total_X && (mask_future == 0 || q_id <= x_id)) { const int shift_x = RCtoFlat(x_id, 0, total_X, dimension, 0); float score = 0; float dp = 0; for(int d = 0; d < dimension; d += 4) { float4 q_d = IsNaNOrInf4((float4)( (d < dimension ? query[shift_q + d] : 0.0f), ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 x_d = IsNaNOrInf4((float4)( (d < dimension ? X[shift_x + d] : 0.0f), ((d + 1) < dimension ? X[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? X[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? X[shift_x + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q_d, x_d), 0.0f); float4 g_d = IsNaNOrInf4((float4)( (d < dimension ? output_gr[shift_q + d] : 0.0f), ((d + 1) < dimension ? output_gr[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output_gr[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output_gr[shift_q + d + 3] : 0.0f) ), 0.0f); dp += IsNaNOrInf(dot(g_d, x_d), 0.0f); } score /= sqrt((float)dimension); const float p = IsNaNOrInf(exp(clamp(score - lse, -120.0f, 0.0f)), 0.0f); ds = IsNaNOrInf(p * (dp - D), 0.0f); } for(int d = 0; d < dimension; d += 4) { float4 x_d = (float4)0; if(x_id < total_X && (mask_future == 0 || q_id <= x_id)) { const int shift_x = RCtoFlat(x_id, 0, total_X, dimension, 0); x_d = IsNaNOrInf4((float4)( (d < dimension ? X[shift_x + d] : 0.0f), ((d + 1) < dimension ? X[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? X[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? X[shift_x + d + 3] : 0.0f) ), 0.0f); } float4 q_dg = LocalSum4(x_d * ds, 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) grad_q += vget4(q_dg, idx); } } if(local_id < dimension) query_gr[shift_q + local_id] = IsNaNOrInf(grad_q, 0.0f); } //--- X gradients: dX[k, d] if(id < total_X && h_id == 0) { float grad_X = 0.0f; const int x_id = id; const int shift_x = RCtoFlat(x_id, 0, total_X, dimension, 0); for(int l_id = 0; l_id < total_q * total_heads; l_id += total_loc) { int loc = l_id + local_id; int h = loc / total_q; int q_id = loc % total_q; float ds = 0; float p = 0; if(h < total_heads && q_id < total_q && (mask_future == 0 || q_id <= x_id)) { const int shift_lse = RCtoFlat(q_id, h, total_q, total_heads, 0); const float lse = IsNaNOrInf(logsumexp[shift_lse], 0.0f); const int shift_q = RCtoFlat(h, 0, total_heads, dimension, q_id); float score = 0; float D = 0; float dp = 0; for(int d = 0; d < dimension; d += 4) { float4 q_d = IsNaNOrInf4((float4)( (d < dimension ? query[shift_q + d] : 0.0f), ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 x_d = IsNaNOrInf4((float4)( (d < dimension ? X[shift_x + d] : 0.0f), ((d + 1) < dimension ? X[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? X[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? X[shift_x + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q_d, x_d), 0.0f); float4 g_d = IsNaNOrInf4((float4)( (d < dimension ? output_gr[shift_q + d] : 0.0f), ((d + 1) < dimension ? output_gr[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output_gr[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output_gr[shift_q + d + 3] : 0.0f) ), 0.0f); dp += IsNaNOrInf(dot(g_d, x_d), 0.0f); float4 o_d = IsNaNOrInf4((float4)( (d < dimension ? output[shift_q + d] : 0.0f), ((d + 1) < dimension ? output[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output[shift_q + d + 3] : 0.0f) ), 0.0f); D += IsNaNOrInf(dot(g_d, o_d), 0.0f); } const float p = IsNaNOrInf(exp(clamp(score - lse, -120.0f, 0.0f)), 0.0f); if(p != 0.0f) ds = IsNaNOrInf(p * (dp - D), 0.0f); } //--- for(int d = 0; d < dimension; d += 4) { float4 q_d = (float4)0; float4 g_d = (float4)0; if(h < total_heads && q_id < total_q && (mask_future == 0 || q_id <= x_id)) { const int shift_q = RCtoFlat(h, 0, total_heads, dimension, q_id); q_d = IsNaNOrInf4((float4)( (d < dimension ? query[shift_q + d] : 0.0f), ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); g_d = IsNaNOrInf4((float4)( (d < dimension ? output_gr[shift_q + d] : 0.0f), ((d + 1) < dimension ? output_gr[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output_gr[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output_gr[shift_q + d + 3] : 0.0f) ), 0.0f); } float4 x_dg = LocalSum4(q_d * ds + g_d * p, 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) grad_X += vget4(x_dg, idx); } } if(local_id < dimension) X_gr[shift_x + local_id] = IsNaNOrInf(grad_X, 0.0f); } } //+------------------------------------------------------------------+ //| Performs the concat vec matrix data-layout operation on the GPU. | //+------------------------------------------------------------------+ __kernel void ConcatVecMatrix(__global const float *vector_in, __global const float *matrix_in, __global float *matrix_out, const int dimension_v, const int colums_m, const int multvarsecond ) { const int r = get_global_id(0); const int c = get_global_id(1); const int v = get_global_id(2); const int rows = get_global_size(0); const int cols = get_global_size(1); const int variables = get_global_size(2); //--- if(c < dimension_v) { int flat_v = RCtoFlat(0, c, 1, dimension_v, v); int flat_out = RCtoFlat(r, c, rows, dimension_v + colums_m, v); matrix_out[flat_out] = IsNaNOrInf(vector_in[flat_v], 0.0f); } if(c < colums_m) { int flat_m = RCtoFlat(r, c, rows, colums_m, multvarsecond * v); int flat_out = RCtoFlat(r, c + dimension_v, rows, dimension_v + colums_m, v); matrix_out[flat_out] = IsNaNOrInf(matrix_in[flat_m], 0.0f); } } //+------------------------------------------------------------------+ //| Computes the concat vec matrix grad gradient contributions on the| //| GPU. | //+------------------------------------------------------------------+ __kernel void ConcatVecMatrixGrad(__global float *vector_in_gr, __global float *matrix_in_gr, __global const float *matrix_out_gr, const int dimension_v, const int colums_m, const int rows, const int variables, const int multvarsecond ) { const int loc_id = get_local_id(0); const int c = get_global_id(1); const int glob_id = get_global_id(2); const int total_loc = get_local_size(0); const int cols = get_global_size(1); const int total_glob = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; //--- float vec_grad = 0; for(int i = 0; (i < rows && glob_id < variables); i += total_loc) { int r = i + loc_id; if(r >= rows) continue; if(c < dimension_v) { int flat_out = RCtoFlat(r, c, rows, dimension_v + colums_m, glob_id); vec_grad += IsNaNOrInf(matrix_out_gr[flat_out], 0.0f); } if(c < colums_m) { int flat_m = RCtoFlat(r, c, rows, colums_m, glob_id); int flat_out = RCtoFlat(r, c + dimension_v, rows, dimension_v + colums_m, glob_id); if((bool)multvarsecond) matrix_in_gr[flat_m] = IsNaNOrInf(matrix_out_gr[flat_out], 0.0f); } } if(c < dimension_v) { vec_grad = LocalSum(vec_grad, 0, temp); if(loc_id == 0) { int flat_v = RCtoFlat(0, c, 1, dimension_v, glob_id); vector_in_gr[flat_v] = vec_grad; } } if(!((bool)multvarsecond) && c < colums_m && glob_id < rows) { float mat_grad = 0; for(int i = 0; i < variables; i += total_loc) { int v = i + loc_id; if(v >= variables) continue; int flat_out = RCtoFlat(glob_id, c + dimension_v, rows, dimension_v + colums_m, v); mat_grad += IsNaNOrInf(matrix_out_gr[flat_out], 0.0f); } mat_grad = LocalSum(mat_grad, 0, temp); if(loc_id == 0) { int flat_m = RCtoFlat(glob_id, c, rows, colums_m, 0); matrix_in_gr[flat_m] = mat_grad; } } } //+------------------------------------------------------------------+ //| Implements the inf net bgu kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void INFNetBGU(__global const float2* scal_and_shift, __global const float* sequence, __global const float* context, __global const float* scenarios, __global float* sequence_out, __global float* context_out, __global float* scenarios_out, const int sequence_units, const int sequence_vars, const int context_units, const int scenarios_units ) { const int id_n = get_global_id(0); const int d = get_global_id(1); const int v = get_global_id(2); const int total_n = get_global_size(0); const int dimension = get_global_size(1); const int variables = get_global_size(2); __global const float* inputs = sequence; __global float* outputs = sequence_out; int units = 0; int var = 0; float2 ss = 0; if(v < sequence_vars) { units = sequence_units; var = v; ss = scal_and_shift[RCtoFlat(v, d, sequence_vars + scenarios_units + 1, dimension, 0)]; } else if(v == sequence_vars) { inputs = context; outputs = context_out; units = context_units; if(id_n < units) ss = scal_and_shift[RCtoFlat(v, d, sequence_vars + scenarios_units + 1, dimension, 0)]; } else { inputs = scenarios; outputs = scenarios_out; units = scenarios_units; if(id_n < units) ss = scal_and_shift[RCtoFlat(sequence_vars + 1 + id_n, d, sequence_vars + scenarios_units + 1, dimension, 0)]; } //--- ss = IsNaNOrInf2(ss, (float2)0); float sig_a = fActivation(ss.s0, ActFunc_SIGMOID); for(int i = id_n; i < units; i += total_n) { int shift = RCtoFlat(i, d, units, dimension, var); float val = IsNaNOrInf(inputs[shift], 0); val = IsNaNOrInf(val * (1 + sig_a) + ss.s1, 0); outputs[shift] = val; } } //+------------------------------------------------------------------+ //| Computes the inf net bgu grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void INFNetBGUGrad(__global const float2* scal_and_shift, __global float2* scal_and_shift_grad, __global const float* sequence, __global const float* context, __global const float* scenarios, __global float* sequence_grad, __global float* context_grad, __global float* scenarios_grad, __global const float* sequence_out_grad, __global const float* context_out_grad, __global const float* scenarios_out_grad, const int sequence_units, const int sequence_vars, const int context_units, const int scenarios_units ) { const int id_n = get_local_id(0); const int d = get_global_id(1); const int v = get_global_id(2); const int total_n = get_local_size(0); const int dimension = get_global_size(1); //--- __local float4 Temp[LOCAL_ARRAY_SIZE]; //--- __global const float* inputs = sequence; __global float* grad_in = sequence_grad; __global const float* grad_out = sequence_out_grad; int units = 0; int var = 0; int ss_idx = 0; float2 ss = (float2)0; bool scenarios_mode = false; const int ss_rows = sequence_vars + scenarios_units + 1; if(v < sequence_vars) { units = sequence_units; var = v; ss_idx = RCtoFlat(v, d, ss_rows, dimension, 0); ss = IsNaNOrInf2(scal_and_shift[ss_idx], (float2)0); } else if(v == sequence_vars) { inputs = context; grad_in = context_grad; grad_out = context_out_grad; units = context_units; if(id_n < units) { ss_idx = RCtoFlat(v, d, ss_rows, dimension, 0); ss = IsNaNOrInf2(scal_and_shift[ss_idx], (float2)0); } } else { inputs = scenarios; grad_in = scenarios_grad; grad_out = scenarios_out_grad; units = scenarios_units; scenarios_mode = true; if(id_n < units) { ss_idx = RCtoFlat(sequence_vars + 1 + id_n, d, ss_rows, dimension, 0); ss = IsNaNOrInf2(scal_and_shift[ss_idx], (float2)0); } } const float sig_a = fActivation(ss.s0, ActFunc_SIGMOID); float d_alpha_acc = 0.0f; float d_beta_acc = 0.0f; for(int i = id_n; i < units; i += total_n) { int shift = RCtoFlat(i, d, units, dimension, var); float x = IsNaNOrInf(inputs[shift], 0.0f); float dy = IsNaNOrInf(grad_out[shift], 0.0f); grad_in[shift] = IsNaNOrInf(dy * (1 + sig_a), 0.0f); d_alpha_acc += dy * x; d_beta_acc += dy; } if(!scenarios_mode) { float4 sums = LocalSum4((float4)(d_alpha_acc, d_beta_acc, 0.0f, 0.0f), 0, Temp); if(get_local_id(0) == 0) { sums.s0 = Deactivation(sums.s0, sig_a, ActFunc_SIGMOID); scal_and_shift_grad[ss_idx] = sums.lo; } } else if(id_n < units) scal_and_shift_grad[ss_idx] = (float2)(Deactivation(d_alpha_acc, sig_a, ActFunc_SIGMOID), IsNaNOrInf(d_beta_acc, 0.0f)); } //+------------------------------------------------------------------+ //| Implements the mh cross attvs sim kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MHCrossAttvsSim(__global const float *query, __global const float *key, __global const float *value, __global const float *prototype, __global const float *levels, __global float *logsumexp, __global float *output_at, __global float *similarity, const int dimension, const int total_X) { const int q_id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_loc = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; __local float4 temp4[LOCAL_ARRAY_SIZE]; //--- const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); const int logse_shift = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float threshold = levels[logse_shift]; /* Loop 1: compute logsumexp via online softmax (no masking) */ float prev_max = MIN_VALUE; float sumexp = 0.0f; for(int id = 0; id < total_X; id += total_loc) { int x_id = id + local_id; const int shift_x = RCtoFlat(h_id, 0, total_heads, dimension, x_id); float score = 0.0f; if(x_id < total_X) { for(int d = 0; d < dimension; d += 4) { float4 q = IsNaNOrInf4((float4)( query[shift_q + d], ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k = IsNaNOrInf4((float4)( key[shift_x + d], ((d + 1) < dimension ? key[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? key[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? key[shift_x + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q, k), 0.0f); } score /= sqrt((float)dimension); } else score = MIN_VALUE; float max = fmax(prev_max, LocalMax(score, 1, temp)); score = (score > MIN_VALUE) ? IsNaNOrInf(exp(score - max), 0.0f) : 0.0f; if(sumexp == 0.0f) sumexp = LocalSum(score, 1, temp); else sumexp = IsNaNOrInf(exp(prev_max - max) * sumexp, 0.0f) + LocalSum(score, 1, temp); prev_max = max; } //--- Write logsumexp const float lse = (sumexp > 0.0f) ? IsNaNOrInf(prev_max + log(sumexp), 0.0f) : 0.0f; if(local_id == 0) logsumexp[logse_shift] = lse; /* Loop 2: accumulate value and similarity score = exp(score - lse); skip if score < threshold score are already normalized — no division by sumexp needed */ float out_at = 0.0f; float out_sim = 0.0f; for(int id = 0; id < total_X; id += total_loc) { int x_id = id + local_id; const int shift_x = RCtoFlat(h_id, 0, total_heads, dimension, x_id); float score = 0.0f; if(x_id < total_X && sumexp > 0.0f) { for(int d = 0; d < dimension; d += 4) { float4 q = IsNaNOrInf4((float4)( query[shift_q + d], ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k = IsNaNOrInf4((float4)( key[shift_x + d], ((d + 1) < dimension ? key[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? key[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? key[shift_x + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q, k), 0.0f); } score /= sqrt((float)dimension); score = IsNaNOrInf(exp(score - lse), 0.0f); if(score < threshold) score = 0.0f; } for(int d = 0; d < dimension; d += 4) { float4 val = (float4)0.0f; float4 sim = (float4)0.0f; if(score > 0.0f) { float4 v = IsNaNOrInf4((float4)( value[shift_x + d], ((d + 1) < dimension ? value[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? value[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? value[shift_x + d + 3] : 0.0f) ), 0.0f); float4 p = IsNaNOrInf4((float4)( prototype[shift_x + d], ((d + 1) < dimension ? prototype[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? prototype[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? prototype[shift_x + d + 3] : 0.0f) ), 0.0f); val = IsNaNOrInf4(v * score, 0.0f); sim = IsNaNOrInf4(p * score, 0.0f); } val = LocalSum4(val, 1, temp4); sim = LocalSum4(sim, 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) { out_at += vget4(val, idx); out_sim += vget4(sim, idx); } } } //--- Write results (weights already normalized via lse) if(local_id < dimension) { output_at[shift_q + local_id] = IsNaNOrInf(out_at, 0.0f); similarity[shift_q + local_id] = IsNaNOrInf(out_sim, 0.0f); } } //+------------------------------------------------------------------+ //| Computes the mh cross attvs sim grad gradient contributions on | //| the GPU. | //+------------------------------------------------------------------+ __kernel void MHCrossAttvsSimGrad(__global const float *query, __global const float *key, __global const float *value, __global const float *prototype, __global const float *levels, __global const float *logsumexp, __global const float *output_at, __global const float *similarity, __global const float *grad_output_at, __global const float *grad_similarity, __global float *grad_query, __global float *grad_key, __global float *grad_value, __global float *grad_prototype, __global float *grad_levels, const int dimension, const int total_X) { const int id = get_global_id(0); const int local_id = get_local_id(1); const int h_id = get_global_id(2); const int total_q = get_global_size(0); const int total_loc = get_local_size(1); const int total_heads = get_global_size(2); //--- __local float temp[LOCAL_ARRAY_SIZE]; __local float4 temp4[LOCAL_ARRAY_SIZE]; const float inv_sqrt_d = 1.0f / sqrt((float)dimension); /* Block 1: grad_query + grad_levels global_id(0) = q_id, owns shift_q — no race condition */ { const int q_id = id; const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); const int logse_shift = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float threshold = levels[logse_shift]; const float lse = logsumexp[logse_shift]; //--- D = dot(grad_output_at, output_at) + dot(grad_similarity, similarity) //--- used in softmax gradient: grad_score = score * (dot(go,v) + dot(gs,p) - D) float D = 0.0f; for(int d = local_id; d < dimension; d += total_loc) { D += IsNaNOrInf(grad_output_at[shift_q + d] * output_at[shift_q + d], 0.0f); D += IsNaNOrInf(grad_similarity[shift_q + d] * similarity[shift_q + d], 0.0f); } D = LocalSum(D, 1, temp); //--- Iterate over x_id float out_gq = 0.0f; float grad_lev = 0.0f; for(int l_id = 0; l_id < total_X; l_id += total_loc) { int x_id = l_id + local_id; const int shift_x = RCtoFlat(h_id, 0, total_heads, dimension, x_id); float score = 0.0f; float grad_score = 0.0f; if(x_id < total_X) { float score = 0.0f; float dot_go_v = 0.0f; float dot_gs_p = 0.0f; for(int d = 0; d < dimension; d += 4) { float4 q_d = IsNaNOrInf4((float4)( query[shift_q + d], ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k_d = IsNaNOrInf4((float4)( key[shift_x + d], ((d + 1) < dimension ? key[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? key[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? key[shift_x + d + 3] : 0.0f) ), 0.0f); float4 v_d = IsNaNOrInf4((float4)( value[shift_x + d], ((d + 1) < dimension ? value[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? value[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? value[shift_x + d + 3] : 0.0f) ), 0.0f); float4 p_d = IsNaNOrInf4((float4)( prototype[shift_x + d], ((d + 1) < dimension ? prototype[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? prototype[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? prototype[shift_x + d + 3] : 0.0f) ), 0.0f); float4 go_d = IsNaNOrInf4((float4)( grad_output_at[shift_q + d], ((d + 1) < dimension ? grad_output_at[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? grad_output_at[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? grad_output_at[shift_q + d + 3] : 0.0f) ), 0.0f); float4 gs_d = IsNaNOrInf4((float4)( grad_similarity[shift_q + d], ((d + 1) < dimension ? grad_similarity[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? grad_similarity[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? grad_similarity[shift_q + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q_d, k_d), 0.0f); dot_go_v += IsNaNOrInf(dot(go_d, v_d), 0.0f); dot_gs_p += IsNaNOrInf(dot(gs_d, p_d), 0.0f); } score /= sqrt((float)dimension); score = IsNaNOrInf(exp(score - lse), 0.0f); grad_score = IsNaNOrInf(score * (dot_go_v + dot_gs_p - D), 0.0f); } //--- Surrogate gradient for threshold: -sign(level) * sum(grad_score) grad_lev -= grad_score; //--- grad_query += grad_score * inv_sqrt_d * key for(int d = 0; d < dimension; d += 4) { float4 k_d = (float4)0.0f; if(fabs(grad_score) > 0 && x_id < total_X) k_d = IsNaNOrInf4((float4)( key[shift_x + d], ((d + 1) < dimension ? key[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? key[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? key[shift_x + d + 3] : 0.0f) ), 0.0f); float4 gq = LocalSum4(k_d * (grad_score * inv_sqrt_d), 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) out_gq += vget4(gq, idx); } } //--- Write grad_query if(local_id < dimension) grad_query[shift_q + local_id] = IsNaNOrInf(out_gq, 0.0f); //--- Write grad_levels grad_lev = LocalSum(grad_lev, 1, temp); if(local_id == 0) { grad_levels[logse_shift] = (fabs(grad_lev) > 0.0f) ? IsNaNOrInf(grad_lev, 0.0f) : 0.0f; } } /* Block 2: grad_key + grad_value + grad_prototype global_id(0) = x_id, global_id(2) = h_id, owns shift_x iterates over q_id in local dim — no race condition */ for(int id2 = 0; id2 < total_X; id2 += total_q) { const int x_id = id2 + id; const int shift_x = RCtoFlat(h_id, 0, total_heads, dimension, x_id); float grad_k = 0.0f; float grad_v = 0.0f; float grad_p = 0.0f; for(int l_id = 0; l_id < total_q; l_id += total_loc) { int q_id = l_id + local_id; float score = 0.0f; float grad_score = 0.0f; if(q_id < total_q && x_id < total_X) { const int shift_q = RCtoFlat(h_id, 0, total_heads, dimension, q_id); const int logse_shift = RCtoFlat(q_id, h_id, total_q, total_heads, 0); const float threshold = levels[logse_shift]; const float lse = logsumexp[logse_shift]; float D = 0.0f; float dot_go_v = 0.0f; float dot_gs_p = 0.0f; for(int d = 0; d < dimension; d += 4) { float4 q_d = IsNaNOrInf4((float4)( query[shift_q + d], ((d + 1) < dimension ? query[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? query[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? query[shift_q + d + 3] : 0.0f) ), 0.0f); float4 k_d = IsNaNOrInf4((float4)( key[shift_x + d], ((d + 1) < dimension ? key[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? key[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? key[shift_x + d + 3] : 0.0f) ), 0.0f); float4 v_d = IsNaNOrInf4((float4)( value[shift_x + d], ((d + 1) < dimension ? value[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? value[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? value[shift_x + d + 3] : 0.0f) ), 0.0f); float4 p_d = IsNaNOrInf4((float4)( prototype[shift_x + d], ((d + 1) < dimension ? prototype[shift_x + d + 1] : 0.0f), ((d + 2) < dimension ? prototype[shift_x + d + 2] : 0.0f), ((d + 3) < dimension ? prototype[shift_x + d + 3] : 0.0f) ), 0.0f); float4 go_d = IsNaNOrInf4((float4)( grad_output_at[shift_q + d], ((d + 1) < dimension ? grad_output_at[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? grad_output_at[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? grad_output_at[shift_q + d + 3] : 0.0f) ), 0.0f); float4 gs_d = IsNaNOrInf4((float4)( grad_similarity[shift_q + d], ((d + 1) < dimension ? grad_similarity[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? grad_similarity[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? grad_similarity[shift_q + d + 3] : 0.0f) ), 0.0f); float4 o_d = IsNaNOrInf4((float4)( output_at[shift_q + d], ((d + 1) < dimension ? output_at[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? output_at[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? output_at[shift_q + d + 3] : 0.0f) ), 0.0f); float4 s_d = IsNaNOrInf4((float4)( similarity[shift_q + d], ((d + 1) < dimension ? similarity[shift_q + d + 1] : 0.0f), ((d + 2) < dimension ? similarity[shift_q + d + 2] : 0.0f), ((d + 3) < dimension ? similarity[shift_q + d + 3] : 0.0f) ), 0.0f); score += IsNaNOrInf(dot(q_d, k_d), 0.0f); dot_go_v += IsNaNOrInf(dot(go_d, v_d), 0.0f); dot_gs_p += IsNaNOrInf(dot(gs_d, p_d), 0.0f); D += IsNaNOrInf(dot(go_d, o_d), 0.0f); D += IsNaNOrInf(dot(gs_d, s_d), 0.0f); } score /= sqrt((float)dimension); score = IsNaNOrInf(exp(score - lse), 0.0f); if(score > 0.0f) grad_score = IsNaNOrInf(score * (dot_go_v + dot_gs_p - D), 0.0f); } //--- grad_key += grad_score * inv_sqrt_d * query //--- grad_value += score * grad_output_at //--- grad_proto += score * grad_similarity for(int d = 0; d < dimension; d += 4) { float4 q_d = (float4)0.0f; float4 go_d = (float4)0.0f; float4 gs_d = (float4)0.0f; if(q_id < total_q && x_id < total_X) { const int sq = RCtoFlat(h_id, 0, total_heads, dimension, q_id); q_d = IsNaNOrInf4((float4)( query[sq + d], ((d + 1) < dimension ? query[sq + d + 1] : 0.0f), ((d + 2) < dimension ? query[sq + d + 2] : 0.0f), ((d + 3) < dimension ? query[sq + d + 3] : 0.0f) ), 0.0f); go_d = IsNaNOrInf4((float4)( grad_output_at[sq + d], ((d + 1) < dimension ? grad_output_at[sq + d + 1] : 0.0f), ((d + 2) < dimension ? grad_output_at[sq + d + 2] : 0.0f), ((d + 3) < dimension ? grad_output_at[sq + d + 3] : 0.0f) ), 0.0f); gs_d = IsNaNOrInf4((float4)( grad_similarity[sq + d], ((d + 1) < dimension ? grad_similarity[sq + d + 1] : 0.0f), ((d + 2) < dimension ? grad_similarity[sq + d + 2] : 0.0f), ((d + 3) < dimension ? grad_similarity[sq + d + 3] : 0.0f) ), 0.0f); } float4 gk = LocalSum4(q_d * (grad_score * inv_sqrt_d), 1, temp4); float4 gv = LocalSum4(go_d * score, 1, temp4); float4 gp = LocalSum4(gs_d * score, 1, temp4); int idx = local_id - d; if(idx >= 0 && idx < 4) { grad_k += vget4(gk, idx); grad_v += vget4(gv, idx); grad_p += vget4(gp, idx); } } } //--- Write grad_key/value/prototype — single work-group owns shift_x, no conflicts if(local_id < dimension) { grad_key[shift_x + local_id] = IsNaNOrInf(grad_k, 0.0f); grad_value[shift_x + local_id] = IsNaNOrInf(grad_v, 0.0f); grad_prototype[shift_x + local_id] = IsNaNOrInf(grad_p, 0.0f); } } } //+------------------------------------------------------------------+ //| Implements the market state density kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MarketStateDensity(__global const float* sequence, __global float* distribution, const uint sequence_size, const uint quantiles, const float quant_step ) { //--- features const uint feature = get_global_id(0); const uint dimension = get_global_size(0); //--- local worker index const uint lid = get_local_id(1); //--- local size over sequence axis const uint local_n = get_local_size(1); //--- total quantiles const int bins = 2 * quantiles + 1; //--- variables const uint var = get_global_id(2); const uint variables = get_global_size(2); //--- __local float4 Temp[LOCAL_ARRAY_SIZE]; //--- current state reference if(lid == 0) Temp[0].s0 = IsNaNOrInf(sequence[RCtoFlat(0, feature, sequence_size, dimension, var)], 0.0f); BarrierLoc; const float current = Temp[0].s0; //--- iterate through sequence uint count = 0; for(uint i = 0; i < sequence_size; i += local_n) { int q = bins + 2; if(i + lid < sequence_size) { const float value = IsNaNOrInf(sequence[RCtoFlat(i + lid, feature, sequence_size, dimension, var)], 0.0f); //--- deviation from current state const float diff = current - value; //--- quantization q = (int)floor(diff / quant_step); // center shift q += (int)quantiles; // clamp q = clamp(q, 0, bins - 1); } // accumulate for(int b = 0; b < bins; b += 4) { float4 b4 = (float4)0.0f; switch(q - b) { case 0: b4.s0 = 1; break; case 1: b4.s1 = 1; break; case 2: b4.s2 = 1; break; case 3: b4.s3 = 1; break; } BarrierLoc; count += (uint)vget4(LocalSum4(b4, 1, Temp), lid - b); } } //--- normalization and save if(lid < bins) distribution[RCtoFlat(lid, feature, bins, dimension, var)] = ((float)count / sequence_size); } //+------------------------------------------------------------------+ //| Computes the nll grad gradient contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void NLLGrad(__global const float* log_b, __global float* grad_in, __global const float* grad_out, __global float* grad_log_b ) { const uint i = get_global_id(0); //--- log-scale resistance const float s = log_b[i]; //--- forecat gradient const float grad = grad_out[i]; //--- gradient const float g = IsNaNOrInf(grad * exp(-s), 0.0f); //--- to inputs grad_in[i] = g; //--- local gradient for s //--- d(g*e^-s)/ds = -g*e^-s grad_log_b[i] = -g; } //+------------------------------------------------------------------+ //| Implements the cog driver forecast shift state kernel on the GPU.| //+------------------------------------------------------------------+ __kernel void CogDriverForecastShiftState(__global const float* plan, __global const float* confidence, __global float* shifted_plan, __global float* shifted_confidence, __global float* delta_mask, const int dimension, const int horizon ) { const int id = get_global_id(0); const int total = dimension * horizon; if(id >= total || dimension <= 0 || horizon <= 0) return; const int h = id / dimension; const int d = id - h * dimension; if(h < horizon - 1) shifted_plan[id] = IsNaNOrInf(plan[(h + 1) * dimension + d], 0.0f); else shifted_plan[id] = 0.0f; if(d == 0) { if(h < horizon - 1) { shifted_confidence[h] = IsNaNOrInf(confidence[h + 1], 0.0f); delta_mask[h] = 1.0f; } else { shifted_confidence[h] = 0.0f; delta_mask[h] = 0.0f; } } } //+------------------------------------------------------------------+ //| Implements the cog driver forecast build kv kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverForecastBuildKV(__global const float* context, __global float* context_kv, const int dimension, const int horizon, const int variables, const int heads, const int head_dim ) { const int variable = get_global_id(0); const int h = get_global_id(1); const int flat_dim = get_global_id(2); if(variable >= variables || h >= horizon || flat_dim >= dimension || heads <= 0 || head_dim <= 0) return; const int head = flat_dim / head_dim; if(head >= heads) return; const int local_dim = flat_dim - head * head_dim; const int token = variable * horizon + h; const int src_base = variable * 2 * horizon * dimension + h * 2 * dimension; const float key = IsNaNOrInf(context[src_base + flat_dim], 0.0f); const float value = IsNaNOrInf(context[src_base + dimension + flat_dim], 0.0f); const int k_shift = RCtoFlat(head, local_dim, 2 * heads, head_dim, token); const int v_shift = RCtoFlat(head + heads, local_dim, 2 * heads, head_dim, token); context_kv[k_shift] = key; context_kv[v_shift] = value; } //+------------------------------------------------------------------+ //| Computes the cog driver forecast build kv grad gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverForecastBuildKVGrad(__global const float* context_kv_gr, __global float* context_gr, const int dimension, const int horizon, const int variables, const int heads, const int head_dim ) { const int variable = get_global_id(0); const int h = get_global_id(1); const int flat_dim = get_global_id(2); if(variable >= variables || h >= horizon || flat_dim >= dimension || heads <= 0 || head_dim <= 0) return; const int head = flat_dim / head_dim; if(head >= heads) return; const int local_dim = flat_dim - head * head_dim; const int token = variable * horizon + h; const int dst_base = variable * 2 * horizon * dimension + h * 2 * dimension; const int k_shift = RCtoFlat(head, local_dim, 2 * heads, head_dim, token); const int v_shift = RCtoFlat(head + heads, local_dim, 2 * heads, head_dim, token); context_gr[dst_base + flat_dim] += IsNaNOrInf(context_kv_gr[k_shift], 0.0f); context_gr[dst_base + dimension + flat_dim] += IsNaNOrInf(context_kv_gr[v_shift], 0.0f); } //+------------------------------------------------------------------+ //| Implements the cog driver forecast confidence kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverForecastConfidence(__global const float* expected_error, __global float* confidence, const int total ) { const int id = get_global_id(0); if(id >= total) return; const float error = fmax(IsNaNOrInf(expected_error[id], 0.0f), 0.0f); confidence[id] = 1.0f / (1.0f + error); } //+------------------------------------------------------------------+ //| Implements the cog driver forecast delta norm kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverForecastDeltaNorm(__global const float* plan, __global const float* shifted_plan, __global float* plan_delta, __global float* plan_delta_norm, const int dimension, const int horizon ) { const int h = get_global_id(0); if(h >= horizon || dimension <= 0) return; float sum = 0.0f; for(int d = 0; d < dimension; d++) { const int idx = h * dimension + d; const float delta = IsNaNOrInf(plan[idx] - shifted_plan[idx], 0.0f); plan_delta[idx] = delta; sum += delta * delta; } plan_delta_norm[h] = sqrt(sum / (float)dimension); } //+------------------------------------------------------------------+ //| Generic RAG retrieval pipeline; every extent is runtime supplied | //+------------------------------------------------------------------+ __kernel void RAGScore(__global const float *embedding, __global const float *centroids, __global const float *used, __global float *scores, const int scenario_count, const int embedding_dimension) { const int scenario = get_global_id(0); if(scenario >= scenario_count || embedding_dimension <= 0) return; if(used[scenario] <= 0.5f) { scores[scenario] = RAG_INVALID_SCORE; return; } float dot = 0.0f, left = 0.0f, right = 0.0f; for(int d = 0; d < embedding_dimension; d++) { float a = IsNaNOrInf(embedding[d], 0.0f); float b = IsNaNOrInf(centroids[scenario * embedding_dimension + d], 0.0f); dot += a * b; left += a * a; right += b * b; } const float score = (left > 1.0e-12f && right > 1.0e-12f ? dot / (sqrt(left) * sqrt(right)) : RAG_INVALID_SCORE); scores[scenario] = (isfinite(score) ? score : RAG_INVALID_SCORE); } //+------------------------------------------------------------------+ //| Implements the rag before helper on the GPU. | //+------------------------------------------------------------------+ inline int RAGBefore(const float2 left, const float2 right) { return (left.y > right.y || (left.y == right.y && left.x < right.x)); } //+------------------------------------------------------------------+ //| Implements the rag sort descending helper on the GPU. | //+------------------------------------------------------------------+ inline void RAGSortDescending(__local float2 candidates[], const int lane, const int block_size) { for(int width = 2; width <= block_size; width <<= 1) for(int stride = width >> 1; stride > 0; stride >>= 1) { const int partner = lane ^ stride; if(partner > lane) { const int descending = ((lane & width) == 0); const int before = RAGBefore(candidates[lane], candidates[partner]); if((descending && !before) || (!descending && before)) { const float2 swap = candidates[lane]; candidates[lane] = candidates[partner]; candidates[partner] = swap; } } BarrierLoc; } } //+------------------------------------------------------------------+ //| Implements the rag partial kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RAGPartial(__global const float *scores, __global float2 *partial, const int scenario_count, const int block_size, const int top_k) { const int lane = get_local_id(0); const int block = get_group_id(0); const int scenario = block * block_size + lane; __local float2 candidates[LOCAL_ARRAY_SIZE]; if(block_size <= 0 || block_size > LOCAL_ARRAY_SIZE || (block_size & (block_size - 1)) != 0 || top_k <= 0 || top_k > block_size) return; const float score = (scenario < scenario_count ? scores[scenario] : RAG_INVALID_SCORE); candidates[lane] = (isfinite(score) && score > RAG_INVALID_SCORE ? (float2)((float)scenario, score) : (float2)(-1.0f, RAG_INVALID_SCORE)); BarrierLoc; RAGSortDescending(candidates, lane, block_size); if(lane < top_k) partial[block * top_k + lane] = candidates[lane]; } //+------------------------------------------------------------------+ //| Implements the rag merge kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RAGMerge(__global const float2 *source, __global float2 *destination, const int candidate_count, const int block_size, const int top_k) { const int lane = get_local_id(0); const int block = get_group_id(0); const int candidate_id = block * block_size + lane; __local float2 candidates[LOCAL_ARRAY_SIZE]; if(block_size <= 0 || block_size > LOCAL_ARRAY_SIZE || (block_size & (block_size - 1)) != 0 || top_k <= 0 || top_k > block_size) return; const float2 candidate = (candidate_id < candidate_count ? source[candidate_id] : (float2)(-1.0f, RAG_INVALID_SCORE)); candidates[lane] = ((int)candidate.x >= 0 && isfinite(candidate.y) && candidate.y > RAG_INVALID_SCORE ? candidate : (float2)(-1.0f, RAG_INVALID_SCORE)); BarrierLoc; RAGSortDescending(candidates, lane, block_size); if(lane < top_k) destination[block * top_k + lane] = candidates[lane]; } //+------------------------------------------------------------------+ //| Implements the rag gather kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RAGGather(__global const float2 *top, __global const float *actions, __global float *tokens, const int scenario_count, const int action_count, const int action_dimension, const int token_width, const int top_k, __global const float *action_used, __global const float *action_mean_reward, __global const float *action_hit_count, __global float *absolute_relevance) { const int token = get_global_id(0); const int total = top_k * action_count; if(token >= total || action_count <= 0 || action_dimension <= 0 || token_width != action_dimension + 2) return; if(token == 0) { float best_score = MIN_VALUE; int has_action = 0; for(int candidate_rank = 0; candidate_rank < top_k; candidate_rank++) { const int candidate_scenario = (int)top[candidate_rank].x; if(candidate_scenario < 0 || candidate_scenario >= scenario_count || top[candidate_rank].y <= MIN_VALUE) continue; for(int candidate_action = 0; candidate_action < action_count; candidate_action++) { const int candidate_slot = candidate_scenario * action_count + candidate_action; if(action_used[candidate_slot] > 0.5f && action_hit_count[candidate_slot] > 0.0f) { best_score = fmax(best_score, IsNaNOrInf(top[candidate_rank].y, MIN_VALUE)); has_action = 1; break; } } } absolute_relevance[0] = (has_action ? IsNaNOrInf(best_score, 0.0f) : 0.0f); } const int rank = token / action_count, action = token - rank * action_count; const int scenario = (int)top[rank].x; const int width = token_width; const int out = token * width; if(scenario < 0 || scenario >= scenario_count || top[rank].y <= MIN_VALUE) { for(int d = 0; d < width; d++) tokens[out + d] = 0.0f; return; } const int slot = scenario * action_count + action; const int valid = (action_used[slot] > 0.5f && action_hit_count[slot] > 0.0f); if(!valid) { for(int d = 0; d < width; d++) tokens[out + d] = 0.0f; return; } const float raw_relevance = exp(clamp(IsNaNOrInf(top[rank].y, MIN_VALUE), -16.0f, 16.0f)) * log(1.0f + fmax(IsNaNOrInf(action_hit_count[slot], 0.0f), 0.0f)); float relevance_sum = 0.0f; for(int candidate_rank = 0; candidate_rank < top_k; candidate_rank++) { const int candidate_scenario = (int)top[candidate_rank].x; if(candidate_scenario < 0 || candidate_scenario >= scenario_count || top[candidate_rank].y <= MIN_VALUE) continue; for(int candidate_action = 0; candidate_action < action_count; candidate_action++) { const int candidate_slot = candidate_scenario * action_count + candidate_action; if(action_used[candidate_slot] <= 0.5f || action_hit_count[candidate_slot] <= 0.0f) continue; relevance_sum += exp(clamp(IsNaNOrInf(top[candidate_rank].y, MIN_VALUE), -16.0f, 16.0f)) * log(1.0f + fmax(IsNaNOrInf(action_hit_count[candidate_slot], 0.0f), 0.0f)); } } const float weighted_relevance = (relevance_sum > 1.0e-20f ? raw_relevance / relevance_sum : 0.0f); if(weighted_relevance <= 0.0f) { for(int d = 0; d < width; d++) tokens[out + d] = 0.0f; return; } for(int d = 0; d < action_dimension; d++) tokens[out + d] = IsNaNOrInf(actions[slot * action_dimension + d], 0.0f); tokens[out + action_dimension] = weighted_relevance; tokens[out + action_dimension + 1] = IsNaNOrInf(action_mean_reward[slot], 0.0f); } //+------------------------------------------------------------------+ //| Actor-owned differentiable rejection gate. Memory remains raw. | //+------------------------------------------------------------------+ __kernel void RAGTokenGate(__global const float *tokens, __global float *gated_tokens, __global const float *absolute_relevance, __global const float *threshold_weights, __global float *branch_activity, const int token_count, const int token_width, const float scale) { if(get_global_id(0) > 0 || token_count <= 0 || token_width < 2) return; int has_tokens = 0; for(int token = 0; token < token_count; token++) { const float weighted_relevance = IsNaNOrInf(tokens[token * token_width + token_width - 2], 0.0f); if(weighted_relevance > 0.0f) { has_tokens = 1; break; } } if(!has_tokens) { for(int i = 0; i < token_count * token_width; i++) gated_tokens[i] = 0.0f; branch_activity[0] = 0.0f; return; } const float threshold = IsNaNOrInf(threshold_weights[1], 0.0f); const float score = IsNaNOrInf(absolute_relevance[0], 0.0f); const float z = clamp(scale * (score - threshold), -40.0f, 40.0f); const float sigmoid = 1.0f / (1.0f + exp(-z)); for(int i = 0; i < token_count * token_width; i++) gated_tokens[i] = IsNaNOrInf(tokens[i] * sigmoid, 0.0f); branch_activity[0] = sigmoid; } //+------------------------------------------------------------------+ //| Accumulate only the threshold derivative; memory has no gradient | //+------------------------------------------------------------------+ __kernel void RAGTokenGateGrad(__global const float *tokens, __global const float *gated_gradient, __global const float *absolute_relevance, __global const float *threshold_weights, __global float *threshold_gradient, __global const float *branch_activity_gradient, const int token_count, const int token_width, const float scale) { if(get_global_id(0) > 0 || token_count <= 0 || token_width < 2) return; const float activity_gradient = IsNaNOrInf(branch_activity_gradient[0], 0.0f); int has_tokens = 0; for(int token = 0; token < token_count; token++) if(IsNaNOrInf(tokens[token * token_width + token_width - 2], 0.0f) > 0.0f) { has_tokens = 1; break; } if(!has_tokens) { threshold_gradient[0] = 0.0f; return; } const float threshold = IsNaNOrInf(threshold_weights[1], 0.0f); const float score = IsNaNOrInf(absolute_relevance[0], 0.0f); const float z = clamp(scale * (score - threshold), -40.0f, 40.0f); const float g = 1.0f / (1.0f + exp(-z)); float token_gradient = 0.0f; for(int token = 0; token < token_count; token++) { const int offset = token * token_width; for(int d = 0; d < token_width; d++) token_gradient += IsNaNOrInf(gated_gradient[offset + d], 0.0f) * IsNaNOrInf(tokens[offset + d], 0.0f); } const float gradient = (token_gradient + activity_gradient) * (-scale * g * (1.0f - g)); threshold_gradient[0] = clamp(IsNaNOrInf(gradient, 0.0f), -MAX_GRAD, MAX_GRAD); } //+------------------------------------------------------------------+ //| Generic bounded-memory nearest-pair reduction. One work-group | //| independently reduces one row of candidates; output is float4 | //| {squared_distance,left,right,valid}. | //+------------------------------------------------------------------+ inline void RAGNearestPairIndex(const int pair, const int count, int *left, int *right) { if(pair < 0 || count < 2) { *left = -1; *right = -1; return; } const long target = (long)pair; int low = 0; int high = count - 1; // Find the last triangular row prefix not greater than target. Exact // 64-bit intermediates keep the mapping stable at count==65536. while(low + 1 < high) { const int middle = low + (high - low) / 2; const long before = ((long)middle * (2L * (long)count - (long)middle - 1L)) / 2L; if(before <= target) low = middle; else high = middle; } const long row_start = ((long)low * (2L * (long)count - (long)low - 1L)) / 2L; const long column = (long)low + 1L + (target - row_start); *left = (column < (long)count ? low : -1); *right = (column < (long)count ? (int)column : -1); } //+------------------------------------------------------------------+ //| Implements the rag nearest before helper on the GPU. | //+------------------------------------------------------------------+ inline int RAGNearestBefore(const float4 left, const float4 right) { // Validity is part of the ordering. A valid distance saturated to // MAXFLOAT must still beat the invalid MAXFLOAT sentinel. if(left.w != right.w) return left.w > right.w; if(left.w <= 0.0f) return 0; if(left.x != right.x) return left.x < right.x; if(left.y != right.y) return left.y < right.y; return left.z < right.z; } //+------------------------------------------------------------------+ //| Implements the rag nearest pair kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void RAGNearestPair(__global const float *centroids, __global const float *used, __global float4 *result, const int candidate_count, const int dimension, const int batch_count) { const int lane = get_local_id(0); const int batch = get_group_id(0); const int lanes = get_local_size(0); __local float4 best[LOCAL_ARRAY_SIZE]; float4 current = (float4)(MAXFLOAT, -1.0f, -1.0f, 0.0f); if(candidate_count > 1 && dimension > 0 && batch >= 0 && batch < batch_count && lanes > 0 && lanes <= LOCAL_ARRAY_SIZE) { // candidate_count<=65536: divide the even factor before multiplying so // the documented maximum (2,147,450,880 pairs) stays in signed int. const int pairs = ((candidate_count & 1) == 0 ? (candidate_count / 2) * (candidate_count - 1) : candidate_count * ((candidate_count - 1) / 2)); const int base = batch * candidate_count; for(int pair = lane; pair < pairs; pair += lanes) { int left = -1, right = -1; RAGNearestPairIndex(pair, candidate_count, &left, &right); if(left < 0 || right < 0 || used[base + left] <= 0.5f || used[base + right] <= 0.5f) continue; float distance = 0.0f; const int lbase = (base + left) * dimension; const int rbase = (base + right) * dimension; for(int d = 0; d < dimension; d++) { const float delta = IsNaNOrInf(centroids[lbase + d] - centroids[rbase + d], MAXFLOAT); // Match the CPU fallback's explicit float rounding and prevent // contraction from changing the nearest-pair ordering. volatile float squared = delta * delta; distance = distance + squared; } const float4 candidate = (float4)(IsNaNOrInf(distance, MAXFLOAT), (float)left, (float)right, 1.0f); if(RAGNearestBefore(candidate, current)) current = candidate; } } best[lane] = current; BarrierLoc; for(int stride = lanes >> 1; stride > 0; stride >>= 1) { if(lane < stride && RAGNearestBefore(best[lane + stride], best[lane])) best[lane] = best[lane + stride]; BarrierLoc; } if(lane == 0 && batch >= 0 && batch < batch_count) result[batch] = best[0]; } //+------------------------------------------------------------------+ //| Computes the cog driver forecast gradient rms gradient | //| contributions on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverForecastGradientRMS(__global const float* gradient, __global float* error_target, const int dimension, const int horizon ) { const int h = get_global_id(0); if(h >= horizon || dimension <= 0) return; float sum = 0.0f; for(int d = 0; d < dimension; d++) { const int idx = h * dimension + d; const float g = IsNaNOrInf(gradient[idx], 0.0f); sum += g * g; } error_target[h] = sqrt(sum / (float)dimension); } //+------------------------------------------------------------------+ //| Implements the sparse softmax hausdorff kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void SparseSoftmaxHausdorff(__global const float *plan, __global const float *decis, __global float *output, __global float *top_index, const int dimension, const int top_k ) { const int d = get_local_id(0); const int plan_id = get_global_id(1); const int decis_id = get_global_id(2); const int total_decis = get_global_size(2); const int pair = plan_id * total_decis + decis_id; //--- __local float values[LOCAL_ARRAY_SIZE]; __local float top_values[LOCAL_ARRAY_SIZE]; __local int top_indices[LOCAL_ARRAY_SIZE]; //--- float value = 0.0f; if(d < dimension) { const float diff = plan[plan_id * dimension + d] - decis[decis_id * dimension + d]; value = IsNaNOrInf(fabs(diff), 0.0f); } values[d] = value; BarrierLoc; //--- if(d == 0) { const int actual_top = min(top_k, dimension); for(int k = 0; k < actual_top; k++) { float best = MIN_VALUE; int best_id = -1; for(int i = 0; i < dimension; i++) { int used = 0; for(int p = 0; p < k; p++) { if(top_indices[p] == i) { used = 1; break; } } if(used) continue; const float candidate = values[i]; if(candidate > best || (candidate == best && (best_id < 0 || i < best_id))) { best = candidate; best_id = i; } } top_values[k] = IsNaNOrInf(best, 0.0f); top_indices[k] = best_id; } //--- sparse softmax on selected dimensions float max_value = top_values[0]; for(int k = 1; k < actual_top; k++) max_value = fmax(max_value, top_values[k]); float sum_exp = 0.0f; float result = 0.0f; for(int k = 0; k < actual_top; k++) { const float weight = IsNaNOrInf(exp(top_values[k] - max_value), 0.0f); sum_exp += weight; result += weight * top_values[k]; } result = (sum_exp > 0.0f ? result / sum_exp : 0.0f); output[pair] = IsNaNOrInf(result, 0.0f); for(int k = 0; k < actual_top; k++) top_index[pair * top_k + k] = (float)top_indices[k]; } } //+------------------------------------------------------------------+ //| Computes the sparse softmax hausdorff grad gradient contributions| //| on the GPU. | //+------------------------------------------------------------------+ __kernel void SparseSoftmaxHausdorffGrad(__global const float *plan, __global float *grad_plan, __global const float *decis, __global float *grad_decis, __global const float *grad_out, __global const float *top_index, const int dimension, const int count_plan, const int count_decis, const int top_k ) { const int d = get_global_id(0); const int id = get_global_id(1); const int side = get_global_id(2); //--- if(d >= dimension) return; //--- float result = 0.0f; if(side == 0) { if(id >= count_plan) return; for(int decis_id = 0; decis_id < count_decis; decis_id++) { const int pair = id * count_decis + decis_id; const int shift = pair * top_k; float max_value = MIN_VALUE; for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); const float diff = plan[id * dimension + top_d] - decis[decis_id * dimension + top_d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); max_value = fmax(max_value, value); } float sum_exp = 0.0f; float out_value = 0.0f; for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); const float diff = plan[id * dimension + top_d] - decis[decis_id * dimension + top_d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); const float exp_value = IsNaNOrInf(exp(value - max_value), 0.0f); sum_exp += exp_value; out_value += exp_value * value; } if(sum_exp <= 0.0f) continue; out_value /= sum_exp; const float grad = IsNaNOrInf(grad_out[pair], 0.0f); for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); if(top_d != d) continue; const float diff = plan[id * dimension + d] - decis[decis_id * dimension + d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); const float weight = IsNaNOrInf(exp(value - max_value) / sum_exp, 0.0f); const float sgn = (diff > 0.0f ? 1.0f : (diff < 0.0f ? -1.0f : 0.0f)); result += grad * weight * (1.0f + value - out_value) * sgn; } } grad_plan[id * dimension + d] = IsNaNOrInf(result, 0.0f); } else { if(id >= count_decis) return; for(int plan_id = 0; plan_id < count_plan; plan_id++) { const int pair = plan_id * count_decis + id; const int shift = pair * top_k; float max_value = MIN_VALUE; for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); const float diff = plan[plan_id * dimension + top_d] - decis[id * dimension + top_d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); max_value = fmax(max_value, value); } float sum_exp = 0.0f; float out_value = 0.0f; for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); const float diff = plan[plan_id * dimension + top_d] - decis[id * dimension + top_d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); const float exp_value = IsNaNOrInf(exp(value - max_value), 0.0f); sum_exp += exp_value; out_value += exp_value * value; } if(sum_exp <= 0.0f) continue; out_value /= sum_exp; const float grad = IsNaNOrInf(grad_out[pair], 0.0f); for(int k = 0; k < top_k; k++) { const int top_d = (int)(top_index[shift + k] + 0.001f); if(top_d != d) continue; const float diff = plan[plan_id * dimension + d] - decis[id * dimension + d]; const float value = IsNaNOrInf(fabs(diff), 0.0f); const float weight = IsNaNOrInf(exp(value - max_value) / sum_exp, 0.0f); const float sgn = (diff > 0.0f ? 1.0f : (diff < 0.0f ? -1.0f : 0.0f)); result -= grad * weight * (1.0f + value - out_value) * sgn; } } grad_decis[id * dimension + d] = IsNaNOrInf(result, 0.0f); } } //+------------------------------------------------------------------+ //| Implements the mom adttm mean scores kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void MomADTTMMeanScores(__global const float *distance, __global float *output, const int decisions ) { const int plan_id = get_global_id(0); if(decisions <= 0) { output[plan_id] = 0.0f; return; } //--- float sum = 0.0f; for(int j = 0; j < decisions; j++) sum += IsNaNOrInf(distance[plan_id * decisions + j], 0.0f); output[plan_id] = IsNaNOrInf(-sum / (float)decisions, 0.0f); } //+------------------------------------------------------------------+ //| Computes the mom adttm mean scores grad gradient contributions on| //| the GPU. | //+------------------------------------------------------------------+ __kernel void MomADTTMMeanScoresGrad(__global const float *scores_gr, __global float *distance_gr, const int decisions ) { const int plan_id = get_global_id(0); const int decis_id = get_global_id(1); if(decisions <= 0) return; //--- distance_gr[plan_id * decisions + decis_id] = IsNaNOrInf(-scores_gr[plan_id] / (float)decisions, 0.0f); } //+------------------------------------------------------------------+ //| Implements the cog driver affine stack kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverAffineStack(__global float* stack, __global float* snapshot, __global const float* affine, const int stack_size, const int quant_dimension ) { const int feature = get_global_id(0); const int t = get_global_id(1); const int shift = feature * stack_size + t; const int modality = feature / quant_dimension; const float scale = IsNaNOrInf(affine[modality * 2], 0.0f); const float bias = IsNaNOrInf(affine[modality * 2 + 1], 0.0f); const float old = IsNaNOrInf(stack[shift], 0.0f); //--- snapshot[shift] = old; stack[shift] = IsNaNOrInf(old * scale + bias, 0.0f); } //+------------------------------------------------------------------+ //| Computes the cog driver affine stack grad gradient contributions | //| on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverAffineStackGrad(__global const float* snapshot, __global const float* stack_gr, __global float* affine_gr, const int stack_size, const int quant_dimension ) { const int modality = get_global_id(0); const int param = get_global_id(1); const int start = modality * quant_dimension; float grad = 0.0f; //--- for(int f = 0; f < quant_dimension; f++) { const int feature = start + f; for(int t = 0; t < stack_size - 1; t++) { const int old_shift = feature * stack_size + t; const int grad_shift = feature * stack_size + t + 1; const float g = IsNaNOrInf(stack_gr[grad_shift], 0.0f); if(param == 0) grad += g * IsNaNOrInf(snapshot[old_shift], 0.0f); else grad += g; } } affine_gr[modality * 2 + param] = IsNaNOrInf(grad, 0.0f); } //+------------------------------------------------------------------+ //| Implements the cog driver rank tcm insert pending kernel on the | //| GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverRankTCMInsertPending(__global float* memory, __global float* age, __global float* score, __global float* valid, __global const float* pending_query, const int memory_len, const int query_dim, const int variables ) { const int variable = get_global_id(0); const int dim = get_global_id(1); const int lid = get_local_id(1); const int lsize = get_local_size(1); if(variable >= variables || dim >= query_dim || memory_len <= 0 || query_dim <= 0) return; __local int invalid_slot[LOCAL_ARRAY_SIZE]; __local int min_slot[LOCAL_ARRAY_SIZE]; __local int min_age[LOCAL_ARRAY_SIZE]; __local float min_score[LOCAL_ARRAY_SIZE]; __local int insert_slot[1]; int local_invalid = memory_len; int local_min_slot = -1; int local_min_age = 0; float local_min_score = MAX_VALUE; //--- each work-item scans a part of one variable memory row int step = min((int)LOCAL_ARRAY_SIZE, (int)lsize); if(lid < step) { for(int slot = lid; slot < memory_len; slot += step) { const int state_shift = RCtoFlat(slot, variable, memory_len, variables, 0); if(valid[state_shift] < 0.5f) { if(slot < local_invalid) local_invalid = slot; continue; } const float sc = IsNaNOrInf(score[state_shift], MIN_VALUE); const int t_age = (int)IsNaNOrInf(age[state_shift], 0.0f); age[state_shift] = (float)(t_age + 1); if(local_min_slot < 0 || sc < local_min_score || (sc == local_min_score && t_age > local_min_age)) { local_min_slot = slot; local_min_score = sc; local_min_age = t_age; } } invalid_slot[lid] = local_invalid; min_slot[lid] = local_min_slot; min_score[lid] = local_min_score; min_age[lid] = local_min_age; } BarrierLoc //--- reduce to one replacement slot per variable if(lid == 0) { int best_invalid = memory_len; int best_min_slot = -1; int best_min_age = 0; float best_min_score = MAX_VALUE; for(int i = 0; i < lsize; i++) { if(invalid_slot[i] < best_invalid) best_invalid = invalid_slot[i]; if(min_slot[i] >= 0 && (best_min_slot < 0 || min_score[i] < best_min_score || (min_score[i] == best_min_score && min_age[i] < best_min_age))) { best_min_slot = min_slot[i]; best_min_score = min_score[i]; best_min_age = min_age[i]; } } insert_slot[0] = (best_invalid < memory_len ? best_invalid : best_min_slot); } BarrierLoc const int target_slot = insert_slot[0]; if(target_slot < 0) return; //--- write the delayed query to the replacement slot memory[RCtoFlat(variable, dim, variables, query_dim, target_slot)] = IsNaNOrInf(pending_query[RCtoFlat(variable, dim, variables, query_dim, 0)], 0.0f); if(lid == 0) { const int state_shift = RCtoFlat(target_slot, variable, memory_len, variables, 0); age[state_shift] = 1.0f; score[state_shift] = 0.0f; valid[state_shift] = 1.0f; } } //+------------------------------------------------------------------+ //| Implements the cog driver rank tcm build kv kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverRankTCMBuildKV(__global const float* memory, __global const float* age, __global const float* valid, __global float* memory_kv, const int memory_count, const int variables, const int heads, const int head_dim ) { const int slot = get_global_id(0); const int variable = get_global_id(1); const int flat_dim = get_global_id(2); if(slot >= memory_count || variable >= variables || heads <= 0 || head_dim <= 0) return; const int query_dim = heads * head_dim; if(flat_dim >= query_dim) return; //--- const int head = flat_dim / head_dim; const int local_dim = flat_dim - head * head_dim; const int state_shift = RCtoFlat(slot, variable, memory_count, variables, 0); float value = 0.0f; if(valid[state_shift] >= 0.5f) { const float a = IsNaNOrInf(age[state_shift], 0.0f); const float period = pow(10000.0f, (float)(2 * (local_dim / 2)) / (float)max(head_dim, 1)); const float phase = a / period; const float pe = ((local_dim & 1) == 0 ? sin(phase) : cos(phase)); value = IsNaNOrInf(memory[RCtoFlat(variable, flat_dim, variables, query_dim, slot)] + pe, 0.0f); } const int total_heads = variables * heads; const int effective_head = variable * heads + head; const int k_shift = RCtoFlat(effective_head, local_dim, 2 * total_heads, head_dim, slot); const int v_shift = RCtoFlat(effective_head + total_heads, local_dim, 2 * total_heads, head_dim, slot); memory_kv[k_shift] = value; memory_kv[v_shift] = value; } //+------------------------------------------------------------------+ //| Implements the cog driver rank tcm mean flash score kernel on the| //| GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverRankTCMMeanFlashScore(__global const float* query, __global const float* memory_kv, __global const float* valid, __global float* out_score, __global const float* logsumexp, const int memory_count, const int variables, const int heads, const int head_dim ) { const int slot = get_global_id(0); const int variable = get_global_id(1); if(slot >= memory_count || variable >= variables || heads <= 0 || head_dim <= 0) return; const int state_shift = RCtoFlat(slot, variable, memory_count, variables, 0); if(valid[state_shift] < 0.5f) { out_score[state_shift] = MIN_VALUE; return; } //--- float score = 0.0f; const int total_heads = variables * heads; for(int head = 0; head < heads; head++) { const int effective_head = variable * heads + head; const int shift_q = RCtoFlat(effective_head, 0, total_heads, head_dim, 0); const int shift_k = RCtoFlat(effective_head, 0, 2 * total_heads, head_dim, slot); const int shift_lse = RCtoFlat(0, effective_head, 1, total_heads, 0); const float head_lse = IsNaNOrInf(logsumexp[shift_lse], 0.0f); float dotv = 0.0f; for(int d = 0; d < head_dim; d++) dotv += IsNaNOrInf(query[shift_q + d], 0.0f) * IsNaNOrInf(memory_kv[shift_k + d], 0.0f); const float raw_score = IsNaNOrInf(dotv / sqrt((float)head_dim), MIN_VALUE); score += IsNaNOrInf(exp(clamp(raw_score - head_lse, -120.0f, 0.0f)), 0.0f); } out_score[state_shift] = IsNaNOrInf(score / (float)heads, 0.0f); } //+------------------------------------------------------------------+ //| Implements the cog driver rank tcm self kv kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void CogDriverRankTCMSelfKV(__global const float* source, __global float* self_kv, const int query_dim, const int variables, const int heads, const int head_dim ) { const int key_variable = get_global_id(0); const int flat_dim = get_global_id(1); if(key_variable >= variables || flat_dim >= query_dim || heads <= 0 || head_dim <= 0) return; //--- const int head = flat_dim / head_dim; const int local_dim = flat_dim - head * head_dim; const int total_heads = variables * heads; const float value = IsNaNOrInf(source[RCtoFlat(key_variable, flat_dim, variables, query_dim, 0)], 0.0f); for(int query_variable = 0; query_variable < variables; query_variable++) { const int effective_head = query_variable * heads + head; const int k_shift = RCtoFlat(effective_head, local_dim, 2 * total_heads, head_dim, key_variable); const int v_shift = RCtoFlat(effective_head + total_heads, local_dim, 2 * total_heads, head_dim, key_variable); self_kv[k_shift] = value; self_kv[v_shift] = value; } } //+------------------------------------------------------------------+ //| Scenario-major forecast: Z[k][h][d], U[k][h], Pi[k] | //+------------------------------------------------------------------+ __kernel void ScenarioForecastForward(__global const float *input, __global const float *prototypes, __global float *generator_input, __global float *router_input, __global float *confidence_input, __global const float *generator_output, __global const float *router_output, __global const float *confidence_output, __global float *z, __global float *u, __global float *pi, const int scenarios, const int horizon, const int dimension, const int phase, __global float *log_pi, __global float *log_u, const int variables, __global float *forward_validity, __global const float *generator_weights, __global const float *router_weights, __global const float *confidence_weights, const int generator_weights_count, const int router_weights_count, const int confidence_weights_count) { const int d = get_global_id(0); const int v = get_global_id(1); const int k = get_global_id(2); const int ld = get_local_id(0); const int lv = get_local_id(1); __local float Temp[SCENARIO_LOCAL_CAP]; if(scenarios <= 0 || variables <= 0 || horizon <= 0 || dimension <= 0 || dimension * variables > SCENARIO_LOCAL_CAP || d >= dimension || v >= variables || k >= scenarios) return; const float raw_state = input[v * dimension + d]; const float state = IsNaNOrInf(raw_state, 0.0f); float valid = (isnan(raw_state) || isinf(raw_state) ? 0.0f : 1.0f); if(phase == 0) { const int global_lane = (k * variables + v) * dimension + d; const int total_lanes = scenarios * variables * dimension; for(int index = global_lane; index < generator_weights_count; index += total_lanes) if(isnan(generator_weights[index]) || isinf(generator_weights[index])) valid = 0.0f; for(int index = global_lane; index < router_weights_count; index += total_lanes) if(isnan(router_weights[index]) || isinf(router_weights[index])) valid = 0.0f; for(int index = global_lane; index < confidence_weights_count; index += total_lanes) if(isnan(confidence_weights[index]) || isinf(confidence_weights[index])) valid = 0.0f; float prototype_mean = 0.0f; for(int h = 0; h < horizon; h++) { const int prototype_shift = ((k * variables + v) * horizon + h) * dimension + d; const int condition_shift = ((k * variables + v) * horizon + h) * 2 * dimension; const float raw_prototype = prototypes[prototype_shift]; const float prototype = IsNaNOrInf(raw_prototype, 0.0f); if(isnan(raw_prototype) || isinf(raw_prototype)) valid = 0.0f; generator_input[condition_shift + d] = state; generator_input[condition_shift + dimension + d] = prototype; confidence_input[condition_shift + d] = state; confidence_input[condition_shift + dimension + d] = prototype; prototype_mean = IsNaNOrInf(prototype_mean + prototype, 0.0f); } const int router_shift = k * 2 * variables * dimension; router_input[router_shift + v * dimension + d] = state; router_input[router_shift + variables * dimension + v * dimension + d] = IsNaNOrInf(prototype_mean / (float)horizon, 0.0f); const float variable_valid = LocalMin(valid, 0, Temp + lv * dimension); BarrierLoc const float joint_valid = LocalMin(ld == 0 ? variable_valid : 1.0f, 1, Temp + ld * variables); if(ld == 0 && lv == 0) forward_validity[k] = (joint_valid > 0.5f ? 1.0f : 0.0f); return; } for(int h = 0; h < horizon; h++) { const int prototype_shift = ((k * variables + v) * horizon + h) * dimension + d; const float raw_prototype = prototypes[prototype_shift]; const float prototype = IsNaNOrInf(raw_prototype, 0.0f); const float raw_residual = generator_output[prototype_shift]; const float residual = IsNaNOrInf(raw_residual, 0.0f); if(isnan(raw_state) || isinf(raw_state) || isnan(raw_prototype) || isinf(raw_prototype) || isnan(raw_residual) || isinf(raw_residual)) valid = 0.0f; z[prototype_shift] = IsNaNOrInf(state + prototype + residual, 0.0f); if(d == 0) { const int confidence_shift = (k * variables + v) * horizon + h; const float raw_confidence = confidence_output[confidence_shift]; if(isnan(raw_confidence) || isinf(raw_confidence)) valid = 0.0f; const float confidence = clamp(IsNaNOrInf(raw_confidence, 0.0f) + 0.001f, 0.001f, 10.0f); u[confidence_shift] = IsNaNOrInf(confidence, 0.001f); log_u[confidence_shift] = IsNaNOrInf(log(confidence + 1.0e-6f), log(0.001001f)); } } if(d == 0 && v == 0) { const float raw_router = router_output[k]; const float raw_probability = pi[k]; if(isnan(raw_router) || isinf(raw_router) || isnan(raw_probability) || isinf(raw_probability)) valid = 0.0f; const float probability = clamp(IsNaNOrInf(raw_probability, 0.0f), 0.0f, 1.0f); log_pi[k] = IsNaNOrInf(log(probability + 1.0e-6f), log(1.0e-6f)); } const float variable_valid = LocalMin(valid, 0, Temp + lv * dimension); BarrierLoc const float joint_valid = LocalMin(ld == 0 ? variable_valid : 1.0f, 1, Temp + ld * variables); if(ld == 0 && lv == 0) forward_validity[k] = (forward_validity[k] > 0.5f && joint_valid > 0.5f ? 1.0f : 0.0f); } //+------------------------------------------------------------------+ //| Stable Laplace mixture posterior with TopK renormalization | //+------------------------------------------------------------------+ __kernel void ScenarioResponsibilities(__global float *target, __global const float *z, __global const float *u, __global const float *pi, __global float *responsibilities, __global float *generator_gradient, __global float *router_gradient, __global float *confidence_gradient, const int scenarios, const int top_k, const int horizon, const int dimension, __global float *scores, __global const float *sparse, __global const float *indexes, const int phase, const int variables, __global const float *prototypes, __global float *batch_control, const float huber_delta, __global const float *target_source, __global const float *forward_validity, __global float *diagnostic_partials, __global float *epoch_diagnostics, __global const float *observation_by_scenario, __global float *diagnostic_scratch, __global const float *usage, __global const float *inactive, __global float *recovery_events, __global float *age_zero_events) { if(variables <= 0 || scenarios <= 0 || top_k <= 0 || top_k > scenarios || horizon <= 0 || dimension <= 0 || dimension * variables > SCENARIO_LOCAL_CAP) return; const int d = get_global_id(0); const int v = get_global_id(1); const int k = get_global_id(2); const int ld = get_local_id(0); const int lv = get_local_id(1); __local float Temp[SCENARIO_LOCAL_CAP]; __local float TempSecond[SCENARIO_LOCAL_CAP]; if(phase == 0) { if(d >= dimension || v >= variables) return; float valid = 1.0f; for(int h = 0; h < horizon; h++) { const int shift = (v * horizon + h) * dimension + d; const float raw = target_source[shift]; if(isnan(raw) || isinf(raw)) valid = 0.0f; target[shift] = IsNaNOrInf(raw, 0.0f); } const float variable_valid = LocalMin(valid, 0, Temp + lv * dimension); BarrierLoc const float joint_valid = LocalMin(ld == 0 ? variable_valid : 1.0f, 1, Temp + ld * variables); if(ld == 0 && lv == 0) { float forecast_valid = 1.0f; for(int scenario = 0; scenario < scenarios; scenario++) forecast_valid = min(forecast_valid, forward_validity[scenario]); batch_control[0] = (joint_valid > 0.5f && forecast_valid > 0.5f ? 1.0f : 0.0f); } return; } if(phase == 1) { if(d >= dimension || v >= variables || k >= scenarios) return; float laplace = 0.0f; float trajectory = 0.0f; float confidence_cost = 0.0f; float latent_abs = 0.0f; for(int h = 0; h < horizon; h++) { const int shift = ((k * variables + v) * horizon + h) * dimension + d; const int target_shift = (v * horizon + h) * dimension + d; const float scale = clamp(IsNaNOrInf(u[(k * variables + v) * horizon + h], 1.0f), 1.0e-3f, 10.0f); const float error = IsNaNOrInf(target[target_shift] - IsNaNOrInf(z[shift], 0.0f), 0.0f); laplace = IsNaNOrInf(laplace + fabs(error) / scale + log(2.0f * scale), 0.0f); const float absolute_error = fabs(error); trajectory = IsNaNOrInf(trajectory + (absolute_error <= huber_delta ? 0.5f * absolute_error * absolute_error : huber_delta * (absolute_error - 0.5f * huber_delta)), 0.0f); confidence_cost = IsNaNOrInf(confidence_cost + log(scale) + absolute_error / scale, 0.0f); latent_abs = IsNaNOrInf(latent_abs + absolute_error, 0.0f); } const float variable_laplace = LocalSum(laplace, 0, Temp + lv * dimension); BarrierLoc const float joint_laplace = LocalSum(ld == 0 ? variable_laplace : 0.0f, 1, Temp + ld * variables); const float variable_trajectory = LocalSum(trajectory, 0, Temp + lv * dimension); const float variable_confidence = LocalSum(confidence_cost, 0, TempSecond + lv * dimension); BarrierLoc const float joint_trajectory = LocalSum(ld == 0 ? variable_trajectory : 0.0f, 1, Temp + ld * variables); const float joint_confidence = LocalSum(ld == 0 ? variable_confidence : 0.0f, 1, TempSecond + ld * variables); const float variable_latent = LocalSum(latent_abs, 0, Temp + lv * dimension); BarrierLoc const float joint_latent = LocalSum(ld == 0 ? variable_latent : 0.0f, 1, Temp + ld * variables); if(ld == 0 && lv == 0) { const bool valid = (batch_control[0] > 0.5f); //--- The likelihood is accumulated across the whole joint trajectory. Scale //--- its complete log-posterior by the root coordinate count so the //--- sparse TopK posterior remains representable for all selected scenarios. const float score_scale = sqrt(max((float)variables * (float)horizon * (float)dimension, 1.0f)); scores[k] = (valid ? IsNaNOrInf((log(max(IsNaNOrInf(pi[k], 0.0f), 1.0e-12f)) - joint_laplace) / score_scale, MIN_VALUE) : MIN_VALUE); diagnostic_partials[k] = (valid ? IsNaNOrInf(joint_trajectory, 0.0f) : 0.0f); diagnostic_partials[scenarios + k] = (valid ? IsNaNOrInf(joint_confidence, 0.0f) : 0.0f); diagnostic_partials[2 * scenarios + k] = (valid ? IsNaNOrInf(joint_latent, 0.0f) : 0.0f); } return; } if(phase == 2) { if(d >= dimension || v >= variables || k >= scenarios) return; float responsibility = 0.0f; if(batch_control[0] > 0.5f) for(int selected = 0; selected < top_k; selected++) if((int)indexes[selected] == k) { responsibility = IsNaNOrInf(sparse[selected], 0.0f); break; } //--- Scores are normalized by the joint coordinate count, so every //--- negative objective derivative uses the same inverse scale. const float score_scale = sqrt(max((float)variables * (float)horizon * (float)dimension, 1.0f)); if(d == 0 && v == 0) { responsibilities[k] = responsibility; //--- Update kernels add their gradient. Use the library convention //--- target-output, i.e. posterior responsibility minus router Pi. router_gradient[k] = (batch_control[0] > 0.5f ? (responsibility - IsNaNOrInf(pi[k], 0.0f)) / score_scale : 0.0f); } for(int h = 0; h < horizon; h++) { const int confidence_shift = (k * variables + v) * horizon + h; const int shift = confidence_shift * dimension + d; const int target_shift = (v * horizon + h) * dimension + d; const float scale = clamp(IsNaNOrInf(u[confidence_shift], 1.0f), 1.0e-3f, 10.0f); const float error = IsNaNOrInf(target[target_shift] - z[shift], 0.0f); generator_gradient[shift] = (responsibility > 0.0f ? (error > 0.0f ? 1.0f : (error < 0.0f ? -1.0f : 0.0f)) / (scale * score_scale) : 0.0f); const float absolute_sum = LocalSum(responsibility > 0.0f ? fabs(error) : 0.0f, 0, Temp + lv * dimension); if(ld == 0) { //--- U is SoftPlus output plus epsilon. Convert -dL/dU to the //--- same target-output convention used by convolution updates. //--- A tiny floor lets a confidence collapsed at epsilon recover. const float softplus_output = max(scale - 0.001f, 0.0f); const float softplus_gradient = max(1.0f - exp(-softplus_output), 1.0e-3f); const float confidence_signal = absolute_sum / (scale * scale) - (float)dimension / scale; confidence_gradient[confidence_shift] = (responsibility > 0.0f && scale < 10.0f ? IsNaNOrInf(confidence_signal * softplus_gradient / score_scale, 0.0f) : 0.0f); } BarrierLoc } return; } if(phase == 3) { const int scenario = get_local_id(0); if(scenario >= scenarios) return; const float score = scores[scenario]; const float responsibility = clamp(IsNaNOrInf(responsibilities[scenario], 0.0f), 0.0f, 1.0f); const float maximum = LocalMax(score, 0, Temp); const float sum_exp = LocalSum(exp(score - maximum), 0, TempSecond); const float router_value = -responsibility * log(max(IsNaNOrInf(pi[scenario], 0.0f), 1.0e-12f)); const float router_sum = LocalSum(router_value, 0, Temp); const float trajectory_sum = LocalSum(responsibility * diagnostic_partials[scenario], 0, TempSecond); const float confidence_sum = LocalSum(responsibility * diagnostic_partials[scenarios + scenario], 0, Temp); const float latent_sum = LocalSum(responsibility * diagnostic_partials[2 * scenarios + scenario], 0, TempSecond); const float observation_sum = LocalSum(IsNaNOrInf(observation_by_scenario[scenario], 0.0f), 0, Temp); if(scenario == 0) { if(batch_control[0] > 0.5f) { epoch_diagnostics[0] += IsNaNOrInf(-(maximum + log(max(sum_exp, 1.0e-30f))), 0.0f); epoch_diagnostics[1] += IsNaNOrInf(router_sum, 0.0f); epoch_diagnostics[2] += IsNaNOrInf(trajectory_sum, 0.0f); epoch_diagnostics[3] += IsNaNOrInf(confidence_sum, 0.0f); epoch_diagnostics[4] += IsNaNOrInf(latent_sum, 0.0f); epoch_diagnostics[5] += IsNaNOrInf(observation_sum, 0.0f); epoch_diagnostics[6] += 1.0f; } else epoch_diagnostics[7] += 1.0f; } return; } if(phase == 4) { if(get_global_id(0) == 0) for(int i = 0; i < 12; i++) epoch_diagnostics[i] = 0.0f; if(get_global_id(0) < scenarios) { recovery_events[get_global_id(0)] = 0.0f; age_zero_events[get_global_id(0)] = 0.0f; } return; } if(phase == 5) { const int pair = get_global_id(2); const int pairs = scenarios * (scenarios - 1) / 2; if(d >= dimension || v >= variables || pair >= pairs) return; int left = 0; int right = 1; int remainder = pair; for(left = 0; left < scenarios - 1; left++) { const int row = scenarios - left - 1; if(remainder < row) { right = left + 1 + remainder; break; } remainder -= row; } float square_sum = 0.0f; for(int h = 0; h < horizon; h++) { const int coordinate = (v * horizon + h) * dimension + d; const int stride = variables * horizon * dimension; const float difference = IsNaNOrInf(prototypes[left * stride + coordinate] - prototypes[right * stride + coordinate], 0.0f); square_sum += difference * difference; } const float variable_sum = LocalSum(square_sum, 0, Temp + lv * dimension); BarrierLoc const float joint_sum = LocalSum(ld == 0 ? variable_sum : 0.0f, 1, Temp + ld * variables); if(ld == 0 && lv == 0) diagnostic_scratch[pair] = sqrt(max(joint_sum, 0.0f) / max((float)(variables * horizon * dimension), 1.0f)); return; } if(phase == 6) { const int lane = get_local_id(0); const int pairs = scenarios * (scenarios - 1) / 2; const float usage_value = (lane < scenarios ? max(IsNaNOrInf(usage[lane], 0.0f), 0.0f) : 0.0f); const float usage_sum = LocalSum(usage_value, 0, Temp); const float probability = (usage_sum > 0.0f ? usage_value / usage_sum : 0.0f); const float entropy_value = (probability > 0.0f ? -probability * log(probability) : 0.0f); const float entropy_sum = LocalSum(entropy_value, 0, TempSecond); if(lane == 0) epoch_diagnostics[8] = IsNaNOrInf(entropy_sum, 0.0f); BarrierLoc const float inactive_sum = LocalSum(lane < scenarios && IsNaNOrInf(inactive[lane], 0.0f) > 0.0f ? 1.0f : 0.0f, 0, Temp); if(lane == 0) epoch_diagnostics[10] = IsNaNOrInf(inactive_sum, 0.0f); BarrierLoc const float pair_sum = LocalSum(lane < pairs ? IsNaNOrInf(diagnostic_scratch[lane], 0.0f) : 0.0f, 0, Temp); if(lane == 0) epoch_diagnostics[9] = (pairs > 0 ? IsNaNOrInf(pair_sum / (float)pairs, 0.0f) : 0.0f); BarrierLoc const float recovered_sum = LocalSum(lane < scenarios ? IsNaNOrInf(recovery_events[lane], 0.0f) : 0.0f, 0, Temp); if(lane == 0) epoch_diagnostics[11] = IsNaNOrInf(recovered_sum, 0.0f); return; } } //+------------------------------------------------------------------+ //| Codebook EMA; prototypes are updated statistically, not by grad | //+------------------------------------------------------------------+ __kernel void ScenarioCodebookEMA(__global const float *target, __global float *prototypes, __global float *ema_sums, __global float *ema_counts, __global float *usage, __global float *inactive, __global const float *batch_control, const int scenarios, const int dimension, const float decay, __global float *inactivity_age, const int seed, const int recovery_age, __global float *recovery, const int phase, const int horizon, const int variables, __global const float *responsibilities, __global float *recovery_events, __global float *age_zero_events) { if(phase == 0) { const int k = get_global_id(0); if(k >= scenarios) return; recovery[k] = 0.0f; if(batch_control[0] <= 0.5f) return; //--- The same normalized TopK responsibility that trains Z_k updates //--- this scenario's Delta-Z EMA. Non-TopK scenarios receive zero. const float hit = clamp(IsNaNOrInf(responsibilities[k], 0.0f), 0.0f, 1.0f); const float count = decay * IsNaNOrInf(ema_counts[k], 0.0f) + (1.0f - decay) * hit; const float age = (hit > 0.0f ? 0.0f : IsNaNOrInf(inactivity_age[k], 0.0f) + 1.0f); const bool recover = (hit == 0.0f && age >= (float)recovery_age); recovery[k] = (recover ? 1.0f : 0.0f); if(recover) recovery_events[k] = IsNaNOrInf(recovery_events[k], 0.0f) + 1.0f; ema_counts[k] = (recover ? 1.0f : count); usage[k] = IsNaNOrInf(usage[k], 0.0f) + hit; const float final_age = (recover ? 0.0f : age); inactivity_age[k] = final_age; if(final_age == 0.0f) age_zero_events[k] = IsNaNOrInf(age_zero_events[k], 0.0f) + 1.0f; inactive[k] = (!recover && count < 1.0e-4f ? 1.0f : 0.0f); return; } const int d = get_global_id(0); const int v = get_global_id(1); const int k = get_global_id(2); if(phase != 1 || d >= dimension || v >= variables || k >= scenarios || batch_control[0] <= 0.5f) return; const float hit = clamp(IsNaNOrInf(responsibilities[k], 0.0f), 0.0f, 1.0f); const float count = IsNaNOrInf(ema_counts[k], 0.0f); const bool recover = (recovery[k] > 0.5f); for(int h = 0; h < horizon; h++) { const int coordinate = (v * horizon + h) * dimension + d; const int shift = k * variables * horizon * dimension + coordinate; const float sample = IsNaNOrInf(target[coordinate], 0.0f); if(recover) { const float prototype = IsNaNOrInf(sample + 0.01f * sin((float)seed + 17.0f * (float)k + 0.61803398875f * (float)(coordinate + 1)), sample); prototypes[shift] = prototype; ema_sums[shift] = prototype; } else { const float sum = IsNaNOrInf(decay * ema_sums[shift] + (1.0f - decay) * hit * sample, 0.0f); ema_sums[shift] = sum; if(count > 1.0e-6f) prototypes[shift] = IsNaNOrInf(sum / count, 0.0f); } } } //+------------------------------------------------------------------+ //| Per-scenario RMS normalization; inactive branches remain zero | //+------------------------------------------------------------------+ __kernel void ScenarioGradientNormalize(__global float *generator_gradient, __global const float *generator_input_gradient, __global const float *router_input_gradient, __global const float *confidence_input_gradient, __global float *input_gradient, __global const float *responsibilities, const int scenarios, const int horizon, const int dimension, const int phase, __global float *confidence_gradient, const int variables, __global const float *batch_control, const int normalize_gradients) { const int d = get_global_id(0); const int v = get_global_id(1); const int k = get_global_id(2); const int ld = get_local_id(0); const int lv = get_local_id(1); __local float Temp[SCENARIO_LOCAL_CAP]; __local float TempSecond[SCENARIO_LOCAL_CAP]; if(phase == 0) { if(d >= dimension || v >= variables || k >= scenarios) return; const float responsibility = (batch_control[0] > 0.5f ? IsNaNOrInf(responsibilities[k], 0.0f) : 0.0f); float generator_square_sum = 0.0f; float confidence_square_sum = 0.0f; for(int h = 0; h < horizon; h++) { const int confidence_shift = (k * variables + v) * horizon + h; const float generator_value = IsNaNOrInf(generator_gradient[confidence_shift * dimension + d], 0.0f); generator_square_sum += generator_value * generator_value; if(d == 0) { const float confidence_value = IsNaNOrInf(confidence_gradient[confidence_shift], 0.0f); confidence_square_sum += confidence_value * confidence_value; } } const float variable_generator = LocalSum(generator_square_sum, 0, Temp + lv * dimension); const float variable_confidence = LocalSum(confidence_square_sum, 0, TempSecond + lv * dimension); BarrierLoc const float joint_generator = LocalSum(ld == 0 ? variable_generator : 0.0f, 1, Temp + ld * variables); const float joint_confidence = LocalSum(ld == 0 ? variable_confidence : 0.0f, 1, TempSecond + ld * variables); if(ld == 0 && lv == 0) { Temp[0] = joint_generator; TempSecond[0] = joint_confidence; } BarrierLoc const float generator_rms = sqrt(Temp[0] / max((float)(variables * horizon * dimension), 1.0f) + 1.0e-12f); const float confidence_rms = sqrt(TempSecond[0] / max((float)(variables * horizon), 1.0f) + 1.0e-12f); const float generator_scale = (responsibility > 0.0f ? responsibility / (normalize_gradients != 0 ? max(generator_rms, 1.0e-6f) : 1.0f) : 0.0f); const float confidence_scale = (responsibility > 0.0f ? responsibility / (normalize_gradients != 0 ? max(confidence_rms, 1.0e-6f) : 1.0f) : 0.0f); for(int h = 0; h < horizon; h++) { const int confidence_shift = (k * variables + v) * horizon + h; const int shift = confidence_shift * dimension + d; generator_gradient[shift] = IsNaNOrInf(generator_gradient[shift] * generator_scale, 0.0f); if(d == 0) confidence_gradient[confidence_shift] = IsNaNOrInf(confidence_gradient[confidence_shift] * confidence_scale, 0.0f); } return; } if(phase == 1) { const int lane = get_local_id(0); const int dimension_index = get_global_id(1); const int variable = get_global_id(2); const int lanes = get_local_size(0); if(dimension_index >= dimension || variable >= variables) return; float partial = 0.0f; if(batch_control[0] > 0.5f) for(int q = lane; q < scenarios * horizon; q += lanes) { const int scenario = q / horizon; const int h = q - scenario * horizon; const int router_shift = scenario * 2 * variables * dimension; if(h == 0) partial += IsNaNOrInf(router_input_gradient[router_shift + variable * dimension + dimension_index], 0.0f); const int shift = ((scenario * variables + variable) * horizon + h) * 2 * dimension + dimension_index; const int direct_shift = ((scenario * variables + variable) * horizon + h) * dimension + dimension_index; partial += IsNaNOrInf(generator_gradient[direct_shift], 0.0f) + IsNaNOrInf(generator_input_gradient[shift], 0.0f) + IsNaNOrInf(confidence_input_gradient[shift], 0.0f); } const float aggregate = LocalSum(partial, 0, Temp); if(lane == 0) input_gradient[variable * dimension + dimension_index] = IsNaNOrInf(aggregate, 0.0f); return; } } //+------------------------------------------------------------------+ //| Implements the D2Skill item partial kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillItemPartial(__global const float *source, __global float *partials, const int dimension, const int output_offset) { const int id = get_global_id(0); if(id < 0 || id >= dimension) return; const float value = IsNaNOrInf(source[id], 0.0f); partials[output_offset + id] = IsNaNOrInf(value * value, 0.0f); } //+------------------------------------------------------------------+ //| Recursive no-atomics reduction into disjoint scratch regions. | //+------------------------------------------------------------------+ __kernel void D2SkillItemReduce(__global float *partials, __global float *reduced, const int input_count, const int input_offset, const int output_offset, const int final_output) { const int index = get_global_id(0); const int left = 2 * index; if(left >= input_count) return; const float sum = IsNaNOrInf(partials[input_offset + left], 0.0f) + (left + 1 < input_count ? IsNaNOrInf(partials[input_offset + left + 1], 0.0f) : 0.0f); if(final_output != 0) reduced[0] = IsNaNOrInf(sum, 0.0f); else partials[output_offset + index] = IsNaNOrInf(sum, 0.0f); } //+------------------------------------------------------------------+ //| Parallel Item state update and correction materialization. | //+------------------------------------------------------------------+ #define D2_ITEM_APPLY_UPDATE 0 #define D2_ITEM_APPLY_CORRECTION 1 //+------------------------------------------------------------------+ //| Implements the D2Skill item apply kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillItemApply(__global const float *gradient, __global float *correction, __global float *direction, __global float *scale, __global float *observations, __global float *mass, __global const float *reduced_norm, const int dimension, const int representation, const float beta_correction, const float beta_direction, const float beta_magnitude, const int phase) { const int id = get_global_id(0); const float norm = sqrt(fmax(IsNaNOrInf(reduced_norm[0], 0.0f), 0.0f)); if(phase == D2_ITEM_APPLY_UPDATE) { if(id < dimension) { const float g = IsNaNOrInf(gradient[id], 0.0f); if(representation == 0) { const float old_value = IsNaNOrInf(correction[id], 0.0f); correction[id] = (1.0f - beta_correction) * old_value + beta_correction * g; } else { const float inv_norm = (norm > 1.0e-12f ? 1.0f / norm : 0.0f); const float old_direction = IsNaNOrInf(direction[id], 0.0f); direction[id] = (1.0f - beta_direction) * old_direction + beta_direction * g * inv_norm; } } if(id == 0) { if(representation != 0) scale[0] = clamp((1.0f - beta_magnitude) * IsNaNOrInf(scale[0], 0.0f) + beta_magnitude * norm, 0.0f, MAX_VALUE); observations[0] = IsNaNOrInf(observations[0], 0.0f) + 1.0f; mass[0] = IsNaNOrInf(mass[0], 0.0f) + norm; } return; } if(id >= dimension || representation == 0) return; const float inv_direction = (norm > 1.0e-12f ? 1.0f / norm : 0.0f); correction[id] = IsNaNOrInf(scale[0], 0.0f) * IsNaNOrInf(direction[id], 0.0f) * inv_direction; } //+------------------------------------------------------------------+ //| Write one score contribution per slot/component lane. | //+------------------------------------------------------------------+ __kernel void D2SkillScoreSlots(__global const float *query, __global const float *keys, __global float *partials, const int slots, const int dimension) { const int index = get_global_id(0); const int total = slots * dimension; if(index < 0 || index >= total || dimension <= 0) return; const int component = index % dimension; const float q = IsNaNOrInf(query[component], 0.0f); const float k = IsNaNOrInf(keys[index], 0.0f); const float dot = q * k; const float query2 = q * q; const float key2 = k * k; const int base = index * 4; partials[base] = IsNaNOrInf(dot, 0.0f); partials[base + 1] = IsNaNOrInf(query2, 0.0f); partials[base + 2] = IsNaNOrInf(key2, 0.0f); partials[base + 3] = (!isfinite(dot) || !isfinite(query2) || !isfinite(key2) ? 1.0f : 0.0f); } //+------------------------------------------------------------------+ //| Finalize one D2Skill score per slot after staged reduction. | //+------------------------------------------------------------------+ __kernel void D2SkillScoreFinalize(__global const float *metrics, __global const float *utility, __global const float *used, __global const float *state, __global float *scores, const int slots, const float similarity_threshold, const float utility_weight, const float min_utility, const int utility_policy) { const int slot = get_global_id(0); if(slot < 0 || slot >= slots) return; const int base = slot * 4; const float dot = IsNaNOrInf(metrics[base], 0.0f); const float query2 = IsNaNOrInf(metrics[base + 1], 0.0f); const float key2 = IsNaNOrInf(metrics[base + 2], 0.0f); const float invalid_reduction = IsNaNOrInf(metrics[base + 3], 1.0f); float score = RAG_INVALID_SCORE; const int slot_state = (int)IsNaNOrInf(state[slot], (float)D2SKILL_INACTIVE); if(invalid_reduction <= 0.0f && IsNaNOrInf(used[slot], 0.0f) > 0.5f && slot_state != D2SKILL_INACTIVE && slot_state != D2SKILL_CANDIDATE && query2 > 1.0e-12f && key2 > 1.0e-12f) { const float cosine = dot / (sqrt(query2) * sqrt(key2)); const float slot_utility = clamp(IsNaNOrInf(utility[slot], 0.0f), -1.0f, 1.0f); if(isfinite(cosine) && cosine >= similarity_threshold && (utility_policy == 0 || slot_utility >= min_utility)) { const float candidate_score = cosine + utility_weight * slot_utility; if(isfinite(candidate_score)) score = candidate_score; } } scores[slot] = score; } //+------------------------------------------------------------------+ //| Gather the selected signed correction in parallel by component. | //+------------------------------------------------------------------+ __kernel void D2SkillGatherCorrection(__global const float2 *top, __global const float *corrections, __global const float *directions, __global const float *scales, __global float *selected_correction, __global float *selected_slot, __global float *selected_score, __global float *uses, __global float *diagnostics, const int slots, const int dimension, const int representation, const float alpha, __global float *retrieved, const int mark_retrieved) { const int component = get_global_id(0); const float2 top_slot = top[0]; const int winner = (int)top_slot.x; const float score = IsNaNOrInf(top_slot.y, RAG_INVALID_SCORE); const int valid = (winner >= 0 && winner < slots && score > RAG_INVALID_SCORE); if(component < dimension) { float correction = 0.0f; if(valid) { const int shift = winner * dimension + component; correction = (representation == 1 ? IsNaNOrInf(directions[shift], 0.0f) * IsNaNOrInf(scales[winner], 0.0f) : IsNaNOrInf(corrections[shift], 0.0f)); correction = IsNaNOrInf(correction * IsNaNOrInf(alpha, 0.0f), 0.0f); } selected_correction[component] = correction; } if(component == 0) { selected_slot[0] = (valid ? (float)winner : -1.0f); selected_score[0] = (valid ? score : RAG_INVALID_SCORE); if(valid && mark_retrieved != 0) { uses[winner] = IsNaNOrInf(uses[winner], 0.0f) + 1.0f; retrieved[winner] = 1.0f; diagnostics[0] = IsNaNOrInf(diagnostics[0], 0.0f) + 1.0f; } else if(!valid && mark_retrieved != 0) { diagnostics[1] = IsNaNOrInf(diagnostics[1], 0.0f) + 1.0f; } } } //+------------------------------------------------------------------+ //| D2Skill metric and decision constants used by the rank kernels. | //+------------------------------------------------------------------+ #define D2SKILL_METRIC_COUNT 10 #define D2SKILL_DIAGNOSTIC_PARTIALS 22 #define D2SKILL_DECISION_NONE 0 #define D2SKILL_DECISION_MATCH 1 #define D2SKILL_DECISION_SEED 2 #define D2SKILL_DECISION_RESET 3 #define D2SKILL_DECISION_CANDIDATE 4 #define D2SKILL_DECISION_FREE 5 #define D2SKILL_DECISION_EVICT 6 //+------------------------------------------------------------------+ //| Implements the D2Skill age lifecycle kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillAgeLifecycle(__global const float *used, __global float *state, __global float *age, __global float *protection, __global const float *utility, __global float *free_scores, __global float *eviction_scores, const int slots, const int inactivity_age, __global float *retrieved, __global float *candidate_scores) { const int slot = get_global_id(0); if(slot >= slots) return; const float is_used = IsNaNOrInf(used[slot], 0.0f); int slot_state = (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE); float slot_age = max(IsNaNOrInf(age[slot], 0.0f), 0.0f); float slot_protection = max(IsNaNOrInf(protection[slot], 0.0f), 0.0f); if(is_used > 0.5f && slot_state != D2SKILL_CANDIDATE) { const bool was_retrieved = (IsNaNOrInf(retrieved[slot], 0.0f) > 0.5f); slot_age = (was_retrieved ? 0.0f : min(slot_age + 1.0f, (float)MAX_VALUE)); retrieved[slot] = 0.0f; slot_protection = max(slot_protection - 1.0f, 0.0f); if(slot_state == D2SKILL_PROTECTED && slot_protection <= 0.0f) slot_state = D2SKILL_ACTIVE; if(slot_state == D2SKILL_ACTIVE && slot_age >= (float)max(1, inactivity_age)) slot_state = D2SKILL_INACTIVE; state[slot] = (float)slot_state; age[slot] = slot_age; protection[slot] = slot_protection; } free_scores[slot] = (is_used <= 0.5f && slot_state == D2SKILL_FREE ? -(float)slot : RAG_INVALID_SCORE); eviction_scores[slot] = (is_used > 0.5f && slot_state == D2SKILL_INACTIVE && slot_protection <= 0.0f ? -clamp(IsNaNOrInf(utility[slot], 0.0f), -1.0f, 1.0f) + 1.0e-4f * slot_age : RAG_INVALID_SCORE); candidate_scores[slot] = (is_used > 0.5f && slot_state == D2SKILL_CANDIDATE ? -(float)slot : RAG_INVALID_SCORE); } //+------------------------------------------------------------------+ //| Implements the D2Skill metric partial kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillMetricPartial(__global const float *query, __global const float *gradient, __global const float *selected_slot, __global const float *keys, __global const float *directions, __global const float *used, __global const float2 *candidate_top, __global const float *candidate_key, __global const float *candidate_direction, __global float *partials, const int slots, const int dimension) { const int component = get_global_id(0); if(component < 0 || component >= dimension) return; const int selected = (int)round(selected_slot[0]); const int selected_valid = (selected >= 0 && selected < slots && IsNaNOrInf(used[selected], 0.0f) > 0.5f); const int candidate_index = (int)round(candidate_top[0].x); const int candidate = (candidate_index >= 0 && candidate_index < slots && candidate_top[0].y > RAG_INVALID_SCORE ? candidate_index : -1); const float q = IsNaNOrInf(query[component], 0.0f); const float g = IsNaNOrInf(gradient[component], 0.0f); float values[D2SKILL_METRIC_COUNT]; for(int metric = 0; metric < D2SKILL_METRIC_COUNT; metric++) values[metric] = 0.0f; values[0] = IsNaNOrInf(g * g, 0.0f); values[1] = IsNaNOrInf(q * q, 0.0f); if(selected_valid) { const int shift = selected * dimension + component; const float key = IsNaNOrInf(keys[shift], 0.0f); const float direction = IsNaNOrInf(directions[shift], 0.0f); values[2] = IsNaNOrInf(q * key, 0.0f); values[3] = IsNaNOrInf(key * key, 0.0f); values[4] = IsNaNOrInf(g * direction, 0.0f); values[5] = IsNaNOrInf(direction * direction, 0.0f); } if(candidate >= 0) { const int shift = candidate * dimension + component; const float key = IsNaNOrInf(candidate_key[shift], 0.0f); const float direction = IsNaNOrInf(candidate_direction[shift], 0.0f); values[6] = IsNaNOrInf(q * key, 0.0f); values[7] = IsNaNOrInf(key * key, 0.0f); values[8] = IsNaNOrInf(g * direction, 0.0f); values[9] = IsNaNOrInf(direction * direction, 0.0f); } const int base = component * D2SKILL_METRIC_COUNT; for(int metric = 0; metric < D2SKILL_METRIC_COUNT; metric++) partials[base + metric] = values[metric]; } //+------------------------------------------------------------------+ //| Generic radix-8 field reduction with disjoint global outputs. | //+------------------------------------------------------------------+ inline float D2SkillMetricReducePair(const float left, const float right, const int field, const int max_field_a, const int max_field_b) { if(field == max_field_a || field == max_field_b) return(max(left, right)); return(IsNaNOrInf(left + right, 0.0f)); } //+------------------------------------------------------------------+ //| Implements the D2Skill metric reduce kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillMetricReduce(__global float *partials, __global float *metrics, const int batch_count, const int input_count, const int fields, const int input_offset, const int output_offset, const int max_field_a, const int max_field_b, const int final_output) { const int index = get_global_id(0); const int reduction_radix = 8; const int output_count = (input_count + reduction_radix - 1) / reduction_radix; const int total = batch_count * output_count * fields; if(index < 0 || index >= total || input_count <= 0 || fields <= 0) return; const int field = index % fields; const int item_index = index / fields; const int output_index = item_index % output_count; const int batch = item_index / output_count; const int first = reduction_radix * output_index; const int input_base = input_offset + (batch * input_count + first) * fields + field; float value = IsNaNOrInf(partials[input_base], 0.0f); if(first + 1 < input_count) { const float right = IsNaNOrInf(partials[input_base + fields], 0.0f); value = D2SkillMetricReducePair(value, right, field, max_field_a, max_field_b); } if(first + 2 < input_count) { float right = IsNaNOrInf(partials[input_base + 2 * fields], 0.0f); if(first + 3 < input_count) { const float pair = IsNaNOrInf(partials[input_base + 3 * fields], 0.0f); right = D2SkillMetricReducePair(right, pair, field, max_field_a, max_field_b); } value = D2SkillMetricReducePair(value, right, field, max_field_a, max_field_b); } if(first + 4 < input_count) { float right = IsNaNOrInf(partials[input_base + 4 * fields], 0.0f); if(first + 5 < input_count) { const float pair = IsNaNOrInf(partials[input_base + 5 * fields], 0.0f); right = D2SkillMetricReducePair(right, pair, field, max_field_a, max_field_b); } if(first + 6 < input_count) { float pair = IsNaNOrInf(partials[input_base + 6 * fields], 0.0f); if(first + 7 < input_count) { const float pair_right = IsNaNOrInf(partials[input_base + 7 * fields], 0.0f); pair = D2SkillMetricReducePair(pair, pair_right, field, max_field_a, max_field_b); } right = D2SkillMetricReducePair(right, pair, field, max_field_a, max_field_b); } value = D2SkillMetricReducePair(value, right, field, max_field_a, max_field_b); } if(final_output != 0) metrics[batch * fields + field] = value; else partials[output_offset + (batch * output_count + output_index) * fields + field] = value; } //+------------------------------------------------------------------+ //| Implements the D2Skill candidate leader kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillCandidateLeader(__global const float *selected_slot, __global float *used, __global float *scales, __global float *observations, __global float *mass, __global float *age, __global float *candidate_streak, __global float *candidate_valid, __global float *candidate_scale, __global const float *metrics, __global const float2 *free_top, __global const float2 *eviction_top, __global float *decision, __global float *diagnostics, const int slots, const int representation, const float beta_magnitude, const float max_correction, const int min_confirmations, const float similarity_threshold, const float direction_threshold, __global const float2 *candidate_top, __global float *candidate_protection) { if(get_global_id(0) != 0) return; decision[0] = -1.0f; decision[1] = D2SKILL_DECISION_NONE; const float gradient_norm = sqrt(fmax(IsNaNOrInf(metrics[0], 0.0f), 0.0f)); decision[2] = gradient_norm; decision[3] = 0.0f; //--- Collection must preserve a non-zero pseudo-residual even before the //--- untrained Actor has amplified it. Only an exact zero has no direction. if(slots <= 0 || gradient_norm <= 0.0f) return; const float query_norm = sqrt(fmax(IsNaNOrInf(metrics[1], 0.0f), 0.0f)); const int selected = (int)round(selected_slot[0]); const int selected_valid = (selected >= 0 && selected < slots && IsNaNOrInf(used[selected], 0.0f) > 0.5f); const float selected_key_norm = sqrt(fmax(IsNaNOrInf(metrics[3], 0.0f), 0.0f)); const float selected_direction_norm = sqrt(fmax(IsNaNOrInf(metrics[5], 0.0f), 0.0f)); const float key_cosine = (query_norm > 1.0e-8f && selected_key_norm > 1.0e-8f ? metrics[2] / (query_norm * selected_key_norm) : -1.0f); const float direction_cosine = (selected_direction_norm > 1.0e-8f ? metrics[4] / (gradient_norm * selected_direction_norm) : 0.0f); if(selected_valid) { diagnostics[2] = IsNaNOrInf(diagnostics[2], 0.0f) + 1.0f; if(IsNaNOrInf(metrics[5], 0.0f) > 1.0e-8f) { diagnostics[19] = IsNaNOrInf(diagnostics[19], 0.0f) + 1.0f; diagnostics[20] = IsNaNOrInf(diagnostics[20], 0.0f) + direction_cosine; } } if(selected_valid && isfinite(key_cosine) && isfinite(direction_cosine) && key_cosine >= similarity_threshold && direction_cosine >= direction_threshold) { if(representation == 1) scales[selected] = clamp((1.0f - beta_magnitude) * IsNaNOrInf(scales[selected], 0.0f) + beta_magnitude * gradient_norm, 0.0f, max_correction); observations[selected] = IsNaNOrInf(observations[selected], 0.0f) + 1.0f; mass[selected] = 0.95f * IsNaNOrInf(mass[selected], 0.0f) + 0.05f; age[selected] = 0.0f; decision[0] = (float)selected; decision[1] = D2SKILL_DECISION_MATCH; return; } const int candidate_index = (int)round(candidate_top[0].x); int candidate = (candidate_index >= 0 && candidate_index < slots && candidate_top[0].y > RAG_INVALID_SCORE ? candidate_index : -1); if(candidate < 0) { const int free_slot = (int)round(free_top[0].x); const int evict_slot = (int)round(eviction_top[0].x); const bool free_available = (free_slot >= 0 && free_slot < slots && free_top[0].y > RAG_INVALID_SCORE); const bool evict_available = (evict_slot >= 0 && evict_slot < slots && eviction_top[0].y > RAG_INVALID_SCORE); candidate = (free_available ? free_slot : (evict_available ? evict_slot : -1)); if(candidate < 0) { diagnostics[5] = IsNaNOrInf(diagnostics[5], 0.0f) + 1.0f; return; } candidate_valid[candidate] = D2SKILL_CANDIDATE; used[candidate] = 1.0f; candidate_streak[candidate] = 1.0f; candidate_scale[candidate] = gradient_norm; mass[candidate] = gradient_norm; age[candidate] = 0.0f; candidate_protection[candidate] = (free_available ? 0.0f : -1.0f); decision[0] = (float)candidate; decision[1] = D2SKILL_DECISION_SEED; diagnostics[3] = IsNaNOrInf(diagnostics[3], 0.0f) + 1.0f; return; } const float candidate_key_norm = sqrt(fmax(IsNaNOrInf(metrics[7], 0.0f), 0.0f)); const float candidate_direction_norm = sqrt(fmax(IsNaNOrInf(metrics[9], 0.0f), 0.0f)); const float candidate_key_cosine = (query_norm > 1.0e-8f && candidate_key_norm > 1.0e-8f ? metrics[6] / (query_norm * candidate_key_norm) : -1.0f); const float candidate_direction_cosine = (candidate_direction_norm > 1.0e-8f ? metrics[8] / (gradient_norm * candidate_direction_norm) : 0.0f); if(candidate_direction_norm > 1.0e-8f && isfinite(candidate_direction_cosine)) { diagnostics[19] = IsNaNOrInf(diagnostics[19], 0.0f) + 1.0f; diagnostics[20] = IsNaNOrInf(diagnostics[20], 0.0f) + candidate_direction_cosine; } const bool direction_rejected = (IsNaNOrInf(metrics[9], 0.0f) > 1.0e-8f && (!isfinite(candidate_direction_cosine) || candidate_direction_cosine < direction_threshold)); if(!isfinite(candidate_key_cosine) || candidate_key_cosine < similarity_threshold || direction_rejected) { candidate_streak[candidate] = 1.0f; candidate_scale[candidate] = gradient_norm; age[candidate] = 0.0f; decision[0] = (float)candidate; decision[1] = D2SKILL_DECISION_RESET; return; } candidate_streak[candidate] = IsNaNOrInf(candidate_streak[candidate], 0.0f) + 1.0f; candidate_scale[candidate] = clamp((1.0f - beta_magnitude) * IsNaNOrInf(candidate_scale[candidate], 0.0f) + beta_magnitude * gradient_norm, 0.0f, max_correction); decision[0] = (float)candidate; if(candidate_streak[candidate] < (float)max(1, min_confirmations)) { decision[1] = D2SKILL_DECISION_CANDIDATE; return; } decision[1] = (candidate_protection[candidate] < 0.0f ? D2SKILL_DECISION_EVICT : D2SKILL_DECISION_FREE); } //+------------------------------------------------------------------+ //| Implements the D2Skill vector update kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillVectorUpdate(__global const float *query, __global const float *gradient, __global float *keys, __global float *corrections, __global float *directions, __global const float *scales, __global float *candidate_key, __global float *candidate_direction, __global const float *candidate_scale, __global const float *decision, __global const float *reduced_norm, const int dimension, const int representation, const float beta_correction, const float beta_direction, const float beta_key, const float max_correction, const int phase) { const int component = get_global_id(0); if(component >= dimension || dimension <= 0) return; const int action = (int)round(decision[1]); const int target = (int)round(decision[0]); const float gradient_norm = decision[2]; const float query_value = IsNaNOrInf(query[component], 0.0f); const float gradient_value = IsNaNOrInf(gradient[component], 0.0f); const float unit = (gradient_norm > 0.0f ? gradient_value / gradient_norm : 0.0f); if(phase == 1) { if((action != D2SKILL_DECISION_MATCH && action != D2SKILL_DECISION_FREE && action != D2SKILL_DECISION_EVICT) || target < 0) return; const int shift = target * dimension + component; const float vector_norm = sqrt(fmax(IsNaNOrInf(reduced_norm[0], 0.0f), 0.0f)); if(representation == 0) { const float factor = (vector_norm > max_correction && vector_norm > 1.0e-12f ? max_correction / vector_norm : 1.0f); corrections[shift] = IsNaNOrInf(corrections[shift] * factor, 0.0f); } else { const float factor = (vector_norm > 1.0e-12f ? 1.0f / vector_norm : 0.0f); directions[shift] = IsNaNOrInf(directions[shift] * factor, 0.0f); corrections[shift] = IsNaNOrInf(directions[shift] * fmin(scales[target], max_correction), 0.0f); } return; } if((action == D2SKILL_DECISION_MATCH || action == D2SKILL_DECISION_FREE || action == D2SKILL_DECISION_EVICT) && target >= 0) { const int shift = target * dimension + component; keys[shift] = IsNaNOrInf((1.0f - beta_key) * keys[shift] + beta_key * query_value, 0.0f); if(representation == 0) corrections[shift] = IsNaNOrInf((1.0f - beta_correction) * IsNaNOrInf(corrections[shift], 0.0f) + beta_correction * gradient_value, 0.0f); else directions[shift] = IsNaNOrInf((1.0f - beta_direction) * IsNaNOrInf(directions[shift], 0.0f) + beta_direction * unit, 0.0f); return; } if((action == D2SKILL_DECISION_SEED || action == D2SKILL_DECISION_RESET) && target >= 0) { const int shift = target * dimension + component; keys[shift] = query_value; directions[shift] = unit; return; } if((action == D2SKILL_DECISION_CANDIDATE || action == D2SKILL_DECISION_FREE || action == D2SKILL_DECISION_EVICT) && target >= 0) { const int shift = target * dimension + component; keys[shift] = IsNaNOrInf((1.0f - beta_key) * keys[shift] + beta_key * query_value, 0.0f); directions[shift] = IsNaNOrInf((1.0f - beta_direction) * directions[shift] + beta_direction * unit, 0.0f); return; } } //+------------------------------------------------------------------+ //| Applies the online direction EMA to one retrieved stable slot. | //| It intentionally does not touch keys, correction, scale or | //| lifecycle. | //+------------------------------------------------------------------+ __kernel void D2SkillDirectionEMA(__global const float *gradient, __global float *directions, __global const float *selected_slot, __global const float *used, __global const float *state, __global float *reduced_norm, const int slots, const int dimension, const float beta_direction, __global float *partials, const int phase) { const int component = get_global_id(0); if(component < 0 || component >= dimension || slots <= 0) return; const int slot = (int)round(IsNaNOrInf(selected_slot[0], -1.0f)); if(slot < 0 || slot >= slots || IsNaNOrInf(used[slot], 0.0f) <= 0.5f) return; const int slot_state = (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE); if(slot_state == D2SKILL_FREE || slot_state == D2SKILL_CANDIDATE || slot_state == D2SKILL_INACTIVE) return; const int index = slot * dimension + component; float gradient_squared = IsNaNOrInf(reduced_norm[0], 0.0f); if(phase == 0 && component == 0) reduced_norm[1] = gradient_squared; if(phase == 1) gradient_squared = IsNaNOrInf(reduced_norm[1], 0.0f); const float gradient_norm = sqrt(fmax(gradient_squared, 0.0f)); const float unit = (gradient_norm > 1.0e-12f ? IsNaNOrInf(gradient[component], 0.0f) / gradient_norm : 0.0f); const float candidate = IsNaNOrInf((1.0f - beta_direction) * IsNaNOrInf(directions[index], 0.0f) + beta_direction * unit, 0.0f); if(phase == 0) { partials[component] = IsNaNOrInf(candidate * candidate, 0.0f); return; } if(phase == 1) { const float candidate_norm = sqrt(fmax(IsNaNOrInf(reduced_norm[0], 0.0f), 0.0f)); if(candidate_norm > 1.0e-12f) directions[index] = IsNaNOrInf(candidate / candidate_norm, 0.0f); } } //+------------------------------------------------------------------+ //| Reduces and normalizes only the retrieved stable direction row. | //| Phase 0 emits per-component squares; phase 1 consumes its staged | //| reduction. No durable buffer except directions is written. | //+------------------------------------------------------------------+ __kernel void D2SkillNormalizeDirection(__global float *directions, __global const float *selected_slot, __global const float *used, __global const float *state, __global float *partials, __global const float *reduced_norm, const int slots, const int dimension, const int phase) { const int component = get_global_id(0); if(component < 0 || component >= dimension || slots <= 0) return; const int slot = (int)round(IsNaNOrInf(selected_slot[0], -1.0f)); if(slot < 0 || slot >= slots || IsNaNOrInf(used[slot], 0.0f) <= 0.5f) return; const int slot_state = (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE); if(slot_state == D2SKILL_FREE || slot_state == D2SKILL_CANDIDATE || slot_state == D2SKILL_INACTIVE) return; const int index = slot * dimension + component; if(phase == 0) { const float value = IsNaNOrInf(directions[index], 0.0f); partials[component] = IsNaNOrInf(value * value, 0.0f); return; } if(phase == 1) { const float norm = sqrt(fmax(IsNaNOrInf(reduced_norm[0], 0.0f), 0.0f)); if(norm > 1.0e-12f) directions[index] = IsNaNOrInf(directions[index] / norm, 0.0f); else directions[index] = (component == 0 ? 1.0f : 0.0f); } } //+------------------------------------------------------------------+ //| Implements the D2Skill bank vector partial kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillBankVectorPartial(__global const float *corrections, __global const float *directions, __global const float *decision, __global float *partials, const int dimension, const int representation, const int output_offset) { const int component = get_global_id(0); if(component < 0 || component >= dimension) return; const int target = (int)round(decision[0]); const int action = (int)round(decision[1]); float value = 0.0f; if((action == D2SKILL_DECISION_MATCH || action == D2SKILL_DECISION_FREE || action == D2SKILL_DECISION_EVICT) && target >= 0) { const int shift = target * dimension + component; value = (representation == 0 ? IsNaNOrInf(corrections[shift], 0.0f) : IsNaNOrInf(directions[shift], 0.0f)); } partials[output_offset + component] = IsNaNOrInf(value * value, 0.0f); } //+------------------------------------------------------------------+ //| Implements the D2Skill promote slot kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillPromoteSlot(__global float *scales, __global float *utility, __global float *observations, __global float *uses, __global float *mass, __global float *used, __global float *state, __global float *age, __global float *protection, __global float *candidate_streak, __global float *candidate_valid, __global const float *candidate_scale, __global const float *decision, __global float *diagnostics, const int slots, const int protection_age) { if(get_global_id(0) != 0) return; const int action = (int)round(decision[1]); const int target = (int)round(decision[0]); if((action != D2SKILL_DECISION_FREE && action != D2SKILL_DECISION_EVICT) || target < 0 || target >= slots || (int)IsNaNOrInf(candidate_valid[target], (float)D2SKILL_FREE) != D2SKILL_CANDIDATE) return; scales[target] = IsNaNOrInf(candidate_scale[target], 0.0f); utility[target] = 0.0f; observations[target] = IsNaNOrInf(candidate_streak[target], 1.0f); uses[target] = 0.0f; mass[target] = IsNaNOrInf(candidate_streak[target], 1.0f); used[target] = 1.0f; state[target] = D2SKILL_PROTECTED; age[target] = 0.0f; protection[target] = (float)protection_age; if(action == D2SKILL_DECISION_EVICT) diagnostics[7] = IsNaNOrInf(diagnostics[7], 0.0f) + 1.0f; else diagnostics[4] = IsNaNOrInf(diagnostics[4], 0.0f) + 1.0f; } //+------------------------------------------------------------------+ //| Implements the D2Skill diagnostics norm partial kernel on the GPU| //+------------------------------------------------------------------+ __kernel void D2SkillDiagnosticsNormPartial(__global const float *corrections, __global float *partials, const int slots, const int dimension) { const int index = get_global_id(0); const int total = slots * dimension; if(index < 0 || index >= total) return; const float value = IsNaNOrInf(corrections[index], 0.0f); partials[index] = IsNaNOrInf(value * value, 0.0f); } //+------------------------------------------------------------------+ //| Implements the D2Skill diagnostics partial kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillDiagnosticsPartial(__global const float *norms, __global const float *utility, __global const float *uses, __global const float *used, __global const float *state, __global const float *age, __global float *partials, const int slots) { const int slot = get_global_id(0); if(slot < 0 || slot >= slots) return; const int base = slot * D2SKILL_DIAGNOSTIC_PARTIALS; for(int item = 0; item < D2SKILL_DIAGNOSTIC_PARTIALS; item++) partials[base + item] = 0.0f; const float slot_used = IsNaNOrInf(used[slot], 0.0f); const int slot_state = (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE); if(slot_used <= 0.5f && slot_state == D2SKILL_FREE) partials[base] = 1.0f; if(slot_state == D2SKILL_PROTECTED) partials[base + 1] = 1.0f; if(slot_state == D2SKILL_ACTIVE) partials[base + 2] = 1.0f; if(slot_state == D2SKILL_INACTIVE) partials[base + 3] = 1.0f; if(slot_state == D2SKILL_CANDIDATE) partials[base + 21] = 1.0f; if(slot_used <= 0.5f) return; const float slot_age = max(IsNaNOrInf(age[slot], 0.0f), 0.0f); const float slot_usage = max(IsNaNOrInf(uses[slot], 0.0f), 0.0f); partials[base + 4] = 1.0f; partials[base + 5] = IsNaNOrInf(utility[slot], 0.0f); partials[base + 6] = sqrt(fmax(IsNaNOrInf(norms[slot], 0.0f), 0.0f)); partials[base + 7] = slot_age; partials[base + 8] = slot_usage; partials[base + 9] = slot_age; partials[base + 10] = slot_usage; int utility_bin = clamp((int)floor((clamp(IsNaNOrInf(utility[slot], 0.0f), -1.0f, 1.0f) + 1.0f) * 2.5f), 0, 4); const int usage_bin = (slot_usage < 1.0f ? 0 : (slot_usage < 2.0f ? 1 : (slot_usage < 4.0f ? 2 : (slot_usage < 8.0f ? 3 : 4)))); partials[base + 11 + utility_bin] = 1.0f; partials[base + 16 + usage_bin] = 1.0f; } //+------------------------------------------------------------------+ //| Implements the D2Skill diagnostics reduce kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillDiagnosticsReduce(__global const float *totals, __global float *diagnostics, __global float *utility_distribution, __global float *usage_distribution, const int slots) { const int output = get_global_id(0); const float used_count = IsNaNOrInf(totals[4], 0.0f); switch(output) { case 0: diagnostics[8] = (float)slots; break; case 1: diagnostics[9] = IsNaNOrInf(totals[0], 0.0f); break; case 2: diagnostics[10] = (IsNaNOrInf(totals[21], 0.0f) > 0.0f ? 1.0f : 0.0f); break; case 3: diagnostics[11] = IsNaNOrInf(totals[1], 0.0f); break; case 4: diagnostics[12] = IsNaNOrInf(totals[2], 0.0f); break; case 5: diagnostics[13] = IsNaNOrInf(totals[3], 0.0f); break; case 6: diagnostics[14] = (used_count > 0.0f ? totals[5] / used_count : 0.0f); break; case 7: diagnostics[15] = (used_count > 0.0f ? totals[6] / used_count : 0.0f); break; case 8: diagnostics[16] = (diagnostics[19] > 0.0f ? diagnostics[20] / diagnostics[19] : 0.0f); break; case 9: diagnostics[17] = (used_count > 0.0f ? totals[7] / used_count : 0.0f); break; case 10: diagnostics[18] = (used_count > 0.0f ? totals[8] / used_count : 0.0f); break; case 11: diagnostics[21] = IsNaNOrInf(totals[9], 0.0f); break; case 12: diagnostics[22] = IsNaNOrInf(totals[10], 0.0f); break; default: if(output >= 13 && output < 18) utility_distribution[output - 13] = IsNaNOrInf(totals[output - 2], 0.0f); else if(output >= 18 && output < 23) usage_distribution[output - 18] = IsNaNOrInf(totals[output - 2], 0.0f); break; } } //+------------------------------------------------------------------+ //| Implements the D2Skill influence total kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillInfluenceTotal(__global const float *influence, __global const float *used, __global const float *state, __global float *partials, const int slots) { const int slot = get_global_id(0); if(slot < 0 || slot >= slots) return; const bool eligible = (IsNaNOrInf(used[slot], 0.0f) > 0.5f && (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE) != D2SKILL_CANDIDATE); partials[slot] = (eligible ? fmax(IsNaNOrInf(influence[slot], 0.0f), 0.0f) : 0.0f); } //+------------------------------------------------------------------+ //| Gradient contribution for the selected forward correction. | //+------------------------------------------------------------------+ __kernel void D2SkillInfluencePartial(__global const float *gradient, __global const float *correction, __global float *partials, const int dimension) { const int component = get_global_id(0); if(component < 0 || component >= dimension) return; partials[component] = IsNaNOrInf(IsNaNOrInf(gradient[component], 0.0f) * IsNaNOrInf(correction[component], 0.0f), 0.0f); } //+------------------------------------------------------------------+ //| Single writer after staged reduction: no atomics are required. | //+------------------------------------------------------------------+ __kernel void D2SkillInfluenceAccumulate(__global const float *selected_slot, __global const float *reduced_influence, __global float *influence, const int slots) { if(get_global_id(0) != 0 || slots <= 0) return; const int slot = (int)round(IsNaNOrInf(selected_slot[0], -1.0f)); if(slot < 0 || slot >= slots) return; influence[slot] = IsNaNOrInf(influence[slot], 0.0f) + IsNaNOrInf(reduced_influence[0], 0.0f); } //+------------------------------------------------------------------+ //| Implements the D2Skill reset influence kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillResetInfluence(__global float *influence, const int slots) { const int slot = get_global_id(0); if(slot < slots) influence[slot] = 0.0f; } //+------------------------------------------------------------------+ //| Implements the D2Skill utility kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void D2SkillUtility(__global float *influence, __global const float *influence_total, __global float *utility, __global const float *used, __global const float *state, const float delta_j, const float beta, const int slots, __global float *diagnostics, __global float *utility_applied) { const int slot = get_global_id(0); if(slot < 0 || slot >= slots || slots <= 0) return; utility_applied[slot] = 0.0f; const float mass = fmax(IsNaNOrInf(influence[slot], 0.0f), 0.0f); const float total = fmax(IsNaNOrInf(influence_total[0], 0.0f), 0.0f); const bool eligible = (IsNaNOrInf(used[slot], 0.0f) > 0.5f && (int)IsNaNOrInf(state[slot], (float)D2SKILL_FREE) != D2SKILL_CANDIDATE); if(eligible && total > 0.0f && mass > 0.0f && fabs(delta_j) > 0.0f) { const float utility_before = utility[slot]; const float utility_current = IsNaNOrInf(utility_before, 0.0f); const float utility_target = clamp(IsNaNOrInf(delta_j, 0.0f), -1.0f, 1.0f) * mass / total; const float utility_proposed = (1.0f - beta) * utility_current + beta * utility_target; utility[slot] = utility_proposed; if(utility_before != utility_proposed) utility_applied[slot] = 1.0f; } influence[slot] = 0.0f; if(slot == 0 && total > 0.0f && fabs(delta_j) > 0.0f) diagnostics[6] = IsNaNOrInf(diagnostics[6], 0.0f) + 1.0f; } //+------------------------------------------------------------------+ //| OMPB regularizers: elementwise reductions only | //+------------------------------------------------------------------+ __kernel void OMPBDisagreement(__global const float *reference, __global const float *current, __global float *output, const int source_states, const int target_states, const int samples, const float inv_tau2, __global const float *gate, __global float *alpha_gradient, const int gate_stride, const float multiplier, __global float *numeric_flags, const int flags_enabled) { const int lane = (int)get_local_id(0); const int variable = (int)get_global_id(1); __local float source_scratch[LOCAL_ARRAY_SIZE]; __local float target_scratch[LOCAL_ARRAY_SIZE]; __local float source_alpha_scratch[LOCAL_ARRAY_SIZE]; __local float target_alpha_scratch[LOCAL_ARRAY_SIZE]; __local float invalid_scratch[LOCAL_ARRAY_SIZE]; float source_sum = 0.0f; float target_sum = 0.0f; float source_alpha = 0.0f; float target_alpha = 0.0f; int invalid = 0; if(source_states > 0 && target_states > 0 && samples > 1 && gate_stride > 0) { const float raw_gate = gate[variable * gate_stride]; if(flags_enabled) invalid |= isnan(raw_gate) || isinf(raw_gate) || isnan(inv_tau2) || isinf(inv_tau2) || isnan(multiplier) || isinf(multiplier); const float g = IsNaNOrInf(raw_gate, 0.0f); const float gate2 = g * g; const float gate_alpha = 2.0f * gate2 * (1.0f - g); if(flags_enabled) invalid |= isnan(gate2) || isinf(gate2) || isnan(gate_alpha) || isinf(gate_alpha); const int source_total = source_states * samples * samples; const int target_total = target_states * samples * samples; //--- Traverse source states and unordered posterior-sample pairs. for(int index = lane; index < source_total; index += get_local_size(0)) { const int sample_index = index % (samples * samples); const int left = sample_index / samples; const int right = sample_index % samples; if(left < right) { const float raw = reference[(variable * source_states * samples * samples) + index]; const float raw_scaled = raw * gate2 * inv_tau2; if(flags_enabled) invalid |= isnan(raw) || isinf(raw) || isnan(raw_scaled) || isinf(raw_scaled); const float scaled = IsNaNOrInf(raw_scaled, 1.0f); source_sum += fmin(scaled, 1.0f); if(flags_enabled) invalid |= isnan(source_sum) || isinf(source_sum); if(scaled < 1.0f) { const float raw_alpha = gate_alpha * raw * inv_tau2; if(flags_enabled) invalid |= isnan(raw_alpha) || isinf(raw_alpha); source_alpha += raw_alpha; if(flags_enabled) invalid |= isnan(source_alpha) || isinf(source_alpha); } } } //--- Traverse target states and unordered posterior-sample pairs. for(int index = lane; index < target_total; index += get_local_size(0)) { const int sample_index = index % (samples * samples); const int left = sample_index / samples; const int right = sample_index % samples; if(left < right) { const float raw = current[(variable * target_states * samples * samples) + index]; const float raw_scaled = raw * gate2 * inv_tau2; if(flags_enabled) invalid |= isnan(raw) || isinf(raw) || isnan(raw_scaled) || isinf(raw_scaled); const float scaled = IsNaNOrInf(raw_scaled, 1.0f); target_sum += fmin(scaled, 1.0f); if(flags_enabled) invalid |= isnan(target_sum) || isinf(target_sum); if(scaled < 1.0f) { const float raw_alpha = gate_alpha * raw * inv_tau2; if(flags_enabled) invalid |= isnan(raw_alpha) || isinf(raw_alpha); target_alpha += raw_alpha; if(flags_enabled) invalid |= isnan(target_alpha) || isinf(target_alpha); } } } } //--- Reduce source and target statistics through independent local scratch. source_sum = LocalSum(source_sum, 0, source_scratch); target_sum = LocalSum(target_sum, 0, target_scratch); source_alpha = LocalSum(source_alpha, 0, source_alpha_scratch); target_alpha = LocalSum(target_alpha, 0, target_alpha_scratch); if(flags_enabled) { invalid |= isnan(source_scratch[0]) || isinf(source_scratch[0]) || isnan(target_scratch[0]) || isinf(target_scratch[0]) || isnan(source_alpha_scratch[0]) || isinf(source_alpha_scratch[0]) || isnan(target_alpha_scratch[0]) || isinf(target_alpha_scratch[0]); invalid = (LocalSum((float)invalid, 0, invalid_scratch) > 0.0f ? 1 : 0); } //--- Finalize the per-variable disagreement and alpha update direction. if(lane == 0) { float delta = 0.0f; float alpha_direction = 0.0f; if(source_states > 0 && target_states > 0 && samples > 1 && gate_stride > 0) { const int pairs = samples * (samples - 1) / 2; const float source_mean = source_sum / (float)(source_states * pairs); const float target_mean = target_sum / (float)(target_states * pairs); const float raw_delta = source_mean - target_mean; if(flags_enabled) invalid |= isnan(source_mean) || isinf(source_mean) || isnan(target_mean) || isinf(target_mean) || isnan(raw_delta) || isinf(raw_delta); delta = IsNaNOrInf(raw_delta, 0.0f); const float sign_delta = (delta > 0.0f ? 1.0f : (delta < 0.0f ? -1.0f : 0.0f)); const float raw_alpha_direction = multiplier * sign_delta * (source_alpha / (float)(source_states * pairs) - target_alpha / (float)(target_states * pairs)); if(flags_enabled) invalid |= isnan(raw_alpha_direction) || isinf(raw_alpha_direction); alpha_direction = raw_alpha_direction; } output[variable] = delta; const float final_alpha = IsNaNOrInf(alpha_direction, 0.0f); if(flags_enabled) invalid |= isnan(final_alpha) || isinf(final_alpha); alpha_gradient[variable] = final_alpha; if(flags_enabled) numeric_flags[variable] = (invalid ? 1.0f : 0.0f); } } //+------------------------------------------------------------------+ //| Computes the OMPB disagreement grad gradient contributions on | //| GPU. | //+------------------------------------------------------------------+ __kernel void OMPBDisagreementGrad(__global const float *corrections, __global const float *distance, __global const float *disagreement, __global float *gradient, const int states, const int variables, const int samples, const int dimension, const int source_sign, const float multiplier, const float inv_tau2, __global const float *gate, const int gate_stride, __global float *numeric_flags, const int flags_enabled) { const int group = (int)get_global_id(0); const int sample = (int)get_global_id(1); const int coordinate = (int)get_global_id(2); //--- All buffers are indexed as [group, sample, coordinate] with fixed extents. if(states <= 0 || variables <= 0 || samples <= 0 || dimension <= 0 || group < 0 || sample < 0 || coordinate < 0 || group >= states * variables || sample >= samples || coordinate >= dimension) return; const int variable = group / states; const int distance_base = group * samples * samples; const int correction_base = (group * samples + sample) * dimension + coordinate; int invalid = 0; const float raw_delta = disagreement[variable]; const float raw_correction = corrections[correction_base]; if(flags_enabled) invalid |= isnan(raw_delta) || isinf(raw_delta) || isnan(raw_correction) || isinf(raw_correction) || isnan(multiplier) || isinf(multiplier) || isnan(inv_tau2) || isinf(inv_tau2); const float delta = IsNaNOrInf(raw_delta, 0.0f); if(gate_stride <= 0) { if(flags_enabled) numeric_flags[correction_base] = 1.0f; return; } const float raw_gate = gate[variable * gate_stride]; if(flags_enabled) invalid |= isnan(raw_gate) || isinf(raw_gate); const float g = IsNaNOrInf(raw_gate, 0.0f); const float gate2 = g * g; if(flags_enabled) invalid |= isnan(gate2) || isinf(gate2); if(samples == 1) { if(flags_enabled) numeric_flags[correction_base] = (invalid ? 1.0f : 0.0f); return; } const int pairs = samples * (samples - 1) / 2; //--- delta = D_S - D_T; its sign drives the pair direction. The buffer is //--- appended (+=) and must be pre-zeroed by the wrapper. In line with the //--- library contract (optimizers add gradient buffers), the multiplier //--- carries the negative derivative of the mismatch penalty. const float sign_delta = (delta > 0.0f ? 1.0f : (delta < 0.0f ? -1.0f : 0.0f)); float value = 0.0f; const float correction = IsNaNOrInf(raw_correction, 0.0f); //--- Saturated pair-distance terms do not contribute to gradient. for(int other = 0; other < samples; other++) { if(other == sample) continue; const int didx = distance_base + sample * samples + other; const float raw_distance = distance[didx]; const float raw_pair_distance = raw_distance * gate2 * inv_tau2; if(flags_enabled) invalid |= isnan(raw_distance) || isinf(raw_distance) || isnan(raw_pair_distance) || isinf(raw_pair_distance); const float pair_distance = IsNaNOrInf(raw_pair_distance, 1.0f); if(pair_distance >= 1.0f) continue; const int other_index = (group * samples + other) * dimension + coordinate; const float raw_other = corrections[other_index]; const float raw_difference = raw_correction - raw_other; const float raw_pair_value = 2.0f * gate2 * raw_difference; if(flags_enabled) invalid |= isnan(raw_other) || isinf(raw_other) || isnan(raw_difference) || isinf(raw_difference) || isnan(raw_pair_value) || isinf(raw_pair_value); value += 2.0f * gate2 * (correction - IsNaNOrInf(raw_other, 0.0f)); if(flags_enabled) invalid |= isnan(value) || isinf(value); } const float direction_factor = multiplier * (float)source_sign * sign_delta * inv_tau2 / (float)(states * pairs); const float raw_direction = value * direction_factor; if(flags_enabled) invalid |= isnan(direction_factor) || isinf(direction_factor) || isnan(raw_direction) || isinf(raw_direction); const float raw_gradient = gradient[correction_base]; const float final_gradient = raw_gradient + clamp(IsNaNOrInf(raw_direction, 0.0f), -MAX_GRAD, MAX_GRAD); if(flags_enabled) invalid |= isnan(raw_gradient) || isinf(raw_gradient) || isnan(final_gradient) || isinf(final_gradient); gradient[correction_base] = final_gradient; if(flags_enabled) numeric_flags[correction_base] = (invalid ? 1.0f : 0.0f); } //+------------------------------------------------------------------+ //| Implements the OMPB gaussian kl kernel on the GPU. | //+------------------------------------------------------------------+ __kernel void OMPBGaussianKL(__global const float *mean, __global const float *logvar, __global float *output, const int total) { const int id = (int)get_global_id(0); if(id < total) { const float mu = IsNaNOrInf(mean[id], 0.0f); const float lv = IsNaNOrInf(logvar[id], 0.0f); //--- expm1(lv) = exp(lv) - 1 avoids cancellation for small lv; //--- the exact KL is non-negative, so guard float round-off. output[id] = fmax(0.5f * (expm1(lv) - lv + mu * mu), 0.0f); } } //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| FQF Monotone Rearrange (new independent primitive, v1 bitonic) | //+------------------------------------------------------------------+ //| Weighted discrete monotone rearrangement of one FQF distribution | //| per work-group. N = 2^k local work-items (bitonic network); the | //| canonical size is N = 32 and N must fit LOCAL_ARRAY_SIZE and the | //| device max work-group size. Builds the permutation I so that | //| Z*_j = Z_{I_j}, P*_j = P_{I_j}, Z*_0 <= ... <= Z*_{N-1}, | //| using the deterministic comparator (z_i,i) < (z_j,j): first the | //| centroid value, ties by the ORIGINAL index. The probability | //| mass always travels together with its centroid; the raw FQF | //| buffers are never modified (only the ordered copies are written | //| and the SRM rank-sensitive path consumes them). Non-finite | //| centroids map to +MAX_VALUE (trailing) so the network stays | //| total; the MQL5 caller rejects such distributions before the | //| SRM launch per the numerical contract. | //| Diagnostics (6 floats per task, computed on the ORIGINAL order):| //| [0] crossing_rate = mean_i 1[z_{i+1} < z_i] | //| [1] crossing_magnitude = mean_i max(z_i - z_{i+1}, 0) | //| [2] prob_sum = sum_i p_i | //| [3] prob_min = min_i p_i (sanitized, invalid -> 0) | //| [4] prob_max = max_i p_i (sanitized, invalid -> 0) | //| [5] invalid_probability_count = sum_i (not finite(p_i) or p<0)| //+------------------------------------------------------------------+ __kernel void FQF_MonotoneRearrange(__global float *quantiles, __global float *probabilities, __global float *ordered_quantiles, __global float *ordered_probabilities, __global float *order, __global float *diagnostics) { const size_t i = get_local_id(0); const size_t total = get_local_size(0); const size_t task = get_group_id(1); const size_t tasks = get_num_groups(1); //--- geometry contract: one work-group per task, N = 2^k, N <= 64 if(total == 0 || (total & (total - 1)) != 0 || total > LOCAL_ARRAY_SIZE) return; //--- __local float lz[LOCAL_ARRAY_SIZE]; __local float lp[LOCAL_ARRAY_SIZE]; __local float li[LOCAL_ARRAY_SIZE]; __local float Red[LOCAL_ARRAY_SIZE]; //--- row = task, column = quantile, row width = total (single variable) const size_t index = RCtoFlat(task, i, tasks, total, 0); const float zi = quantiles[index]; const float pi = probabilities[index]; const int z_invalid = !(isfinite(zi)); const int p_invalid = !(isfinite(pi)) || pi < 0.0f; //--- lz[i] = (z_invalid ? MAX_VALUE : zi); lp[i] = (p_invalid ? 0.0f : pi); li[i] = (float)i; //--- every lane reads neighbours/partners below; make the initial local //--- representation visible to the whole work-group before diagnostics //--- and before the first bitonic compare-exchange stage. BarrierLoc; //--- diagnostics on the ORIGINAL order (raw local copies before the //--- bitonic network runs) const float crossing_flag = (i + 1 < total && lz[i + 1] < lz[i] ? 1.0f : 0.0f); const float crossing_mag = (i + 1 < total ? fmax(lz[i] - lz[i + 1], 0.0f) : 0.0f); const float denom_inv = (total > 1 ? 1.0f / (float)(total - 1) : 0.0f); //--- const float crossing_rate = LocalSum(crossing_flag, 0, Red) * denom_inv; BarrierLoc; const float crossing_total = LocalSum(crossing_mag, 0, Red) * denom_inv; BarrierLoc; const float prob_sum = LocalSum(lp[i], 0, Red); BarrierLoc; const float prob_min = LocalMin(lp[i], 0, Red); BarrierLoc; const float prob_max = LocalMax(lp[i], 0, Red); BarrierLoc; const float invalid_count = LocalSum((float)p_invalid, 0, Red); BarrierLoc; //--- bitonic merge network sorting (z, index) ascending for(size_t width = 2; width <= total; width <<= 1) for(size_t stride = width >> 1; stride > 0; stride >>= 1) { const size_t partner = i ^ stride; if(partner > i) { const int ascending = ((i & width) == 0); const float az = lz[i]; const float bz = lz[partner]; const float ai = li[i]; const float bi = li[partner]; const int before = (az < bz) || (az == bz && ai < bi); if((ascending && !before) || (!ascending && before)) { const float tz = lz[i]; lz[i] = lz[partner]; lz[partner] = tz; const float tp = lp[i]; lp[i] = lp[partner]; lp[partner] = tp; const float ti = li[i]; li[i] = li[partner]; li[partner] = ti; } } BarrierLoc; } //--- ordered outputs: provenance kept per item (order[j] = I_j) const size_t out_index = RCtoFlat(task, i, tasks, total, 0); ordered_quantiles[out_index] = lz[i]; ordered_probabilities[out_index] = lp[i]; order[out_index] = li[i]; //--- diagnostics written by lane 0 (6 floats per task) if(i == 0) { const size_t d = task * 6; diagnostics[d + 0] = crossing_rate; diagnostics[d + 1] = crossing_total; diagnostics[d + 2] = prob_sum; diagnostics[d + 3] = prob_min; diagnostics[d + 4] = prob_max; diagnostics[d + 5] = invalid_count; } } //+------------------------------------------------------------------+ //| FQF Rearrange Gradient (new independent primitive) | //+------------------------------------------------------------------+ //| Undoes ::FQF_MonotoneRearrange in the backward pass by scattering| //| the sorted-pair gradients back to the original FQF indexing: | //| g_Z[I_j] = g_{Z*},j, g_P[I_j] = g_{P*},j. | //| One work-item per (task, sorted position j); the order buffer | //| stores the original index I_j of every sorted position. An | //| out-of-range permutation entry is rejected (no write) and the | //| MQL5 side treats a bad permutation as an invalid SRM state. | //+------------------------------------------------------------------+ __kernel void FQF_RearrangeGradient(__global float *ordered_quantiles_gradient, __global float *ordered_probabilities_gradient, __global float *order, __global float *quantiles_gradient, __global float *probabilities_gradient) { const size_t j = get_global_id(0); const size_t total = get_global_size(0); const size_t task = get_global_id(1); const size_t tasks = get_global_size(1); //--- if(total == 0 || tasks == 0 || j >= total || task >= tasks) return; //--- row = task, column = quantile, row width = total (single variable) const size_t index = RCtoFlat(task, j, tasks, total, 0); const int dst_index = (int)IsNaNOrInf(order[index], -1.0f); if(dst_index < 0 || dst_index >= (int)total) return; //--- each (task, j) writes exactly one destination; order is a //--- permutation, so destinations are disjoint const size_t dst = RCtoFlat(task, (size_t)dst_index, tasks, total, 0); quantiles_gradient[dst] = ordered_quantiles_gradient[index]; probabilities_gradient[dst] = ordered_probabilities_gradient[index]; } //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| DPCC pure account transition and independent scenario rollouts | //+------------------------------------------------------------------+ #define DPCC_RISK_COUNT 16 #define DPCC_NATIVE_RISK_CONTINUATION_FLOATS 20 #define DPCC_NATIVE_PENDING_PHASE_FLOATS 21 #define DPCC_STATUS_INVALID 3 #define DPCC_STATUS_NUMERIC 5 #define DPCC_CALC_FOREX 0 #define DPCC_CALC_FUTURES 1 #define DPCC_CALC_CFD 2 #define DPCC_CALC_CFDINDEX 3 #define DPCC_CALC_CFDLEVERAGE 4 #define DPCC_CALC_FOREX_NOLEV 5 #define DPCC_CALC_STOCKS 32 #define DPCC_CALC_EXCH_FUT 33 #define DPCC_CALC_FORTS 34 #define DPCC_CALC_BONDS 35 #define DPCC_CALC_MOEX_STOCKS 36 #define DPCC_CALC_MOEX_BONDS 37 #define DPCC_CALC_COLLATERAL 64 typedef struct { float account[24]; float equity_anchor; float entry_interest; float native_mark; float accrued_interest; float maximum_drawdown; uint currency_digits; int source_action; int exit_kind; int numeric_status; } DPCCAccountState; typedef struct { float maximum[DPCC_RISK_COUNT]; int first_h; int constraint; int phase; int source_action; } DPCCRiskState; #define DPCC_NATIVE_CONTINUATION_FLOATS 33 #define DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX 16777216U //+------------------------------------------------------------------+ //| DPCC scalar domains and native calculation-mode families | //+------------------------------------------------------------------+ int DPCCInteger(float value, float minimum, float maximum) { return(isfinite(value) && value >= minimum && value <= maximum && value == floor(value)); } int DPCCBoolean(float value) { return(value == 0.0f || value == 1.0f); } int DPCCBondMode(int mode) { return(mode == DPCC_CALC_BONDS || mode == DPCC_CALC_MOEX_BONDS); } int DPCCTickMode(int mode) { return(mode == DPCC_CALC_FUTURES || mode == DPCC_CALC_EXCH_FUT || mode == DPCC_CALC_FORTS); } int DPCCSupportedMode(int mode) { return((mode >= DPCC_CALC_FOREX && mode <= DPCC_CALC_FOREX_NOLEV) || (mode >= DPCC_CALC_STOCKS && mode <= DPCC_CALC_MOEX_BONDS) || mode == DPCC_CALC_COLLATERAL); } int DPCCOnGrid(float value, float step, float tolerance) { const float units = value / step; const float rounded = rint(units) * step; return(isfinite(units) && isfinite(rounded) && fabs(value - rounded) <= tolerance); } int DPCCPriceValid(float price, __global const float *instrument) { if(!isfinite(price)) return(0); if(instrument[27] == 1.0f && price < instrument[25]) return(0); if(instrument[28] == 1.0f && price > instrument[26]) return(0); return(1); } //+------------------------------------------------------------------+ //| Native PNL; each conversion has exactly one currency boundary | //+------------------------------------------------------------------+ float DPCCProfit(float signed_lots, float entry, float close, float entry_interest, float interest, float fx_gain, float fx_loss, __global const float *instrument) { const int mode = (int)instrument[11]; if(signed_lots == 0.0f || mode == DPCC_CALC_COLLATERAL) return(0.0f); float native = signed_lots * (close - entry) * instrument[2]; if(DPCCBondMode(mode)) native = signed_lots * instrument[2] * ((close - entry) * instrument[22] * instrument[44] + interest - entry_interest); else if(DPCCTickMode(mode)) { const float difference = signed_lots * (close - entry); native = difference * (difference >= 0.0f ? instrument[20] : instrument[21]) / instrument[1]; if(instrument[43] == 1.0f) return(native); } return(native * (native >= 0.0f ? fx_gain : fx_loss)); } //+------------------------------------------------------------------+ //| Initial/maintenance margin, including FORTS discount and Last | //+------------------------------------------------------------------+ // Nearest money unit only when FP32 factor/input uncertainty cannot cross its // half-unit boundary. Refusal prevents silent cent changes, not a feasible // result. No legacy calculation calls this function. float DPCCMarginRound(float value, uint digits) { if(!isfinite(value) || value < 0.0f || digits > 8U) return(NAN); if(value == 0.0f) return(0.0f); float scale = 1.0f; for(uint i = 0; i < digits; i++) scale *= 10.0f; const float units = value * scale; const float uncertainty = fmax(fabs(units) * (32.0f * FLT_EPSILON), 32.0f * FLT_EPSILON); if(!isfinite(units) || uncertainty >= 0.5f) return(NAN); if(fabs((units - floor(units)) - 0.5f) <= uncertainty) return((floor(units) + 1.0f) / scale); //--- Conservatively round ambiguous half-unit margins upward. return(floor(units + 0.5f) / scale); } float DPCCMargin(float signed_lots, float average, float market, float native_last, int initial, float fx_margin, float fx_gain, __global const float *instrument, uint currency_digits) { const float volume = fabs(signed_lots); const int mode = (int)instrument[11]; if(volume == 0.0f || mode == DPCC_CALC_COLLATERAL) return(0.0f); const int is_buy = signed_lots > 0.0f; float rate = initial ? instrument[is_buy ? 17 : 18] : instrument[is_buy ? 41 : 42]; if(instrument[30] == 4.0f && !initial && rate == 0.0f) rate = instrument[is_buy ? 17 : 18]; if(instrument[30] == 3.0f) { if((mode != DPCC_CALC_FOREX && mode != DPCC_CALC_FOREX_NOLEV) || instrument[31] != 0.0f || instrument[32] != 0.0f || instrument[15] != 0.0f || instrument[16] != 0.0f || instrument[46] != 0.0f || instrument[47] != 0.0f || fx_margin != 1.0f || !isfinite(market) || market <= 0.0f || !isnormal(instrument[2]) || instrument[2] <= 0.0f || !isnormal(market) || !isnormal(volume) || !isfinite(rate) || rate < 0.0f || (mode == DPCC_CALC_FOREX && (!isfinite(instrument[19]) || instrument[19] <= 0.0f))) return(NAN); const float effective_rate = !initial && rate == 0.0f ? instrument[is_buy ? 17 : 18] : rate; if(!isfinite(effective_rate) || effective_rate < 0.0f || (effective_rate > 0.0f && !isnormal(effective_rate))) return(NAN); if(effective_rate == 0.0f) return(0.0f); //--- Relative FP32 rounding bounds require every nonzero intermediate normal. //--- Reject subnormals and underflow-to-zero before a later factor can recover. const float contracted = volume * instrument[2]; if(!isnormal(contracted) || contracted <= 0.0f) return(NAN); const float leveraged = contracted / (mode == DPCC_CALC_FOREX ? instrument[19] : 1.0f); if(!isnormal(leveraged) || leveraged <= 0.0f) return(NAN); const float rated = leveraged * effective_rate; if(!isnormal(rated) || rated <= 0.0f) return(NAN); const float margin = rated * market; if(!isnormal(margin) || margin <= 0.0f) return(NAN); return(DPCCMarginRound(margin, currency_digits)); } float margin = 0.0f; int already_account = 0; if((!initial && instrument[47] == 1.0f) || (initial && instrument[46] == 1.0f)) margin = volume * instrument[initial ? 15 : 16] * rate; else if(!initial && instrument[46] == 1.0f) margin = volume * instrument[15] * rate; else switch(mode) { case DPCC_CALC_FOREX: margin = volume * instrument[2] / instrument[19] * rate; break; case DPCC_CALC_FOREX_NOLEV: margin = volume * instrument[2] * rate; break; case DPCC_CALC_CFD: margin = volume * instrument[2] * market * rate; break; case DPCC_CALC_CFDLEVERAGE: margin = volume * instrument[2] * market / instrument[19] * rate; break; case DPCC_CALC_CFDINDEX: margin = volume * instrument[2] * market * instrument[20] / instrument[1] * rate; already_account = instrument[43] == 1.0f; break; case DPCC_CALC_FUTURES: case DPCC_CALC_EXCH_FUT: case DPCC_CALC_FORTS: margin = volume * (initial || instrument[16] == 0.0f ? instrument[15] : instrument[16]) * rate; break; case DPCC_CALC_STOCKS: case DPCC_CALC_MOEX_STOCKS: margin = volume * instrument[2] * native_last * rate; break; case DPCC_CALC_BONDS: case DPCC_CALC_MOEX_BONDS: margin = volume * instrument[2] * instrument[22] * average * instrument[44]; break; default: return(NAN); } if(instrument[30] == 4.0f && ((initial && instrument[46] == 1.0f) || (!initial && (instrument[46] == 1.0f || instrument[47] == 1.0f))) && (mode == DPCC_CALC_FOREX || mode == DPCC_CALC_CFDLEVERAGE)) margin /= instrument[19]; if(!already_account) margin *= fx_margin; if(mode == DPCC_CALC_FORTS) { const float favorable = is_buy ? instrument[40] - average : average - instrument[40]; if(favorable > 0.0f) { float discount = volume * favorable * instrument[20] / instrument[1]; if(instrument[43] == 0.0f) discount *= fx_gain; margin -= discount; } } return(instrument[30] == 4.0f ? DPCCMarginRound(margin, currency_digits) : margin); } //+------------------------------------------------------------------+ //| Select the frozen conversion side for the signed exposure | //+------------------------------------------------------------------+ float DPCCMarginFX(float signed_lots, __global const float *execution, __global const float *instrument) { return(instrument[30] == 4.0f && signed_lots < 0.0f ? execution[23] : execution[15]); } //+------------------------------------------------------------------+ //| Mark account while retaining the verified initial equity anchor | //+------------------------------------------------------------------+ void DPCCMark(__private DPCCAccountState *state, float bid, float ask, float native, float interest, float fx_gain, float fx_loss, float fx_margin, __global const float *instrument) { const float q = state->account[2]; const float close = q >= 0.0f ? bid : ask; const float profit = DPCCProfit(q, state->account[3], close, state->entry_interest, interest, fx_gain, fx_loss, instrument); const float collateral = (int)instrument[11] == DPCC_CALC_COLLATERAL ? q * instrument[2] * native * instrument[24] * fx_gain : 0.0f; state->account[15] = bid; state->account[16] = ask; state->native_mark = native; state->accrued_interest = interest; state->account[1] = state->equity_anchor + state->account[0] + profit - state->account[6] + collateral; const float margin_market = (instrument[30] == 3.0f || instrument[30] == 4.0f) ? (q >= 0.0f ? ask : bid) : close; state->account[10] = DPCCMargin(q, state->account[3], margin_market, native, 0, fx_margin, fx_gain, instrument, state->currency_digits); state->account[12] = state->account[1] - state->account[10] - state->account[11]; if(state->account[20] == 0.0f) { state->account[8] = state->account[1]; state->account[20] = 1.0f; } state->account[8] = fmax(state->account[8], state->account[1]); state->maximum_drawdown = fmax(state->maximum_drawdown, state->account[8] - state->account[1]); for(int i = 0; i < 24; i++) if(!isfinite(state->account[i])) state->numeric_status = DPCC_STATUS_NUMERIC; if(!isfinite(native) || !isfinite(interest) || state->account[10] < 0.0f || !isfinite(state->entry_interest) || !isfinite(state->equity_anchor) || !isfinite(state->maximum_drawdown)) state->numeric_status = DPCC_STATUS_NUMERIC; } //+------------------------------------------------------------------+ //| Per-constraint maxima prevent event count from changing risk | //+------------------------------------------------------------------+ void DPCCRiskInit(__private DPCCRiskState *risk) { for(int i = 0; i < DPCC_RISK_COUNT; i++) risk->maximum[i] = 0.0f; risk->first_h = -1; risk->constraint = -1; risk->phase = -1; risk->source_action = -1; } void DPCCRiskAdd(__private DPCCRiskState *risk, int constraint, float amount, float scale, int h, int phase, int source_action) { if(amount <= 0.0f) return; const float value = amount / scale; risk->maximum[constraint] = fmax(risk->maximum[constraint], value); if(risk->first_h < 0 || h < risk->first_h || (h == risk->first_h && phase < risk->phase)) { risk->first_h = h; risk->constraint = constraint; risk->phase = phase; risk->source_action = source_action; } } float DPCCRiskMaximum(__private const DPCCRiskState *risk) { float result = 0.0f; for(int i = 0; i < DPCC_RISK_COUNT; i++) result = fmax(result, risk->maximum[i]); return(result); } void DPCCRiskMerge(__private DPCCRiskState *destination, __private const DPCCRiskState *source) { for(int i = 0; i < DPCC_RISK_COUNT; i++) destination->maximum[i] = fmax(destination->maximum[i], source->maximum[i]); if(source->first_h >= 0 && (destination->first_h < 0 || source->first_h < destination->first_h || (source->first_h == destination->first_h && source->phase < destination->phase))) { destination->first_h = source->first_h; destination->constraint = source->constraint; destination->phase = source->phase; destination->source_action = source->source_action; } } void DPCCObserve(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *limits, float volume_scale, int h, int phase) { for(int i = 0; i < 24; i++) if(!isfinite(state->account[i])) state->numeric_status = DPCC_STATUS_NUMERIC; if(!isfinite(state->entry_interest) || !isfinite(state->equity_anchor) || !isfinite(state->native_mark) || !isfinite(state->accrued_interest) || !isfinite(state->maximum_drawdown)) state->numeric_status = DPCC_STATUS_NUMERIC; const uint mask = (uint)limits[20]; const float cash = limits[21]; const float equity = state->account[1]; const float total_margin = state->account[10] + state->account[11]; const int source = state->source_action; if(mask & 1U) DPCCRiskAdd(risk, 0, fabs(state->account[2]) - limits[0], volume_scale, h, phase, source); if(mask & 2U) DPCCRiskAdd(risk, 1, limits[1] + limits[11] - state->account[12], cash, h, phase, source); if(mask & 4U) { const float fraction = equity > 0.0f ? total_margin / equity : (total_margin > 0.0f ? 1.0f : 0.0f); DPCCRiskAdd(risk, 2, fraction - limits[2], 1.0f, h, phase, source); if(!isfinite(fraction)) state->numeric_status = DPCC_STATUS_NUMERIC; } if(mask & 8U) DPCCRiskAdd(risk, 3, limits[3] - equity, cash, h, phase, source); if(mask & 16U) DPCCRiskAdd(risk, 4, state->account[9] - equity - (limits[4] - limits[12]), cash, h, phase, source); if(mask & 32U) DPCCRiskAdd(risk, 5, state->account[8] - equity - limits[5], cash, h, phase, source); if(mask & 64U) DPCCRiskAdd(risk, 6, state->account[7] - equity - limits[6], cash, h, phase, source); if(equity <= 0.0f) DPCCRiskAdd(risk, 14, fmax(1.0f, -equity / cash), 1.0f, h, phase, source); for(int i = 0; i < DPCC_RISK_COUNT; i++) if(!isfinite(risk->maximum[i])) state->numeric_status = DPCC_STATUS_NUMERIC; } //+------------------------------------------------------------------+ //| Real close and opening legs; no fee for unchanged exposure | //+------------------------------------------------------------------+ int DPCCTradePermitted(float old_q, float target, int trade_mode) { if(old_q == target) return(1); if(trade_mode == 0) return(0); const int same_side = (old_q > 0.0f && target > 0.0f) || (old_q < 0.0f && target < 0.0f); const int reduction = target == 0.0f || (same_side && fabs(target) <= fabs(old_q)); if(trade_mode == 3) return(reduction); if(trade_mode == 1 && target < 0.0f) return(reduction); if(trade_mode == 2 && target > 0.0f) return(reduction); return(1); } void DPCCClose(__private DPCCAccountState *state, float volume, __global const float *instrument, __global const float *execution, __global const float *limits) { const float q = state->account[2]; if(volume == 0.0f || q == 0.0f) return; const int buy = q > 0.0f; const float signed_volume = buy ? volume : -volume; const float extra = limits[13] * instrument[0]; const float fill = buy ? state->account[15] - execution[6] - extra : state->account[16] + execution[7] + extra; const float gross = DPCCProfit(signed_volume, state->account[3], fill, state->entry_interest, state->accrued_interest, execution[13], execution[14], instrument); const float cost = volume * execution[buy ? 10 : 11] + execution[12]; const float old_cost = state->account[6] * volume / fabs(q); state->account[0] += gross - cost - old_cost; state->account[13] += gross; state->account[14] += cost + old_cost; state->account[6] -= old_cost; state->account[2] = buy ? q - volume : q + volume; if(state->account[2] == 0.0f) { state->account[3] = 0.0f; state->account[4] = 0.0f; state->account[5] = 0.0f; state->account[17] = 0.0f; state->account[18] = 0.0f; state->entry_interest = 0.0f; } if(!isfinite(fill) || !isfinite(gross) || !isfinite(cost) || !isfinite(old_cost)) state->numeric_status = DPCC_STATUS_NUMERIC; DPCCMark(state, state->account[15], state->account[16], state->native_mark, state->accrued_interest, execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); } void DPCCOpen(__private DPCCAccountState *state, float target, float volume, int h, __global const float *instrument, __global const float *execution, __global const float *limits) { if(volume == 0.0f) return; const int buy = target > 0.0f; const float extra = limits[13] * instrument[0]; const float fill = buy ? state->account[16] + execution[4] + extra : state->account[15] - execution[5] - extra; const float old_volume = fabs(state->account[2]); const float total_volume = old_volume + volume; state->account[3] = (state->account[3] * old_volume + fill * volume) / total_volume; state->entry_interest = (state->entry_interest * old_volume + state->accrued_interest * volume) / total_volume; const float cost = volume * execution[buy ? 8 : 9] + execution[12]; state->account[0] -= cost; state->account[14] += cost; state->account[2] = target; state->source_action = h; if(!isfinite(fill) || !isfinite(cost) || !isfinite(total_volume)) state->numeric_status = DPCC_STATUS_NUMERIC; DPCCMark(state, state->account[15], state->account[16], state->native_mark, state->accrued_interest, execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); } //+------------------------------------------------------------------+ //| Protective exit, including unavailable/disabled real operation | //+------------------------------------------------------------------+ int DPCCProtectionKind(__private const DPCCAccountState *state) { const float q = state->account[2]; if(q == 0.0f) return(0); const float price = q > 0.0f ? state->account[15] : state->account[16]; if(state->account[17] == 1.0f && (q > 0.0f ? price <= state->account[4] : price >= state->account[4])) return(1); if(state->account[18] == 1.0f && (q > 0.0f ? price >= state->account[5] : price <= state->account[5])) return(2); return(0); } void DPCCProtect(__private DPCCAccountState *state, __private DPCCRiskState *risk, int h, int phase, __global const float *instrument, __global const float *execution, __global const float *limits) { const int kind = DPCCProtectionKind(state); if(kind == 0) return; if(!DPCCTradePermitted(state->account[2], 0.0f, (int)instrument[29]) || (((uint)instrument[34] & 1U) == 0U) || (int)instrument[11] == DPCC_CALC_COLLATERAL) { DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, phase, state->source_action); return; } state->exit_kind = kind; DPCCClose(state, fabs(state->account[2]), instrument, execution, limits); } void DPCCTerminate(__private DPCCAccountState *state, __private DPCCRiskState *risk, int h, int phase, __global const float *instrument, __global const float *execution, __global const float *limits, float volume_scale) { if(state->account[1] > 0.0f || state->account[19] == 1.0f) return; if(state->account[2] != 0.0f) { if(DPCCTradePermitted(state->account[2], 0.0f, (int)instrument[29]) && ((uint)instrument[34] & 1U) != 0U && (int)instrument[11] != DPCC_CALC_COLLATERAL) DPCCClose(state, fabs(state->account[2]), instrument, execution, limits); else DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, phase, state->source_action); } DPCCObserve(state, risk, limits, volume_scale, h, phase); state->account[19] = 1.0f; } //+------------------------------------------------------------------+ //| Calendar on the current mark; DAY_RESET follows outgoing risk | //+------------------------------------------------------------------+ void DPCCCalendar(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *event, __global const float *instrument, __global const float *execution, __global const float *limits, float volume_scale, int h, int phase) { if(state->account[19] == 1.0f) return; DPCCObserve(state, risk, limits, volume_scale, h, phase); if(event[1] == 1.0f) { state->account[7] = state->account[1]; state->account[22] = 1.0f; } else { const float flow = fabs(state->account[2]) * event[state->account[2] >= 0.0f ? 2 : 3] * event[4]; state->account[0] += flow; state->account[13] += flow; if(flow < 0.0f) state->account[14] -= flow; if(!isfinite(flow)) state->numeric_status = DPCC_STATUS_NUMERIC; DPCCMark(state, state->account[15], state->account[16], state->native_mark, state->accrued_interest, execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); } DPCCObserve(state, risk, limits, volume_scale, h, phase); DPCCTerminate(state, risk, h, phase, instrument, execution, limits, volume_scale); } //+------------------------------------------------------------------+ //| One request volume on the broker minimum/step grid | //+------------------------------------------------------------------+ float DPCCDealVolume(float requested, float minimum, float maximum, float step) { const float limited = fmin(requested, maximum); const float scale = fmax(fmax(fabs(limited), minimum), step); const float tolerance = fmin(16.0f * FLT_EPSILON * scale, step * 0.0001f); if(requested < minimum && minimum - requested > tolerance) return(0.0f); const float normalized = fmax(limited, minimum); const float raw_steps = (normalized - minimum) / step; const float nearest = round(raw_steps); const float steps = fabs(raw_steps - nearest) <= tolerance / step ? nearest : floor(raw_steps); const float result = minimum + steps * step; if(steps < 0.0f || result - maximum > tolerance) return(0.0f); return(fmin(result, maximum)); } //+------------------------------------------------------------------+ //| Nearest direct-lot position target within the sigmoid range | //+------------------------------------------------------------------+ float DPCCPositionTarget(float desired, float minimum, float step) { const float magnitude = fabs(desired); if(magnitude < minimum) return(0.0f); const float rounded = minimum + round((magnitude - minimum) / step) * step; const float target = DPCCDealVolume(rounded, minimum, 1.0f, step); return(desired > 0.0f ? target : -target); } //+------------------------------------------------------------------+ //| Decode direct lots and close broker-sized legs to the target | //+------------------------------------------------------------------+ int DPCCAction(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *action, __global const float *instrument, __global const float *execution, __global const float *limits, float volume_scale, float price_scale, int h) { for(int i = 0; i < 6; i++) if(!isfinite(action[i])) { DPCCRiskAdd(risk, 15, 1.0f, 1.0f, h, 1, state->source_action); return(0); } const float desired = action[0] - action[3]; const float target = DPCCPositionTarget(desired, instrument[3], instrument[5]); if(!isfinite(desired) || !isfinite(target)) { state->numeric_status = DPCC_STATUS_NUMERIC; return(0); } const float old_q = state->account[2]; const float old_volume = fabs(old_q); const float new_volume = fabs(target); const int buy = target > 0.0f; const int reversal = (old_q > 0.0f && target < 0.0f) || (old_q < 0.0f && target > 0.0f); const float close_requested = reversal ? old_volume : fmax(0.0f, old_volume - new_volume); const float retained_volume = reversal ? 0.0f : fmin(old_volume, new_volume); const float close_tolerance = fmin(16.0f * FLT_EPSILON * fmax(old_volume, instrument[3]), instrument[5] * 0.0001f); const float open_requested = reversal ? new_volume : fmax(0.0f, new_volume - old_volume); int valid = 1; if(action[0] < 0.0f || action[0] > DPCC_ACTION_VOLUME_MAX || action[3] < 0.0f || action[3] > DPCC_ACTION_VOLUME_MAX || (target != 0.0f && (action[buy ? 1 : 4] < 0.0f || action[buy ? 2 : 5] < 0.0f))) { DPCCRiskAdd(risk, 15, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } if(!DPCCTradePermitted(old_q, target, (int)instrument[29]) || ((open_requested > 0.0f || close_requested > 0.0f) && (((uint)instrument[34] & 1U) == 0U)) || ((int)instrument[11] == DPCC_CALC_COLLATERAL && target != old_q)) { DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } float sl = 0.0f; float tp = 0.0f; float sl_points = 0.0f; float tp_points = 0.0f; int sl_active = 0; int tp_active = 0; if(target != 0.0f) { sl_points = action[buy ? 2 : 5] * instrument[7]; tp_points = action[buy ? 1 : 4] * instrument[6]; sl_active = sl_points > 0.0f; tp_active = tp_points > 0.0f; const float reference = state->account[buy ? 16 : 15]; sl = reference + (buy ? -sl_points : sl_points) * instrument[0]; tp = reference + (buy ? tp_points : -tp_points) * instrument[0]; if(sl_active) sl = round(sl / instrument[1]) * instrument[1]; if(tp_active) tp = round(tp / instrument[1]) * instrument[1]; if(!sl_active) sl = 0.0f; if(!tp_active) tp = 0.0f; const float trigger = state->account[buy ? 15 : 16]; const float minimum = instrument[8] * instrument[0]; if(!isfinite(sl_points) || !isfinite(tp_points) || !isfinite(sl) || !isfinite(tp)) { state->numeric_status = DPCC_STATUS_NUMERIC; return(0); } if(sl_active) sl_points = fabs(reference - sl) / instrument[0]; if(tp_active) tp_points = fabs(tp - reference) / instrument[0]; if((sl_active && (!DPCCOnGrid(sl, instrument[1], limits[15]) || !DPCCPriceValid(sl, instrument) || (buy ? trigger - sl : sl - trigger) < minimum)) || (tp_active && (!DPCCOnGrid(tp, instrument[1], limits[15]) || !DPCCPriceValid(tp, instrument) || (buy ? tp - trigger : trigger - tp) < minimum))) { DPCCRiskAdd(risk, 12, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } const uint mask = (uint)limits[20]; if(mask & 128U) { const float amount = fmax(limits[7] - sl_points, sl_points - limits[8]) * instrument[0]; DPCCRiskAdd(risk, 7, amount, price_scale, h, 1, state->source_action); } if(mask & 256U) { const float amount = fmax(limits[9] - tp_points, tp_points - limits[10]) * instrument[0]; DPCCRiskAdd(risk, 8, amount, price_scale, h, 1, state->source_action); } } if((int)instrument[11] == DPCC_CALC_COLLATERAL && (sl_active || tp_active)) { DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } const int change_sl = old_q != target || state->account[4] != sl || state->account[17] != (sl_active ? 1.0f : 0.0f); const int change_tp = old_q != target || state->account[5] != tp || state->account[18] != (tp_active ? 1.0f : 0.0f); if((change_sl || change_tp) && (int)instrument[29] == 0) { DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } if((change_sl && (state->account[17] == 1.0f || sl_active) && (((uint)instrument[34] & 16U) == 0U)) || (change_tp && (state->account[18] == 1.0f || tp_active) && (((uint)instrument[34] & 32U) == 0U))) { DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } const float old_trigger = state->account[old_q > 0.0f ? 15 : 16]; const float freeze = instrument[9] * instrument[0]; if(old_q == target && old_q != 0.0f && freeze > 0.0f && ((change_sl && state->account[17] == 1.0f && fabs(old_trigger - state->account[4]) <= freeze) || (change_tp && state->account[18] == 1.0f && fabs(old_trigger - state->account[5]) <= freeze))) { DPCCRiskAdd(risk, 12, 1.0f, 1.0f, h, 1, state->source_action); valid = 0; } if(!valid) { if(state->numeric_status != DPCC_STATUS_NUMERIC) state->numeric_status = DPCC_STATUS_INVALID; return(0); } //--- Every completed close leg precedes the opposite opening, within this H1 step. while(retained_volume == 0.0f ? state->account[2] != 0.0f : fabs(state->account[2]) > retained_volume + close_tolerance) { const float before = fabs(state->account[2]); const float remaining = before - retained_volume; float close_volume = DPCCDealVolume(remaining, instrument[3], instrument[4], instrument[5]); //--- Use the exact legal last quantity to avoid an untradeable FP32 residue. if(close_volume > 0.0f && remaining >= instrument[3] - close_tolerance && remaining <= instrument[4] + close_tolerance && fabs(close_volume - remaining) <= close_tolerance) close_volume = remaining; if(close_volume <= 0.0f) { if(retained_volume == 0.0f) { DPCCRiskAdd(risk, 9, remaining, volume_scale, h, 1, state->source_action); state->numeric_status = DPCC_STATUS_INVALID; return(0); } break; } DPCCClose(state, close_volume, instrument, execution, limits); DPCCObserve(state, risk, limits, volume_scale, h, 1); if(state->numeric_status == DPCC_STATUS_NUMERIC) return(0); if(fabs(state->account[2]) >= before) { state->numeric_status = DPCC_STATUS_NUMERIC; return(0); } } DPCCObserve(state, risk, limits, volume_scale, h, 1); DPCCTerminate(state, risk, h, 1, instrument, execution, limits, volume_scale); if(state->account[19] == 1.0f) return(0); //--- An opposite opening requires the old exposure to be exactly flat. if(target == 0.0f || (state->account[2] != 0.0f && ((state->account[2] > 0.0f) != buy))) return(1); //--- Existing same-side protection changes precede the attempted opening leg. if(state->account[2] != 0.0f) { state->account[4] = sl; state->account[5] = tp; state->account[17] = sl_active ? 1.0f : 0.0f; state->account[18] = tp_active ? 1.0f : 0.0f; } const float open_volume = DPCCDealVolume(fmax(0.0f, new_volume - fabs(state->account[2])), instrument[3], instrument[4], instrument[5]); const float filled_volume = fabs(state->account[2]) + open_volume; const float filled_target = buy ? filled_volume : -filled_volume; if(open_volume > 0.0f) { const float market = buy ? state->account[16] : state->account[15]; const float old_initial = DPCCMargin(state->account[2], state->account[3], market, state->native_mark, 1, DPCCMarginFX(state->account[2], execution, instrument), execution[13], instrument, state->currency_digits); const float entry = market + (buy ? execution[4] + limits[13] * instrument[0] : -execution[5] - limits[13] * instrument[0]); const float average = (state->account[3] * fabs(state->account[2]) + entry * open_volume) / filled_volume; const float target_initial = DPCCMargin(filled_target, average, market, state->native_mark, 1, DPCCMarginFX(filled_target, execution, instrument), execution[13], instrument, state->currency_digits); const float extra_margin = fmax(target_initial - old_initial, 0.0f); const float fees = open_volume * execution[buy ? 8 : 9] + execution[12]; if(!isfinite(old_initial) || old_initial < 0.0f || !isfinite(target_initial) || target_initial < 0.0f || !isfinite(fees)) { state->numeric_status = DPCC_STATUS_NUMERIC; return(0); } if(instrument[38] > 0.0f && filled_volume > instrument[38]) { DPCCRiskAdd(risk, 9, filled_volume - instrument[38], volume_scale, h, 1, h); return(1); } if(extra_margin + fees > state->account[12]) { DPCCRiskAdd(risk, 13, extra_margin + fees - state->account[12], limits[21], h, 1, h); return(1); } DPCCOpen(state, filled_target, open_volume, h, instrument, execution, limits); } if(state->account[2] != 0.0f) { state->account[4] = sl; state->account[5] = tp; state->account[17] = sl_active ? 1.0f : 0.0f; state->account[18] = tp_active ? 1.0f : 0.0f; } DPCCObserve(state, risk, limits, volume_scale, h, 1); DPCCTerminate(state, risk, h, 1, instrument, execution, limits, volume_scale); return(1); } //+------------------------------------------------------------------+ //| Exact protective intersection with a linear Bid/Ask segment | //+------------------------------------------------------------------+ float DPCCCrossing(__private const DPCCAccountState *state, float end_bid, float end_ask, __private int *kind) { *kind = 0; const float q = state->account[2]; if(q == 0.0f) return(2.0f); const float start = state->account[q > 0.0f ? 15 : 16]; const float end = q > 0.0f ? end_bid : end_ask; float first = 2.0f; for(int protection = 0; protection < 2; protection++) { if(state->account[17 + protection] == 0.0f) continue; const float level = state->account[4 + protection]; const int reached_start = protection == 0 ? (q > 0.0f ? start <= level : start >= level) : (q > 0.0f ? start >= level : start <= level); const int reached_end = protection == 0 ? (q > 0.0f ? end <= level : end >= level) : (q > 0.0f ? end >= level : end <= level); if(!reached_start && !reached_end) continue; const float fraction = reached_start ? 0.0f : (level - start) / (end - start); if(fraction >= 0.0f && fraction <= 1.0f && fraction < first) { first = fraction; *kind = protection + 1; } } return(first); } //+------------------------------------------------------------------+ //| Advance to event/node, inserting protection before calendar ties | //+------------------------------------------------------------------+ void DPCCLinearAdvance(__private DPCCAccountState *state, __private DPCCRiskState *risk, float end_bid, float end_spread, float end_native, float end_interest, float seconds, __global const float *instrument, __global const float *execution, __global const float *limits, float volume_scale, int h, int phase) { if(state->account[19] == 1.0f) return; const float end_ask = end_bid + end_spread; const float start_bid = state->account[15]; const float start_ask = state->account[16]; const float start_native = state->native_mark; const float start_interest = state->accrued_interest; int kind = 0; const float fraction = DPCCCrossing(state, end_bid, end_ask, &kind); if(kind != 0) { DPCCMark(state, start_bid + fraction * (end_bid - start_bid), start_ask + fraction * (end_ask - start_ask), start_native + fraction * (end_native - start_native), start_interest + fraction * (end_interest - start_interest), execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); state->account[21] += seconds * fraction; DPCCObserve(state, risk, limits, volume_scale, h, phase); if(DPCCTradePermitted(state->account[2], 0.0f, (int)instrument[29]) && (((uint)instrument[34] & 1U) != 0U) && (int)instrument[11] != DPCC_CALC_COLLATERAL) { state->exit_kind = kind; DPCCClose(state, fabs(state->account[2]), instrument, execution, limits); } else DPCCRiskAdd(risk, 10, 1.0f, 1.0f, h, phase, state->source_action); DPCCObserve(state, risk, limits, volume_scale, h, phase); DPCCTerminate(state, risk, h, phase, instrument, execution, limits, volume_scale); if(state->account[19] == 1.0f) return; state->account[21] += seconds * (1.0f - fraction); } else state->account[21] += seconds; DPCCMark(state, end_bid, end_ask, end_native, end_interest, execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); DPCCObserve(state, risk, limits, volume_scale, h, phase); DPCCTerminate(state, risk, h, phase, instrument, execution, limits, volume_scale); } //+------------------------------------------------------------------+ //| One complete pure macro transition, shared by every caller | //+------------------------------------------------------------------+ uint DPCCAccountPrepareOpen(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *ohlc, __global const float *instrument, __global const float *execution, __global const float *limits, __global const float *calendar, uint event_count, float volume_scale, uint h, __private float *branch_diagnostic, int defer_zero_events) { for(int d = 0; d < 4; d++) branch_diagnostic[d] = 0.0f; branch_diagnostic[0] = -1.0f; if(state->account[19] == 1.0f) return(0); state->exit_kind = 0; state->account[22] = 0.0f; uint cursor = 0; float previous_gap_offset = -execution[19]; while(cursor < event_count && calendar[cursor * 6] < 0.0f) { const float offset = calendar[cursor * 6]; state->account[21] += offset - previous_gap_offset; previous_gap_offset = offset; DPCCCalendar(state, risk, calendar + cursor * 6, instrument, execution, limits, volume_scale, h, 0); cursor++; } if(state->account[19] == 1.0f) return(cursor); state->account[21] -= previous_gap_offset; const float delta = execution[20]; DPCCMark(state, ohlc[0], ohlc[0] + execution[0], ohlc[0] - delta, execution[16], execution[13], execution[14], DPCCMarginFX(state->account[2], execution, instrument), instrument); DPCCObserve(state, risk, limits, volume_scale, h, 0); DPCCProtect(state, risk, h, 0, instrument, execution, limits); DPCCObserve(state, risk, limits, volume_scale, h, 0); DPCCTerminate(state, risk, h, 0, instrument, execution, limits, volume_scale); while(defer_zero_events == 0 && cursor < event_count && calendar[cursor * 6] == 0.0f) { DPCCCalendar(state, risk, calendar + cursor * 6, instrument, execution, limits, volume_scale, h, 0); cursor++; } if(state->account[19] == 1.0f) return(cursor); return(cursor); } //+------------------------------------------------------------------+ //| Commits one action after the causal open preparation | //+------------------------------------------------------------------+ void DPCCAccountCommitPrepared(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *action, __global const float *ohlc, __global const float *instrument, __global const float *execution, __global const float *limits, __global const float *calendar, uint event_count, uint cursor, float volume_scale, float price_scale, uint h, uint horizon, __private float *branch_diagnostic, uint apply_action) { const float delta = execution[20]; if(state->account[19] == 1.0f) return; if(apply_action != 0U) DPCCAction(state, risk, action, instrument, execution, limits, volume_scale, price_scale, h); if(state->numeric_status == DPCC_STATUS_NUMERIC || state->account[19] == 1.0f) return; DPCCAccountState branches[2]; DPCCRiskState branch_risks[2]; for(int branch = 0; branch < 2; branch++) { branches[branch] = *state; branches[branch].maximum_drawdown = 0.0f; branch_risks[branch] = *risk; __private DPCCAccountState *current = &branches[branch]; __private DPCCRiskState *current_risk = &branch_risks[branch]; uint event_index = cursor; float elapsed = 0.0f; float nodes[4]; float spreads[4]; nodes[0] = ohlc[0]; nodes[1] = ohlc[branch == 0 ? 1 : 2]; nodes[2] = ohlc[branch == 0 ? 2 : 1]; nodes[3] = ohlc[3]; spreads[0] = execution[0]; spreads[1] = execution[branch == 0 ? 2 : 3]; spreads[2] = execution[branch == 0 ? 3 : 2]; spreads[3] = execution[1]; for(int segment = 0; segment < 3 && current->account[19] == 0.0f; segment++) { const float start_time = execution[18] * (float)segment / 3.0f; const float end_time = execution[18] * (float)(segment + 1) / 3.0f; while(event_index < event_count && calendar[event_index * 6] <= end_time && current->account[19] == 0.0f) { const float event_time = calendar[event_index * 6]; const float fraction = (event_time - start_time) / (end_time - start_time); const float bid = nodes[segment] + fraction * (nodes[segment + 1] - nodes[segment]); const float spread = spreads[segment] + fraction * (spreads[segment + 1] - spreads[segment]); const float interest = execution[16] + event_time / execution[18] * (execution[17] - execution[16]); DPCCLinearAdvance(current, current_risk, bid, spread, bid - delta, interest, event_time - elapsed, instrument, execution, limits, volume_scale, h, event_time == execution[18] ? 3 : 2); elapsed = event_time; DPCCCalendar(current, current_risk, calendar + event_index * 6, instrument, execution, limits, volume_scale, h, event_time == execution[18] ? 3 : 2); event_index++; } const float interest = execution[16] + end_time / execution[18] * (execution[17] - execution[16]); DPCCLinearAdvance(current, current_risk, nodes[segment + 1], spreads[segment + 1], nodes[segment + 1] - delta, interest, end_time - elapsed, instrument, execution, limits, volume_scale, h, segment == 2 ? 3 : 2); elapsed = end_time; } if(h + 1 == horizon && current->account[19] == 0.0f && current->account[2] != 0.0f && (int)instrument[11] != DPCC_CALC_COLLATERAL) { if(DPCCTradePermitted(current->account[2], 0.0f, (int)instrument[29]) && (((uint)instrument[34] & 1U) != 0U)) DPCCClose(current, fabs(current->account[2]), instrument, execution, limits); else DPCCRiskAdd(current_risk, 10, 1.0f, 1.0f, h, 3, current->source_action); DPCCObserve(current, current_risk, limits, volume_scale, h, 3); DPCCTerminate(current, current_risk, h, 3, instrument, execution, limits, volume_scale); } } const float vmax0 = DPCCRiskMaximum(&branch_risks[0]); const float vmax1 = DPCCRiskMaximum(&branch_risks[1]); int chosen = 0; if(branches[1].account[1] < branches[0].account[1]) chosen = 1; if(branches[0].exit_kind == 1 && branches[1].exit_kind == 2) chosen = 0; else if(branches[1].exit_kind == 1 && branches[0].exit_kind == 2) chosen = 1; else if(branches[1].account[1] == branches[0].account[1] && (vmax1 > vmax0 || (vmax1 == vmax0 && branches[1].maximum_drawdown > branches[0].maximum_drawdown))) chosen = 1; if(branches[0].numeric_status == DPCC_STATUS_NUMERIC || branches[1].numeric_status == DPCC_STATUS_NUMERIC) branches[chosen].numeric_status = DPCC_STATUS_NUMERIC; else if(branches[0].numeric_status == DPCC_STATUS_INVALID || branches[1].numeric_status == DPCC_STATUS_INVALID) branches[chosen].numeric_status = DPCC_STATUS_INVALID; *state = branches[chosen]; DPCCRiskMerge(risk, &branch_risks[0]); DPCCRiskMerge(risk, &branch_risks[1]); branch_diagnostic[0] = (float)chosen; branch_diagnostic[1] = vmax0; branch_diagnostic[2] = vmax1; branch_diagnostic[3] = state->maximum_drawdown; } //+------------------------------------------------------------------+ //| Full legacy transition preserves prepare then commit call order | //+------------------------------------------------------------------+ void DPCCAccountStep(__private DPCCAccountState *state, __private DPCCRiskState *risk, __global const float *action, __global const float *ohlc, __global const float *instrument, __global const float *execution, __global const float *limits, __global const float *calendar, uint event_count, float volume_scale, float price_scale, uint h, uint horizon, __private float *branch_diagnostic) { const uint cursor = DPCCAccountPrepareOpen(state, risk, ohlc, instrument, execution, limits, calendar, event_count, volume_scale, h, branch_diagnostic, 0); DPCCAccountCommitPrepared(state, risk, action, ohlc, instrument, execution, limits, calendar, event_count, cursor, volume_scale, price_scale, h, horizon, branch_diagnostic, 1U); } //+------------------------------------------------------------------+ //| Context validation is never bypassed by an inactive plan slot | //+------------------------------------------------------------------+ int DPCCValidateRolloutContext(__global const float *context, __global const float *account_override, __global const float *account_profile, uint k, uint K, uint H, uint E, uint M, uint episode_start_step, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_count, uint off_calendar, uint off_active, uint total, uint expected_basis, uint expected_adapter, float gamma, float volume_scale, float price_scale, float max_quote_age, float max_last_age, float money_scale, uint expected_calculation_contract, uint currency_digits, int require_volume_grid, int observed_open, uint native_continuation, __private uint *failure_line) { if(K == 0 || K > 64 || H == 0 || E == 0 || M == 0 || expected_basis > 1 || expected_adapter > 1 || expected_basis != expected_adapter || native_continuation > 1U || (observed_open != 0 && observed_open != 1) || !isfinite(gamma) || gamma <= 0.0f || gamma > 1.0f || !isfinite(volume_scale) || volume_scale <= 0.0f || !isfinite(price_scale) || price_scale <= 0.0f || !isfinite(max_quote_age) || max_quote_age < 0.0f || !isfinite(max_last_age) || max_last_age < 0.0f || !isfinite(money_scale) || money_scale <= 0.0f || (expected_calculation_contract != 2U && expected_calculation_contract != 3U && expected_calculation_contract != 4U) || (expected_calculation_contract == 2U ? currency_digits != 0U : currency_digits > 8U)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} const ulong expected_account = (ulong)K * H * 4; const ulong expected_instrument = expected_account + 24; const ulong expected_limits = expected_instrument + 48; const ulong expected_execution = expected_limits + 24; const ulong expected_count = expected_execution + (ulong)K * H * 24; const ulong expected_calendar = expected_count + H; const ulong expected_active = expected_calendar + (ulong)K * H * E * 6; if(expected_account != off_account || expected_instrument != off_instrument || expected_limits != off_limits || expected_execution != off_execution || expected_count != off_count || expected_calendar != off_calendar || expected_active != off_active || expected_active + M != total) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} __global const float *account = account_override; __global const float *profile_account = account_profile; __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; for(int i = 0; i < 24; i++) if(!isfinite(account[i]) || !isfinite(limits[i])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(int i = 0; i < 48; i++) if(!isfinite(instrument[i])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(uint m = 0; m < M; m++) if(!DPCCBoolean(context[off_active + m])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(!DPCCInteger(limits[20], 0.0f, 511.0f) || limits[21] != money_scale || limits[22] != 2.0f || limits[23] != 0.0f || instrument[30] != (float)expected_calculation_contract || instrument[39] != 0.0f || account[23] != 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(expected_calculation_contract == 3U && (((int)instrument[11] != DPCC_CALC_FOREX && (int)instrument[11] != DPCC_CALC_FOREX_NOLEV) || expected_basis != 0U || expected_adapter != 0U || instrument[15] != 0.0f || instrument[16] != 0.0f || instrument[46] != 0.0f || instrument[47] != 0.0f || instrument[12] != 1.0f || instrument[13] != 1.0f || instrument[14] != profile_account[16])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(int i = 11; i < 19; i++) if(limits[i] < 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(limits[19] < 0.0f || limits[14] >= instrument[5] * 0.5f || limits[15] >= instrument[1] * 0.5f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} const uint mask = (uint)limits[20]; if(((mask & 1U) && limits[0] < 0.0f) || ((mask & 4U) && limits[2] < 0.0f) || ((mask & 16U) && limits[12] > limits[4]) || ((mask & 32U) && limits[5] < 0.0f) || ((mask & 64U) && limits[6] < 0.0f) || ((mask & 128U) && (limits[7] < 0.0f || limits[8] < limits[7])) || ((mask & 256U) && (limits[9] < 0.0f || limits[10] < limits[9]))) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(instrument[0] <= 0.0f || instrument[1] <= 0.0f || instrument[2] <= 0.0f || instrument[3] <= 0.0f || instrument[4] < instrument[3] || instrument[5] <= 0.0f || instrument[6] < 0.0f || instrument[7] < 0.0f || instrument[8] < 0.0f || instrument[9] < 0.0f || !DPCCInteger(instrument[10], 0.0f, 16.0f) || !DPCCInteger(instrument[11], 0.0f, 64.0f) || !DPCCSupportedMode((int)instrument[11]) || instrument[12] <= 0.0f || instrument[13] <= 0.0f || instrument[14] <= 0.0f || instrument[15] < 0.0f || instrument[16] < 0.0f || instrument[17] < 0.0f || instrument[18] < 0.0f || instrument[41] < 0.0f || instrument[42] < 0.0f || !DPCCBoolean(instrument[27]) || !DPCCBoolean(instrument[28]) || !DPCCInteger(instrument[29], 0.0f, 4.0f) || instrument[31] != (float)expected_basis || instrument[32] != (float)expected_adapter || !DPCCInteger(instrument[34], 0.0f, 127.0f) || !DPCCInteger(instrument[35], 0.0f, 2.0f) || instrument[36] < 0.0f || instrument[36] > max_quote_age || instrument[37] < 0.0f || instrument[38] < 0.0f || !DPCCBoolean(instrument[43]) || !DPCCBoolean(instrument[46]) || !DPCCBoolean(instrument[47]) || instrument[45] < 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(instrument[27] == 1.0f && instrument[28] == 1.0f && instrument[25] > instrument[26]) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} const int mode = (int)instrument[11]; if((mode == DPCC_CALC_FOREX || mode == DPCC_CALC_CFDLEVERAGE) && instrument[19] <= 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if((DPCCTickMode(mode) || mode == DPCC_CALC_CFDINDEX) && (instrument[20] <= 0.0f || instrument[21] <= 0.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(DPCCTickMode(mode) && instrument[15] <= 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if((mode == DPCC_CALC_STOCKS || mode == DPCC_CALC_MOEX_STOCKS) && expected_basis != 1) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(expected_basis == 1 && (instrument[37] > max_last_age || !DPCCPriceValid(instrument[33], instrument))) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(DPCCBondMode(mode)) { if(instrument[22] <= 0.0f || instrument[44] <= 0.0f || (native_continuation == 0U && account[2] != 0.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} // Account24 has no entry interest; only an explicit State33 carries it. if(native_continuation != 0U) { for(int i = 24; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) if(!isfinite(account[i])) return(DPCC_STATUS_NUMERIC); if(!DPCCInteger(account[29], 0.0f, 8.0f) || (uint)account[29] != currency_digits || !DPCCInteger(account[30], -1.0f, (float)episode_start_step) || !DPCCInteger(account[31], 0.0f, 3.0f) || account[32] != 0.0f || account[28] < 0.0f || !DPCCPriceValid(account[26], instrument) || (expected_basis == 0U && account[26] != account[15]) || (account[2] == 0.0f ? (account[3] != 0.0f || account[25] != 0.0f) : (!DPCCPriceValid(account[3], instrument) || account[30] < 0.0f))) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} } } if(mode == DPCC_CALC_COLLATERAL && (instrument[24] < 0.0f || instrument[24] > 1.0f || account[2] < 0.0f || account[17] != 0.0f || account[18] != 0.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if((instrument[46] == 1.0f && instrument[15] <= 0.0f) || (instrument[47] == 1.0f && instrument[16] <= 0.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(!DPCCBoolean(account[17]) || !DPCCBoolean(account[18]) || !DPCCBoolean(account[19]) || !DPCCBoolean(account[20]) || !DPCCBoolean(account[22]) || account[6] < 0.0f || account[10] < 0.0f || account[11] < 0.0f || account[14] < 0.0f || account[21] < 0.0f || account[16] < account[15] || !DPCCPriceValid(account[15], instrument) || !DPCCPriceValid(account[16], instrument) || (require_volume_grid && !DPCCOnGrid(account[2], instrument[5], limits[14])) || (account[2] == 0.0f && (account[17] != 0.0f || account[18] != 0.0f)) || (account[19] == 1.0f && account[2] != 0.0f) || (account[20] == 1.0f && account[8] < account[1])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} const float free_margin = account[1] - account[10] - account[11]; const float money_tolerance = limits[16] + limits[17] * fmax(fabs(free_margin), fabs(account[12])); if(!isfinite(free_margin) || !isfinite(money_tolerance)) return(DPCC_STATUS_NUMERIC); if(fabs(account[12] - free_margin) > money_tolerance) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if((account[17] == 1.0f && !DPCCPriceValid(account[4], instrument)) || (account[18] == 1.0f && !DPCCPriceValid(account[5], instrument))) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} float frozen_delta = 0.0f; for(uint h = 0; h < H; h++) { __global const float *ohlc = context + ((size_t)k * H + h) * 4; __global const float *execution = context + off_execution + ((size_t)k * H + h) * 24; __global const float *reference_execution = context + off_execution + (size_t)h * 24; const float count_float = context[off_count + h]; if(!DPCCInteger(count_float, 0.0f, (float)E)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} const uint count = (uint)count_float; for(int i = 0; i < 4; i++) if(!DPCCPriceValid(ohlc[i], instrument)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if((observed_open != 0 && (ohlc[1] != ohlc[0] || ohlc[2] != ohlc[0] || ohlc[3] != ohlc[0])) || (observed_open == 0 && (ohlc[1] < ohlc[0] || ohlc[1] < ohlc[3] || ohlc[2] > ohlc[0] || ohlc[2] > ohlc[3] || ohlc[2] > ohlc[1]))) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(int i = 0; i < 24; i++) if(!isfinite(execution[i])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(int i = 0; i < 13; i++) if(execution[i] < 0.0f) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(expected_calculation_contract == 3U && (execution[13] != 1.0f || execution[14] != 1.0f || execution[15] != 1.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(expected_calculation_contract == 4U && (execution[14] < execution[13] || execution[15] < execution[23] || execution[13] != reference_execution[13] || execution[14] != reference_execution[14] || execution[15] != reference_execution[15] || execution[23] != reference_execution[23])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(execution[13] <= 0.0f || execution[14] <= 0.0f || execution[15] <= 0.0f || execution[18] <= 0.0f || execution[19] < 0.0f || (episode_start_step == 0 && h == 0 && execution[19] != 0.0f) || execution[21] != count_float || !DPCCInteger(execution[22], 0.0f, 16777216.0f) || (expected_calculation_contract == 4U ? execution[23] <= 0.0f : execution[23] != 0.0f) || execution[18] != reference_execution[18] || execution[19] != reference_execution[19] || execution[22] != reference_execution[22]) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(observed_open != 0 && (execution[1] != execution[0] || execution[2] != execution[0] || execution[3] != execution[0] || execution[17] != execution[16])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(h == 0) frozen_delta = execution[20]; if(execution[20] != frozen_delta || execution[20] != reference_execution[20] || (expected_adapter == 0 && execution[20] != 0.0f) || (expected_adapter == 1 && fabs(profile_account[15] - instrument[33] - execution[20]) > limits[15])) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} for(int vertex = 0; vertex < 4; vertex++) { const float spread = execution[vertex == 0 ? 0 : (vertex == 3 ? 1 : vertex + 1)]; if(!DPCCPriceValid(ohlc[vertex] + spread, instrument) || !DPCCPriceValid(ohlc[vertex] - execution[20], instrument)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} } __global const float *calendar = context + off_calendar + ((size_t)k * H + h) * E * 6; __global const float *reference_calendar = context + off_calendar + (size_t)h * E * 6; float previous_offset = -execution[19]; float previous_type = 0.0f; for(uint event = 0; event < E; event++) { __global const float *item = calendar + event * 6; for(int i = 0; i < 6; i++) if(!isfinite(item[i]) || (event >= count && item[i] != 0.0f)) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} if(event >= count) continue; if(item[0] > execution[18] || item[0] < 0.0f - execution[19] || ((h > 0 || episode_start_step > 0) && item[0] == -execution[19]) || (h == 0 && episode_start_step == 0 && item[0] < 0.0f) || (observed_open != 0 && item[0] >= 0.0f) || !DPCCInteger(item[1], 1.0f, 3.0f) || item[4] < 0.0f || item[5] != 0.0f || (item[1] == 1.0f && (item[2] != 0.0f || item[3] != 0.0f || item[4] != 0.0f)) || item[0] < previous_offset || (item[0] == previous_offset && item[1] < previous_type) || item[0] != reference_calendar[event * 6] || item[1] != reference_calendar[event * 6 + 1]) {if(failure_line) *failure_line=(uint)__LINE__; return(DPCC_STATUS_INVALID);} previous_offset = item[0]; previous_type = item[1]; } } return(0); } //+------------------------------------------------------------------+ //| Independent slot/scenario work-item; outputs are caller scratch | //+------------------------------------------------------------------+ __kernel void DPCCRollout(__global const float *plans, __global const float *context, __global const float *slot_active, __global float *returns, __global float *states, __global float *violations, __global float *system_status, __global float *branch_diagnostics, uint plan_offset, uint slot_capacity, uint K, uint H, uint E, uint context_M, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_calendar_count, uint off_calendar, uint off_R_active, uint R_total, float gamma, float volume_scale, float price_scale, float max_quote_age_seconds, float max_last_age_seconds, uint slot_active_offset, uint expected_price_basis, uint expected_quote_adapter, float money_scale, uint expected_calculation_contract, uint currency_digits, __global const float *continuations, uint continued, uint episode_start, uint episode_horizon, float training_floor, __global float *step_rewards, __global float *executed_mask) { const uint k = (uint)get_global_id(0); const uint slot = (uint)get_global_id(1); if(k >= K || slot >= slot_capacity) return; const size_t pair = (size_t)slot * K + k; const size_t state_offset = pair * ((size_t)H + 1) * 24; const size_t violation_offset = pair * 8; const size_t diagnostic_offset = pair * H * 4; returns[pair] = 0.0f; system_status[pair] = 0.0f; for(size_t i = 0; i < ((size_t)H + 1) * 24; i++) states[state_offset + i] = 0.0f; for(uint h = 0; h < H; h++) { if(continued) { step_rewards[pair * H + h] = 0.0f; executed_mask[pair * H + h] = 0.0f; } branch_diagnostics[diagnostic_offset + h * 4] = -1.0f; for(int i = 1; i < 4; i++) branch_diagnostics[diagnostic_offset + h * 4 + i] = 0.0f; } for(int i = 0; i < 8; i++) violations[violation_offset + i] = 0.0f; violations[violation_offset] = -1.0f; violations[violation_offset + 1] = -1.0f; violations[violation_offset + 6] = -1.0f; violations[violation_offset + 7] = -1.0f; if(continued > 1U || (continued && (episode_horizon == 0U || episode_horizon > DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX || episode_start >= episode_horizon || H > episode_horizon - episode_start || !isfinite(training_floor) || training_floor < 0.0f))) { system_status[pair] = (float)DPCC_STATUS_INVALID; violations[violation_offset + 5] = (float)DPCC_STATUS_INVALID; return; } __global const float *account = continued ? continuations + pair * DPCC_NATIVE_CONTINUATION_FLOATS : context + off_account; uint context_failure_line=0; const int checked = DPCCValidateRolloutContext(context, account, context + off_account, k, K, H, E, context_M, continued ? episode_start : 0, off_account, off_instrument, off_limits, off_execution, off_calendar_count, off_calendar, off_R_active, R_total, expected_price_basis, expected_quote_adapter, gamma, volume_scale, price_scale, max_quote_age_seconds, max_last_age_seconds, money_scale, expected_calculation_contract, currency_digits, continued ? 0 : 1, 0, continued, &context_failure_line); if(checked != 0) { system_status[pair] = (float)checked; violations[violation_offset + 5] = (float)checked; violations[violation_offset + 6] = (float)context_failure_line; return; } const float active = slot_active[(size_t)slot_active_offset + slot]; if(!DPCCBoolean(active)) { system_status[pair] = (float)DPCC_STATUS_INVALID; violations[violation_offset + 5] = (float)DPCC_STATUS_INVALID; return; } __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; DPCCAccountState state; for(int i = 0; i < 24; i++) { state.account[i] = account[i]; states[state_offset + i] = account[i]; } state.entry_interest = 0.0f; state.native_mark = expected_price_basis == 0 ? account[15] : instrument[33]; state.accrued_interest = instrument[23]; state.maximum_drawdown = 0.0f; state.currency_digits = currency_digits; state.source_action = -1; state.exit_kind = 0; state.numeric_status = 0; const float initial_pnl = continued ? 0.0f : DPCCProfit(account[2], account[3], account[account[2] >= 0.0f ? 15 : 16], 0.0f, instrument[23], instrument[12], instrument[13], instrument); const float initial_collateral = !continued && (int)instrument[11] == DPCC_CALC_COLLATERAL ? account[2] * instrument[2] * state.native_mark * instrument[24] * instrument[12] : 0.0f; state.equity_anchor = account[1] - account[0] - initial_pnl + account[6] - initial_collateral; if(!isfinite(initial_pnl) || !isfinite(initial_collateral) || !isfinite(state.equity_anchor)) { system_status[pair] = (float)DPCC_STATUS_NUMERIC; violations[violation_offset + 5] = (float)DPCC_STATUS_NUMERIC; return; } if(continued) { for(int i = 24; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) if(!isfinite(account[i])) { system_status[pair] = (float)DPCC_STATUS_NUMERIC; violations[violation_offset + 5] = (float)DPCC_STATUS_NUMERIC; return; } if(!DPCCInteger(account[29],0.0f,8.0f) || (uint)account[29] != currency_digits || !DPCCInteger(account[30],-1.0f,(float)episode_start) || !DPCCInteger(account[31],0.0f,3.0f) || !DPCCInteger(account[32],0.0f,0.0f) || account[28] < 0.0f) { system_status[pair] = (float)DPCC_STATUS_INVALID; violations[violation_offset + 5] = (float)DPCC_STATUS_INVALID; return; } state.equity_anchor = account[24]; state.entry_interest = account[25]; state.native_mark = account[26]; state.accrued_interest = account[27]; state.maximum_drawdown = account[28]; state.currency_digits = (uint)account[29]; state.source_action = (int)account[30]; state.exit_kind = (int)account[31]; state.numeric_status = (int)account[32]; } if(active == 0.0f) { for(uint h = 1; h <= H; h++) for(int i = 0; i < 24; i++) states[state_offset + (size_t)h * 24 + i] = account[i]; return; } DPCCRiskState risk; DPCCRiskInit(&risk); if(state.account[20] == 0.0f) { state.account[8] = state.account[1]; state.account[20] = 1.0f; } for(uint h = 0; h < (continued ? 0U : H); h++) { __global const float *action = plans + (size_t)plan_offset + ((size_t)slot * H + h) * 6; int valid_action = 1; for(int i = 0; i < 6; i++) if(!isfinite(action[i])) valid_action = 0; if(valid_action) { const float target = action[0] - action[3]; if(!isfinite(target)) { system_status[pair] = (float)DPCC_STATUS_NUMERIC; state.numeric_status = DPCC_STATUS_NUMERIC; return; } if(action[0] < 0.0f || action[3] < 0.0f || (target > 0.0f && (action[1] < 0.0f || action[2] < 0.0f)) || (target < 0.0f && (action[4] < 0.0f || action[5] < 0.0f))) valid_action = 0; } if(!valid_action) { DPCCRiskAdd(&risk, 15, 1.0f, 1.0f, h, 1, -1); state.numeric_status = DPCC_STATUS_INVALID; } } if(!continued) DPCCObserve(&state, &risk, limits, volume_scale, 0, 0); float result = 0.0f; float discount = 1.0f; for(uint h = 0; h < H; h++) { const float equity_before = state.account[1]; __global const float *action = plans + (size_t)plan_offset + ((size_t)slot * H + h) * 6; __global const float *ohlc = context + ((size_t)k * H + h) * 4; __global const float *execution = context + off_execution + ((size_t)k * H + h) * 24; __global const float *calendar = context + off_calendar + ((size_t)k * H + h) * E * 6; float branch[4]; if(continued) { int valid_action = 1; for(int i = 0; i < 6; i++) if(!isfinite(action[i])) valid_action = 0; if(valid_action) { const float target = action[0] - action[3]; if(!isfinite(target)) { state.numeric_status = DPCC_STATUS_NUMERIC; system_status[pair] = (float)DPCC_STATUS_NUMERIC; break; } if(action[0] < 0.0f || action[3] < 0.0f || (target > 0.0f && (action[1] < 0.0f || action[2] < 0.0f)) || (target < 0.0f && (action[4] < 0.0f || action[5] < 0.0f))) valid_action = 0; } if(!valid_action) state.numeric_status = DPCC_STATUS_INVALID; } DPCCRiskState step_risk; DPCCRiskInit(&step_risk); DPCCAccountStep(&state, &step_risk, action, ohlc, instrument, execution, limits, calendar, (uint)context[off_calendar_count + h], volume_scale, price_scale, continued ? episode_start + h : h, continued ? episode_horizon : H, branch); if(continued) { for(int i = 0; i < 24; i++) if(!isfinite(state.account[i])) state.numeric_status = DPCC_STATUS_NUMERIC; if(!isfinite(state.equity_anchor) || !isfinite(state.entry_interest) || !isfinite(state.native_mark) || !isfinite(state.accrued_interest) || !isfinite(state.maximum_drawdown)) state.numeric_status = DPCC_STATUS_NUMERIC; } DPCCRiskMerge(&risk, &step_risk); result += discount * (state.account[1] - equity_before); discount *= gamma; for(int i = 0; i < 24; i++) states[state_offset + ((size_t)h + 1) * 24 + i] = state.account[i]; for(int i = 0; i < 4; i++) branch_diagnostics[diagnostic_offset + (size_t)h * 4 + i] = branch[i]; if(state.numeric_status == DPCC_STATUS_NUMERIC || !isfinite(result) || !isfinite(discount)) { system_status[pair] = (float)DPCC_STATUS_NUMERIC; state.numeric_status = DPCC_STATUS_NUMERIC; result = 0.0f; break; } if(continued && state.numeric_status != 0) { system_status[pair] = (float)state.numeric_status; result = 0.0f; break; } if(continued) { step_rewards[pair * H + h] = state.account[1] - equity_before; executed_mask[pair * H + h] = 1.0f; } if(continued && training_floor > 0.0f && (state.account[0] < training_floor || state.account[1] < training_floor)) { for(uint remaining = h + 2; remaining <= H; remaining++) for(int i = 0; i < 24; i++) states[state_offset + (size_t)remaining * 24 + i] = state.account[i]; break; } } float maximum = 0.0f; float sum = 0.0f; for(int i = 0; i < DPCC_RISK_COUNT; i++) { maximum = fmax(maximum, risk.maximum[i]); sum += risk.maximum[i]; } if(!isfinite(maximum) || !isfinite(sum)) { system_status[pair] = (float)DPCC_STATUS_NUMERIC; state.numeric_status = DPCC_STATUS_NUMERIC; } returns[pair] = result; violations[violation_offset] = (float)risk.first_h; violations[violation_offset + 1] = (float)risk.constraint; violations[violation_offset + 2] = maximum; violations[violation_offset + 3] = sum; violations[violation_offset + 4] = state.account[19]; violations[violation_offset + 5] = (float)state.numeric_status; violations[violation_offset + 6] = (float)risk.phase; violations[violation_offset + 7] = (float)risk.source_action; } //+------------------------------------------------------------------+ //| Transient risk continuation between open preparation and action | //+------------------------------------------------------------------+ void DPCCStoreRiskContinuation(__private const DPCCRiskState *risk, __global float *destination, size_t offset) { for(int i = 0; i < DPCC_RISK_COUNT; i++) destination[offset + i] = risk->maximum[i]; destination[offset + 16] = (float)risk->first_h; destination[offset + 17] = (float)risk->constraint; destination[offset + 18] = (float)risk->phase; destination[offset + 19] = (float)risk->source_action; } int DPCCLoadRiskContinuation(__private DPCCRiskState *risk, __global const float *source, size_t offset) { for(int i = 0; i < DPCC_RISK_COUNT; i++) { if(!isfinite(source[offset + i]) || source[offset + i] < 0.0f) return(DPCC_STATUS_NUMERIC); risk->maximum[i] = source[offset + i]; } if(!DPCCInteger(source[offset + 16], -1.0f, (float)DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX) || !DPCCInteger(source[offset + 17], -1.0f, (float)DPCC_RISK_COUNT) || !DPCCInteger(source[offset + 18], -1.0f, 3.0f) || !DPCCInteger(source[offset + 19], -1.0f, (float)DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX)) return(DPCC_STATUS_INVALID); risk->first_h = (int)source[offset + 16]; risk->constraint = (int)source[offset + 17]; risk->phase = (int)source[offset + 18]; risk->source_action = (int)source[offset + 19]; return(0); } //+------------------------------------------------------------------+ //| One-bar continuation through the same native account transition | //+------------------------------------------------------------------+ __kernel void DPCCRolloutStep(__global const float *plans, __global const float *context, __global const float *state_in, __global float *state_out, __global float *rewards, __global float *status, __global float *branch_diagnostic, uint plan_offset, uint K, uint E, uint context_M, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_count, uint off_calendar, uint off_active, uint total, uint episode_step, uint episode_horizon, float gamma, float volume_scale, float price_scale, float max_quote_age, float max_last_age, float money_scale, uint expected_basis, uint expected_adapter, uint expected_calculation_contract, uint currency_digits, uint initialize) { const uint k = (uint)get_global_id(0); if(k >= K) return; const size_t state_offset = (size_t)k * DPCC_NATIVE_CONTINUATION_FLOATS; rewards[k] = 0.0f; status[k] = 0.0f; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = 0.0f; branch_diagnostic[(size_t)k * 4] = -1.0f; for(int i = 0; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) state_out[state_offset + i] = 0.0f; if(episode_horizon == 0 || episode_horizon > DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX || episode_step >= episode_horizon || initialize > 1U || (initialize == 1U && episode_step != 0U) || (initialize == 0U && episode_step == 0U)) { status[k] = (float)DPCC_STATUS_INVALID; return; } __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; __global const float *account_context = context + off_account; __global const float *account_override = initialize == 1U ? account_context : state_in + state_offset; uint context_failure_line=0; const int checked = DPCCValidateRolloutContext(context, account_override, account_context, k, K, 1, E, context_M, episode_step, off_account, off_instrument, off_limits, off_execution, off_count, off_calendar, off_active, total, expected_basis, expected_adapter, gamma, volume_scale, price_scale, max_quote_age, max_last_age, money_scale, expected_calculation_contract, currency_digits, initialize == 1U, 0, initialize == 0U, &context_failure_line); if(checked != 0) { status[k] = (float)checked; return; } if(context[off_active] != 1.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } DPCCAccountState state; for(int i = 0; i < 24; i++) state.account[i] = account_override[i]; if(initialize == 1U) { state.entry_interest = 0.0f; state.native_mark = expected_basis == 0U ? state.account[15] : instrument[33]; state.accrued_interest = instrument[23]; state.maximum_drawdown = 0.0f; state.currency_digits = currency_digits; state.source_action = -1; state.exit_kind = 0; state.numeric_status = 0; const float initial_pnl = DPCCProfit(state.account[2], state.account[3], state.account[state.account[2] >= 0.0f ? 15 : 16], 0.0f, instrument[23], instrument[12], instrument[13], instrument); const float initial_collateral = (int)instrument[11] == DPCC_CALC_COLLATERAL ? state.account[2] * instrument[2] * state.native_mark * instrument[24] * instrument[12] : 0.0f; state.equity_anchor = state.account[1] - state.account[0] - initial_pnl + state.account[6] - initial_collateral; if(!isfinite(initial_pnl) || !isfinite(initial_collateral) || !isfinite(state.equity_anchor)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } } else { for(int i = 24; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) if(!isfinite(state_in[state_offset + i])) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } state.equity_anchor = state_in[state_offset + 24]; state.entry_interest = state_in[state_offset + 25]; state.native_mark = state_in[state_offset + 26]; state.accrued_interest = state_in[state_offset + 27]; state.maximum_drawdown = state_in[state_offset + 28]; if(!DPCCInteger(state_in[state_offset + 29], 0.0f, 8.0f) || !DPCCInteger(state_in[state_offset + 30], -1.0f, (float)episode_step) || !DPCCInteger(state_in[state_offset + 31], 0.0f, 3.0f) || !DPCCInteger(state_in[state_offset + 32], 0.0f, 0.0f) || state.maximum_drawdown < 0.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } state.currency_digits = (uint)state_in[state_offset + 29]; state.source_action = (int)state_in[state_offset + 30]; state.exit_kind = (int)state_in[state_offset + 31]; state.numeric_status = (int)state_in[state_offset + 32]; if(state.currency_digits != currency_digits) { status[k] = (float)DPCC_STATUS_INVALID; return; } } if(state.account[20] == 0.0f) { state.account[8] = state.account[1]; state.account[20] = 1.0f; } __global const float *action = plans + (size_t)plan_offset; int valid_action = 1; for(int i = 0; i < 6; i++) if(!isfinite(action[i])) valid_action = 0; if(valid_action) { const float target = action[0] - action[3]; if(!isfinite(target)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } if(action[0] < 0.0f || action[3] < 0.0f || (target > 0.0f && (action[1] < 0.0f || action[2] < 0.0f)) || (target < 0.0f && (action[4] < 0.0f || action[5] < 0.0f))) valid_action = 0; } if(!valid_action) state.numeric_status = DPCC_STATUS_INVALID; const float equity_before = state.account[1]; __global const float *ohlc = context + (size_t)k * 4; __global const float *execution = context + off_execution + (size_t)k * 24; __global const float *calendar = context + off_calendar + (size_t)k * E * 6; const uint event_count = (uint)context[off_count]; float branch[4]; DPCCRiskState step_risk; DPCCRiskInit(&step_risk); DPCCAccountStep(&state, &step_risk, action, ohlc, instrument, execution, limits, calendar, event_count, volume_scale, price_scale, episode_step, episode_horizon, branch); for(int i = 0; i < 24; i++) if(!isfinite(state.account[i])) state.numeric_status = DPCC_STATUS_NUMERIC; if(state.numeric_status != 0) { status[k] = (float)state.numeric_status; return; } const float reward = state.account[1] - equity_before; if(!isfinite(reward) || !isfinite(state.equity_anchor) || !isfinite(state.entry_interest) || !isfinite(state.native_mark) || !isfinite(state.accrued_interest) || !isfinite(state.maximum_drawdown)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } rewards[k] = reward; for(int i = 0; i < 24; i++) state_out[state_offset + i] = state.account[i]; state_out[state_offset + 24] = state.equity_anchor; state_out[state_offset + 25] = state.entry_interest; state_out[state_offset + 26] = state.native_mark; state_out[state_offset + 27] = state.accrued_interest; state_out[state_offset + 28] = state.maximum_drawdown; state_out[state_offset + 29] = (float)state.currency_digits; state_out[state_offset + 30] = (float)state.source_action; state_out[state_offset + 31] = (float)state.exit_kind; state_out[state_offset + 32] = (float)state.numeric_status; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = branch[i]; } //+------------------------------------------------------------------+ //| Prepares exactly the causal open prefix and saves transient risk | //+------------------------------------------------------------------+ __kernel void DPCCRolloutPrepareOpen(__global const float *context, __global const float *state_in, __global float *state_out, __global float *risk_out, __global float *rewards, __global float *status, __global float *branch_diagnostic, uint K, uint E, uint context_M, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_count, uint off_calendar, uint off_active, uint total, uint episode_step, uint episode_horizon, float gamma, float volume_scale, float price_scale, float max_quote_age, float max_last_age, float money_scale, uint expected_basis, uint expected_adapter, uint expected_calculation_contract, uint currency_digits, uint initialize, uint observed_open) { const uint k = (uint)get_global_id(0); if(k >= K) return; const size_t state_offset = (size_t)k * DPCC_NATIVE_CONTINUATION_FLOATS; const size_t risk_offset = (size_t)k * DPCC_NATIVE_PENDING_PHASE_FLOATS; rewards[k] = 0.0f; status[k] = 0.0f; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = 0.0f; branch_diagnostic[(size_t)k * 4] = -1.0f; for(int i = 0; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) state_out[state_offset + i] = 0.0f; for(int i = 0; i < DPCC_NATIVE_PENDING_PHASE_FLOATS; i++) risk_out[risk_offset + i] = 0.0f; if(episode_horizon == 0 || episode_horizon > DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX || episode_step >= episode_horizon || initialize > 1U || observed_open > 1U || (initialize == 1U && episode_step != 0U) || (initialize == 0U && episode_step == 0U)) { status[k] = (float)DPCC_STATUS_INVALID; return; } __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; __global const float *account_context = context + off_account; __global const float *account_override = initialize == 1U ? account_context : state_in + state_offset; uint context_failure_line=0; const int checked = DPCCValidateRolloutContext(context, account_override, account_context, k, K, 1, E, context_M, episode_step, off_account, off_instrument, off_limits, off_execution, off_count, off_calendar, off_active, total, expected_basis, expected_adapter, gamma, volume_scale, price_scale, max_quote_age, max_last_age, money_scale, expected_calculation_contract, currency_digits, initialize == 1U, (int)observed_open, initialize == 0U, &context_failure_line); if(checked != 0) { status[k] = (float)checked; return; } if(context[off_active] != 1.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } DPCCAccountState state; for(int i = 0; i < 24; i++) state.account[i] = account_override[i]; if(initialize == 1U) { state.entry_interest = 0.0f; state.native_mark = expected_basis == 0U ? state.account[15] : instrument[33]; state.accrued_interest = instrument[23]; state.maximum_drawdown = 0.0f; state.currency_digits = currency_digits; state.source_action = -1; state.exit_kind = 0; state.numeric_status = 0; const float initial_pnl = DPCCProfit(state.account[2], state.account[3], state.account[state.account[2] >= 0.0f ? 15 : 16], 0.0f, instrument[23], instrument[12], instrument[13], instrument); const float initial_collateral = (int)instrument[11] == DPCC_CALC_COLLATERAL ? state.account[2] * instrument[2] * state.native_mark * instrument[24] * instrument[12] : 0.0f; state.equity_anchor = state.account[1] - state.account[0] - initial_pnl + state.account[6] - initial_collateral; if(!isfinite(initial_pnl) || !isfinite(initial_collateral) || !isfinite(state.equity_anchor)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } } else { for(int i = 24; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) if(!isfinite(state_in[state_offset + i])) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } state.equity_anchor = state_in[state_offset + 24]; state.entry_interest = state_in[state_offset + 25]; state.native_mark = state_in[state_offset + 26]; state.accrued_interest = state_in[state_offset + 27]; state.maximum_drawdown = state_in[state_offset + 28]; if(!DPCCInteger(state_in[state_offset + 29], 0.0f, 8.0f) || !DPCCInteger(state_in[state_offset + 30], -1.0f, (float)episode_step) || !DPCCInteger(state_in[state_offset + 31], 0.0f, 3.0f) || !DPCCInteger(state_in[state_offset + 32], 0.0f, 0.0f) || state.maximum_drawdown < 0.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } state.currency_digits = (uint)state_in[state_offset + 29]; state.source_action = (int)state_in[state_offset + 30]; state.exit_kind = (int)state_in[state_offset + 31]; state.numeric_status = (int)state_in[state_offset + 32]; if(state.currency_digits != currency_digits) { status[k] = (float)DPCC_STATUS_INVALID; return; } } if(state.account[20] == 0.0f) { state.account[8] = state.account[1]; state.account[20] = 1.0f; } const float equity_before = state.account[1]; DPCCRiskState risk; DPCCRiskInit(&risk); __global const float *ohlc = context + (size_t)k * 4; __global const float *execution = context + off_execution + (size_t)k * 24; __global const float *calendar = context + off_calendar + (size_t)k * E * 6; const uint event_count = (uint)context[off_count]; float branch[4]; DPCCAccountPrepareOpen(&state, &risk, ohlc, instrument, execution, limits, calendar, event_count, volume_scale, episode_step, branch, (int)observed_open); for(int i = 0; i < 24; i++) if(!isfinite(state.account[i])) state.numeric_status = DPCC_STATUS_NUMERIC; if(state.numeric_status != 0) { status[k] = (float)state.numeric_status; return; } const float reward = state.account[1] - equity_before; if(!isfinite(reward) || !isfinite(state.equity_anchor) || !isfinite(state.entry_interest) || !isfinite(state.native_mark) || !isfinite(state.accrued_interest) || !isfinite(state.maximum_drawdown)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } rewards[k] = reward; for(int i = 0; i < 24; i++) state_out[state_offset + i] = state.account[i]; state_out[state_offset + 24] = state.equity_anchor; state_out[state_offset + 25] = state.entry_interest; state_out[state_offset + 26] = state.native_mark; state_out[state_offset + 27] = state.accrued_interest; state_out[state_offset + 28] = state.maximum_drawdown; state_out[state_offset + 29] = (float)state.currency_digits; state_out[state_offset + 30] = (float)state.source_action; state_out[state_offset + 31] = (float)state.exit_kind; state_out[state_offset + 32] = (float)state.numeric_status; DPCCStoreRiskContinuation(&risk, risk_out, risk_offset); risk_out[risk_offset + 20] = state.account[19] == 1.0f ? 0.0f : 1.0f; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = branch[i]; } //+------------------------------------------------------------------+ //| Validates factual data against the causally pinned observed open | //+------------------------------------------------------------------+ int DPCCValidateObservedCommitContext(__global const float *open_context, __global const float *factual_context, __global const float *account_override, uint k, uint K, uint E, uint context_M, uint episode_step, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_count, uint off_calendar, uint off_active, uint total, uint expected_basis, uint expected_adapter, float gamma, float volume_scale, float price_scale, float max_quote_age, float max_last_age, float money_scale, uint expected_calculation_contract, uint currency_digits) { uint context_failure_line=0; const int factual_checked = DPCCValidateRolloutContext(factual_context, account_override, factual_context + off_account, k, K, 1, E, context_M, episode_step, off_account, off_instrument, off_limits, off_execution, off_count, off_calendar, off_active, total, expected_basis, expected_adapter, gamma, volume_scale, price_scale, max_quote_age, max_last_age, money_scale, expected_calculation_contract, currency_digits, 0, 0, 1U, &context_failure_line); if(factual_checked != 0) return(factual_checked); for(uint i = 0; i < 24; i++) if(as_uint(open_context[off_account + i]) != as_uint(factual_context[off_account + i]) || as_uint(open_context[off_limits + i]) != as_uint(factual_context[off_limits + i])) return(DPCC_STATUS_INVALID); for(uint i = 0; i < 48; i++) if(as_uint(open_context[off_instrument + i]) != as_uint(factual_context[off_instrument + i])) return(DPCC_STATUS_INVALID); for(uint m = 0; m < context_M; m++) if(as_uint(open_context[off_active + m]) != as_uint(factual_context[off_active + m])) return(DPCC_STATUS_INVALID); __global const float *open_ohlc = open_context + (size_t)k * 4; __global const float *factual_ohlc = factual_context + (size_t)k * 4; if(as_uint(open_ohlc[0]) != as_uint(factual_ohlc[0])) return(DPCC_STATUS_INVALID); __global const float *open_execution = open_context + off_execution + (size_t)k * 24; __global const float *factual_execution = factual_context + off_execution + (size_t)k * 24; for(uint i = 0; i < 24; i++) if(i != 1U && i != 2U && i != 3U && i != 17U && i != 21U && as_uint(open_execution[i]) != as_uint(factual_execution[i])) return(DPCC_STATUS_INVALID); const float open_count_float = open_context[off_count]; const float factual_count_float = factual_context[off_count]; if(!DPCCInteger(open_count_float, 0.0f, (float)E) || !DPCCInteger(factual_count_float, 0.0f, (float)E)) return(DPCC_STATUS_INVALID); const uint open_count = (uint)open_count_float; const uint factual_count = (uint)factual_count_float; if(open_count > factual_count) return(DPCC_STATUS_INVALID); __global const float *open_calendar = open_context + off_calendar + (size_t)k * E * 6; __global const float *factual_calendar = factual_context + off_calendar + (size_t)k * E * 6; for(uint event = 0; event < open_count; event++) { if(open_calendar[event * 6] >= 0.0f) return(DPCC_STATUS_INVALID); for(uint i = 0; i < 6; i++) if(as_uint(open_calendar[event * 6 + i]) != as_uint(factual_calendar[event * 6 + i])) return(DPCC_STATUS_INVALID); } for(uint event = open_count; event < factual_count; event++) if(factual_calendar[event * 6] < 0.0f) return(DPCC_STATUS_INVALID); return(0); } //+------------------------------------------------------------------+ //| Consumes the prepared phase and applies exactly one action | //+------------------------------------------------------------------+ __kernel void DPCCRolloutCommitAction(__global const float *plans, __global const float *context, __global const float *state_in, __global const float *risk_in, __global const float *factual_context, __global float *state_out, __global float *rewards, __global float *status, __global float *branch_diagnostic, uint plan_offset, uint K, uint E, uint context_M, uint off_account, uint off_instrument, uint off_limits, uint off_execution, uint off_count, uint off_calendar, uint off_active, uint total, uint episode_step, uint episode_horizon, float gamma, float volume_scale, float price_scale, float max_quote_age, float max_last_age, float money_scale, uint expected_basis, uint expected_adapter, uint expected_calculation_contract, uint currency_digits, uint observed_commit, uint apply_action) { const uint k = (uint)get_global_id(0); if(k >= K) return; const size_t state_offset = (size_t)k * DPCC_NATIVE_CONTINUATION_FLOATS; const size_t risk_offset = (size_t)k * DPCC_NATIVE_PENDING_PHASE_FLOATS; rewards[k] = 0.0f; status[k] = 0.0f; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = 0.0f; branch_diagnostic[(size_t)k * 4] = -1.0f; for(int i = 0; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) state_out[state_offset + i] = 0.0f; if(episode_horizon == 0 || episode_horizon > DPCC_NATIVE_MAX_EXACT_FLOAT_INDEX || episode_step >= episode_horizon || observed_commit > 1U || apply_action > 1U) { status[k] = (float)DPCC_STATUS_INVALID; return; } __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; __global const float *account_context = context + off_account; uint context_failure_line=0; const int checked = DPCCValidateRolloutContext(context, state_in + state_offset, account_context, k, K, 1, E, context_M, episode_step, off_account, off_instrument, off_limits, off_execution, off_count, off_calendar, off_active, total, expected_basis, expected_adapter, gamma, volume_scale, price_scale, max_quote_age, max_last_age, money_scale, expected_calculation_contract, currency_digits, 0, 0, 1U, &context_failure_line); if(checked != 0) { status[k] = (float)checked; return; } if(observed_commit != 0U) { const int factual_checked = DPCCValidateObservedCommitContext(context, factual_context, state_in + state_offset, k, K, E, context_M, episode_step, off_account, off_instrument, off_limits, off_execution, off_count, off_calendar, off_active, total, expected_basis, expected_adapter, gamma, volume_scale, price_scale, max_quote_age, max_last_age, money_scale, expected_calculation_contract, currency_digits); if(factual_checked != 0) { status[k] = (float)factual_checked; return; } } if(context[off_active] != 1.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } DPCCAccountState state; for(int i = 0; i < 24; i++) state.account[i] = state_in[state_offset + i]; for(int i = 24; i < DPCC_NATIVE_CONTINUATION_FLOATS; i++) if(!isfinite(state_in[state_offset + i])) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } state.equity_anchor = state_in[state_offset + 24]; state.entry_interest = state_in[state_offset + 25]; state.native_mark = state_in[state_offset + 26]; state.accrued_interest = state_in[state_offset + 27]; state.maximum_drawdown = state_in[state_offset + 28]; if(!DPCCInteger(state_in[state_offset + 29], 0.0f, 8.0f) || !DPCCInteger(state_in[state_offset + 30], -1.0f, (float)episode_step) || !DPCCInteger(state_in[state_offset + 31], 0.0f, 3.0f) || !DPCCInteger(state_in[state_offset + 32], 0.0f, 0.0f) || state.maximum_drawdown < 0.0f) { status[k] = (float)DPCC_STATUS_INVALID; return; } state.currency_digits = (uint)state_in[state_offset + 29]; state.source_action = (int)state_in[state_offset + 30]; state.exit_kind = (int)state_in[state_offset + 31]; state.numeric_status = (int)state_in[state_offset + 32]; if(state.currency_digits != currency_digits) { status[k] = (float)DPCC_STATUS_INVALID; return; } DPCCRiskState risk; const int risk_status = DPCCLoadRiskContinuation(&risk, risk_in, risk_offset); if(risk_status != 0) { status[k] = (float)risk_status; return; } __global const float *action = plans; if(apply_action != 0U) action += (size_t)plan_offset; int valid_action = 1; if(apply_action != 0U) for(int i = 0; i < 6; i++) if(!isfinite(action[i])) valid_action = 0; if(apply_action != 0U && valid_action) { const float target = action[0] - action[3]; if(!isfinite(target)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } if(action[0] < 0.0f || action[3] < 0.0f || (target > 0.0f && (action[1] < 0.0f || action[2] < 0.0f)) || (target < 0.0f && (action[4] < 0.0f || action[5] < 0.0f))) valid_action = 0; } if(!valid_action) state.numeric_status = DPCC_STATUS_INVALID; const float equity_before = state.account[1]; __global const float *market_context = observed_commit != 0U ? factual_context : context; __global const float *ohlc = market_context + (size_t)k * 4; __global const float *execution = market_context + off_execution + (size_t)k * 24; __global const float *calendar = market_context + off_calendar + (size_t)k * E * 6; const uint event_count = (uint)market_context[off_count]; uint cursor = 0; if(observed_commit != 0U) cursor = (uint)context[off_count]; else while(cursor < event_count && calendar[cursor * 6] <= 0.0f) cursor++; float branch[4]; for(int i = 0; i < 4; i++) branch[i] = (i == 0 ? -1.0f : 0.0f); DPCCAccountCommitPrepared(&state, &risk, action, ohlc, instrument, execution, limits, calendar, event_count, cursor, volume_scale, price_scale, episode_step, episode_horizon, branch, apply_action); for(int i = 0; i < 24; i++) if(!isfinite(state.account[i])) state.numeric_status = DPCC_STATUS_NUMERIC; if(state.numeric_status != 0) { status[k] = (float)state.numeric_status; return; } const float reward = state.account[1] - equity_before; if(!isfinite(reward) || !isfinite(state.equity_anchor) || !isfinite(state.entry_interest) || !isfinite(state.native_mark) || !isfinite(state.accrued_interest) || !isfinite(state.maximum_drawdown)) { status[k] = (float)DPCC_STATUS_NUMERIC; return; } rewards[k] = reward; for(int i = 0; i < 24; i++) state_out[state_offset + i] = state.account[i]; state_out[state_offset + 24] = state.equity_anchor; state_out[state_offset + 25] = state.entry_interest; state_out[state_offset + 26] = state.native_mark; state_out[state_offset + 27] = state.accrued_interest; state_out[state_offset + 28] = state.maximum_drawdown; state_out[state_offset + 29] = (float)state.currency_digits; state_out[state_offset + 30] = (float)state.source_action; state_out[state_offset + 31] = (float)state.exit_kind; state_out[state_offset + 32] = (float)state.numeric_status; for(int i = 0; i < 4; i++) branch_diagnostic[(size_t)k * 4 + i] = branch[i]; } //+------------------------------------------------------------------+ //| Slot feasibility and one-writer aggregate system status | //+------------------------------------------------------------------+ __kernel void DPCCRolloutReduce(__global const float *system_status, __global const float *violations, __global const float *slot_active, __global float *validity, __global float *aggregate_system, uint K, uint slots, uint slot_active_offset) { const size_t slot = get_global_id(0); if(slot < slots) { float valid = slot_active[(size_t)slot_active_offset + slot] == 1.0f ? 1.0f : 0.0f; for(uint k = 0; k < K; k++) { const size_t pair = slot * K + k; if(system_status[pair] != 0.0f || violations[pair * 8] >= 0.0f || violations[pair * 8 + 5] != 0.0f || !isfinite(violations[pair * 8 + 2]) || !isfinite(violations[pair * 8 + 3])) valid = 0.0f; } validity[slot] = valid; } else if(slot == slots) { float status = 0.0f; for(size_t pair = 0; pair < (size_t)slots * K; pair++) { const float current = system_status[pair]; if(!isfinite(current) || (current != 0.0f && current != 3.0f && current != 5.0f)) status = (float)DPCC_STATUS_NUMERIC; else status = fmax(status, current); } aggregate_system[0] = status; } } //+------------------------------------------------------------------+ //| DPCC device-resident LOCAL_REPAIR_V2 projection control | //+------------------------------------------------------------------+ #define DPCC_PC_STRIDE 24 #define DPCC_PS_STRIDE 12 #define DPCC_PD_STRIDE 12 #define DPCC_PROPOSALS 9 #define DPCC_PC_ACTIVE 0 #define DPCC_PC_STAGE 1 #define DPCC_PC_EVAL 2 #define DPCC_PC_ITER 3 #define DPCC_PC_COUNT 4 #define DPCC_PC_CURSOR 5 #define DPCC_PC_H 6 #define DPCC_PC_PHASE 7 #define DPCC_PC_K 8 #define DPCC_PC_CONSTRAINT 9 #define DPCC_PC_SOURCE 10 #define DPCC_PC_FEASIBLE 11 #define DPCC_PC_VMAX 12 #define DPCC_PC_VSUM 13 #define DPCC_PC_DISTANCE 14 #define DPCC_PC_IDENTITY 15 #define DPCC_PC_LAST_P 16 #define DPCC_PC_STATUS 17 #define DPCC_PC_SYSTEM 18 #define DPCC_PC_PROPOSED 19 #define DPCC_PC_VERIFIED 20 #define DPCC_PC_ORIGINAL_OK 21 void DPCCProjectionCopy(__global const float *source, __global float *destination, size_t source_offset, size_t destination_offset, size_t count) { __global const uint *source_bits = (__global const uint *)source; __global uint *destination_bits = (__global uint *)destination; for(size_t i = 0; i < count; i++) destination_bits[destination_offset + i] = source_bits[source_offset + i]; } void DPCCProjectionEncode(float q, float tp, float sl, __private float *action) { for(int a = 0; a < 6; a++) action[a] = 0.0f; if(q > 0.0f) { action[0] = q; action[1] = tp; action[2] = sl; } else if(q < 0.0f) { action[3] = -q; action[4] = tp; action[5] = sl; } } void DPCCProjectionBase(__global float *control, __global const float *summary) { control[DPCC_PC_FEASIBLE] = summary[0]; control[DPCC_PC_VMAX] = summary[1]; control[DPCC_PC_VSUM] = summary[2]; control[DPCC_PC_DISTANCE] = summary[3]; for(int i = 0; i < 5; i++) control[DPCC_PC_H + i] = summary[4 + i]; control[DPCC_PC_SYSTEM] = fmax(control[DPCC_PC_SYSTEM], summary[9]); } void DPCCProjectionOrder(__global float *control, __global float *order, uint H, uint lookback) { control[DPCC_PC_COUNT] = 0.0f; control[DPCC_PC_CURSOR] = 0.0f; for(uint i = 0; i < H; i++) order[i] = -1.0f; const int h = (int)control[DPCC_PC_H]; const int phase = (int)control[DPCC_PC_PHASE]; if(h < 0 || h >= (int)H || phase < 0 || phase > 3 || lookback == 0) return; const int last = phase == 0 ? h - 1 : h; if(last < 0) return; const int source = (int)control[DPCC_PC_SOURCE]; uint count = 0; if(source >= 0 && source <= last) order[count++] = (float)source; for(int u = last; u >= 0 && count < lookback; u--) if(u != source) order[count++] = (float)u; control[DPCC_PC_COUNT] = (float)count; } // Only impossible targets are excluded here. Exact pre-action permissions // after gaps/protective closes remain the shared rollout's responsibility. int DPCCProjectionDirection(float target, __global const float *account, __global const float *instrument) { const float initial = account[2]; const int mode = (int)instrument[29]; if((int)instrument[11] == DPCC_CALC_COLLATERAL) return(target == initial); if(target == 0.0f) return(1); if(mode == 0) return(target == initial); if(mode == 3) return(((target > 0.0f && initial > 0.0f) || (target < 0.0f && initial < 0.0f)) && fabs(target) <= fabs(initial)); if(mode == 1 && target < 0.0f) return(initial < 0.0f && fabs(target) <= fabs(initial)); if(mode == 2 && target > 0.0f) return(initial > 0.0f && fabs(target) <= fabs(initial)); return(1); } int DPCCProjectionRiskBetter(__global const float *candidate, float feasible, float vmax, float vsum, float epsilon) { if(candidate[0] == 1.0f) return(feasible == 0.0f); if(feasible == 1.0f) return(0); return(candidate[1] < vmax - epsilon || (fabs(candidate[1] - vmax) <= epsilon && candidate[2] < vsum - epsilon)); } int DPCCProjectionPrefer(__global const float *candidate, __global const float *best, float epsilon, float distance_epsilon) { if(candidate[0] != best[0]) return(candidate[0] > best[0]); if(candidate[0] == 0.0f) { if(DPCCProjectionRiskBetter(candidate, best[0], best[1], best[2], epsilon)) return(1); if(DPCCProjectionRiskBetter(best, candidate[0], candidate[1], candidate[2], epsilon)) return(0); if(fabs(candidate[1] - best[1]) > epsilon || fabs(candidate[2] - best[2]) > epsilon) return(0); } // Stable p is retained on a distance tie because the scan is ascending. return(candidate[3] < best[3] - distance_epsilon); } __kernel void DPCCProjectionInit(__global const float *plans, __global const float *context, __global float *original, __global float *current, __global float *base_active, __global float *final_active, __global float *controls, uint M, uint H, uint off_R_active, uint plan_offset) { const uint m = (uint)get_global_id(0); if(m >= M) return; __global float *control = controls + (size_t)m * DPCC_PC_STRIDE; for(int i = 0; i < DPCC_PC_STRIDE; i++) control[i] = 0.0f; control[DPCC_PC_STATUS] = 2.0f; control[DPCC_PC_LAST_P] = -1.0f; for(int i = DPCC_PC_H; i <= DPCC_PC_SOURCE; i++) control[i] = -1.0f; const float active = context[(size_t)off_R_active + m]; if(!DPCCBoolean(active)) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; else { control[DPCC_PC_ACTIVE] = active; control[DPCC_PC_STAGE] = active == 1.0f ? 1.0f : 0.0f; } const size_t row = (size_t)m * H * 6; DPCCProjectionCopy(plans, original, (size_t)plan_offset + row, row, (size_t)H * 6); DPCCProjectionCopy(plans, current, (size_t)plan_offset + row, row, (size_t)H * 6); for(uint h = 0; h < H; h++) { for(int a = 0; a < 6; a++) if(!isfinite(original[row + (size_t)h * 6 + a])) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; if(control[DPCC_PC_SYSTEM] == 0.0f && !isfinite(original[row + (size_t)h * 6] - original[row + (size_t)h * 6 + 3])) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; } base_active[m] = control[DPCC_PC_SYSTEM] == 0.0f ? control[DPCC_PC_ACTIVE] : 0.0f; final_active[m] = 0.0f; } __kernel void DPCCProjectionSummary(__global const float *plans, __global const float *original, __global const float *active, __global const float *returns, __global const float *states, __global const float *violations, __global float *summaries, __global const float *action_scale, uint M, uint K, uint H, uint P) { const size_t slot = get_global_id(0); if(P == 0 || slot >= (size_t)M * P) return; __global float *summary = summaries + slot * DPCC_PS_STRIDE; for(int i = 0; i < DPCC_PS_STRIDE; i++) summary[i] = 0.0f; for(int i = 4; i <= 8; i++) summary[i] = -1.0f; summary[10] = (float)(slot % P); const float enabled = active[slot]; if(!DPCCBoolean(enabled) || K == 0 || K > 64 || (P != 1 && P != DPCC_PROPOSALS)) { summary[9] = (float)DPCC_STATUS_INVALID; return; } summary[11] = enabled; if(enabled == 0.0f) return; for(int a = 0; a < 6; a++) if(!isfinite(action_scale[a]) || action_scale[a] <= 0.0f) { summary[9] = (float)DPCC_STATUS_INVALID; return; } summary[0] = 1.0f; for(uint k = 0; k < K; k++) { const size_t pair = slot * K + k; __global const float *violation = violations + pair * 8; for(int i = 0; i < 8; i++) if(!isfinite(violation[i])) summary[9] = (float)DPCC_STATUS_NUMERIC; if(!isfinite(returns[pair])) summary[9] = (float)DPCC_STATUS_NUMERIC; for(size_t i = 0; i < ((size_t)H + 1) * 24; i++) if(!isfinite(states[pair * ((size_t)H + 1) * 24 + i])) summary[9] = (float)DPCC_STATUS_NUMERIC; if(summary[9] != 0.0f) continue; if(!DPCCInteger(violation[0], -1.0f, (float)H - 1.0f) || !DPCCInteger(violation[1], -1.0f, 15.0f) || !DPCCInteger(violation[6], -1.0f, 3.0f) || !DPCCInteger(violation[7], -1.0f, (float)H - 1.0f) || violation[2] < 0.0f || violation[3] < 0.0f || (violation[5] != 0.0f && violation[5] != 3.0f && violation[5] != 5.0f)) { summary[9] = (float)DPCC_STATUS_NUMERIC; continue; } if(violation[5] == (float)DPCC_STATUS_NUMERIC) summary[9] = (float)DPCC_STATUS_NUMERIC; if(violation[0] >= 0.0f || violation[5] != 0.0f || violation[2] != 0.0f || violation[3] != 0.0f) summary[0] = 0.0f; summary[1] = fmax(summary[1], violation[2]); summary[2] += violation[3]; if(violation[0] >= 0.0f) { if(violation[1] < 0.0f || violation[6] < 0.0f) { summary[9] = (float)DPCC_STATUS_NUMERIC; continue; } const int earlier = summary[4] < 0.0f || violation[0] < summary[4] || (violation[0] == summary[4] && (violation[6] < summary[5] || (violation[6] == summary[5] && ((float)k < summary[6] || ((float)k == summary[6] && violation[1] < summary[7]))))); if(earlier) { summary[4] = violation[0]; summary[5] = violation[6]; summary[6] = (float)k; summary[7] = violation[1]; summary[8] = violation[7]; } } } const size_t row = slot * H * 6; const size_t input_row = (slot / P) * H * 6; float distance = 0.0f; for(uint h = 0; h < H; h++) for(int a = 0; a < 6; a++) { const size_t i = (size_t)h * 6 + a; const float difference = (plans[row + i] - original[input_row + i]) / action_scale[a]; distance += difference * difference; } summary[3] = distance; if(!isfinite(distance) || !isfinite(summary[1]) || !isfinite(summary[2])) summary[9] = (float)DPCC_STATUS_NUMERIC; if(summary[9] != 0.0f) summary[0] = 0.0f; } __kernel void DPCCProjectionPrepare(__global float *current, __global const float *context, __global const float *summaries, __global float *controls, __global float *base_active, __global float *final_active, __global float *orders, uint M, uint H, uint stage, uint off_account, uint off_instrument, uint off_limits, uint max_eval, uint max_iter, uint lookback) { const uint m = (uint)get_global_id(0); if(m >= M) return; __global float *control = controls + (size_t)m * DPCC_PC_STRIDE; base_active[m] = 0.0f; if(control[DPCC_PC_ACTIVE] == 0.0f || control[DPCC_PC_SYSTEM] != 0.0f) return; if((stage != 1 && stage != 2) || max_eval == 0 || max_eval > 160 || max_iter == 0 || max_iter > 16 || lookback > H) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } if(control[DPCC_PC_STAGE] != (float)stage) return; __global const float *summary = summaries + (size_t)m * DPCC_PS_STRIDE; DPCCProjectionBase(control, summary); if(summary[11] != 1.0f) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; if(control[DPCC_PC_SYSTEM] != 0.0f) return; control[DPCC_PC_EVAL] += 1.0f; if(control[DPCC_PC_EVAL] > (float)max_eval) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } if(stage == 1) { control[DPCC_PC_ORIGINAL_OK] = summary[0]; if(summary[0] == 1.0f) { control[DPCC_PC_IDENTITY] = 1.0f; control[DPCC_PC_VERIFIED] = 1.0f; control[DPCC_PC_STATUS] = 0.0f; control[DPCC_PC_STAGE] = 5.0f; return; } // Do not mutate the original without both canonical and final reserves. if((float)max_eval - control[DPCC_PC_EVAL] < 2.0f) { control[DPCC_PC_STAGE] = 5.0f; control[DPCC_PC_STATUS] = 2.0f; return; } __global const float *limits = context + off_limits; __global const float *instrument = context + off_instrument; if(!DPCCInteger(limits[20], 0.0f, 511.0f)) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } const uint mask = (uint)limits[20]; const size_t row = (size_t)m * H * 6; for(uint h = 0; h < H; h++) { const size_t offset = row + (size_t)h * 6; float q = current[offset] - current[offset + 3]; float tp = q > 0.0f ? current[offset + 1] : current[offset + 4]; float sl = q > 0.0f ? current[offset + 2] : current[offset + 5]; q = copysign(fmin(fabs(q), DPCC_ACTION_VOLUME_MAX), q); if(mask & 1U) q = copysign(fmin(fabs(q), limits[0]), q); tp = fmax(0.0f, tp); sl = fmax(0.0f, sl); if(q != 0.0f) { if(((mask & 128U) && instrument[7] <= 0.0f) || ((mask & 256U) && instrument[6] <= 0.0f)) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } if(mask & 128U) sl = clamp(sl, limits[7] / instrument[7], limits[8] / instrument[7]); if(mask & 256U) tp = clamp(tp, limits[9] / instrument[6], limits[10] / instrument[6]); } float action[6]; DPCCProjectionEncode(q, tp, sl, action); for(int a = 0; a < 6; a++) { if(!isfinite(action[a])) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } current[offset + a] = action[a]; } } control[DPCC_PC_STAGE] = 2.0f; base_active[m] = 1.0f; return; } DPCCProjectionOrder(control, orders + (size_t)m * H, H, lookback); if(summary[0] == 1.0f || control[DPCC_PC_COUNT] == 0.0f || control[DPCC_PC_EVAL] >= (float)max_eval - 1.0f) { control[DPCC_PC_STAGE] = 4.0f; final_active[m] = 1.0f; } else control[DPCC_PC_STAGE] = 3.0f; } __kernel void DPCCProjectionProposals(__global const float *current, __global const float *context, __global float *controls, __global const float *orders, __global float *proposals, __global float *proposal_active, uint M, uint H, uint off_account, uint off_instrument, uint off_limits, uint max_eval, uint max_iter) { const uint m = (uint)get_global_id(0); if(m >= M) return; __global float *control = controls + (size_t)m * DPCC_PC_STRIDE; const size_t row = (size_t)m * H * 6; for(uint p = 0; p < DPCC_PROPOSALS; p++) { const size_t slot = (size_t)m * DPCC_PROPOSALS + p; proposal_active[slot] = 0.0f; DPCCProjectionCopy(current, proposals, row, slot * H * 6, (size_t)H * 6); } control[DPCC_PC_PROPOSED] = 0.0f; if(control[DPCC_PC_STAGE] != 3.0f || control[DPCC_PC_SYSTEM] != 0.0f || control[DPCC_PC_ACTIVE] != 1.0f) return; const int available = (int)max_eval - (int)control[DPCC_PC_EVAL] - 1; if(available <= 0 || control[DPCC_PC_ITER] >= (float)max_iter || control[DPCC_PC_CURSOR] >= control[DPCC_PC_COUNT]) return; const uint cursor = (uint)control[DPCC_PC_CURSOR]; const float selected = orders[(size_t)m * H + cursor]; if(!DPCCInteger(selected, 0.0f, (float)H - 1.0f)) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } const uint u = (uint)selected; __global const float *account = context + off_account; __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; if(!DPCCInteger(limits[20], 0.0f, 511.0f)) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } const uint mask = (uint)limits[20]; const size_t action_offset = row + (size_t)u * 6; const float q = current[action_offset] - current[action_offset + 3]; const float original_tp = current[action_offset + (q > 0.0f ? 1 : 4)]; const float original_sl = current[action_offset + (q > 0.0f ? 2 : 5)]; int permitted[DPCC_PROPOSALS]; for(uint p = 0; p < DPCC_PROPOSALS; p++) permitted[p] = 0; int emitted = 0; for(uint p = 0; p < DPCC_PROPOSALS; p++) { float target = q; float tp = original_tp; float sl = original_sl; if(p < 4) { const float multiplier = p == 0 ? 0.75f : (p == 1 ? 0.5f : (p == 2 ? 0.25f : 0.0f)); target = q * multiplier; } else if(p == 4) { if(u > 0) { const size_t previous = row + ((size_t)u - 1) * 6; target = current[previous] - current[previous + 3]; tp = current[previous + (target > 0.0f ? 1 : 4)]; sl = current[previous + (target > 0.0f ? 2 : 5)]; } else { target = account[2]; tp = 0.0f; sl = 0.0f; if(target != 0.0f) { if((account[17] == 1.0f && instrument[7] <= 0.0f) || (account[18] == 1.0f && instrument[6] <= 0.0f) || instrument[0] <= 0.0f) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } const float reference = account[target > 0.0f ? 16 : 15]; if(account[17] == 1.0f) sl = (target > 0.0f ? reference - account[4] : account[4] - reference) / instrument[0] / instrument[7]; if(account[18] == 1.0f) tp = (target > 0.0f ? account[5] - reference : reference - account[5]) / instrument[0] / instrument[6]; } } } else if(p == 5 || p == 6) { if((mask & 128U) == 0) continue; if(instrument[7] <= 0.0f) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } sl = limits[p == 5 ? 7 : 8] / instrument[7]; } else { if((mask & 256U) == 0) continue; if(instrument[6] <= 0.0f) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_INVALID; return; } tp = limits[p == 7 ? 9 : 10] / instrument[6]; } float action[6]; DPCCProjectionEncode(target, tp, sl, action); for(int a = 0; a < 6; a++) if(!isfinite(action[a])) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } if(!DPCCProjectionDirection(target, account, instrument)) continue; const size_t slot = (size_t)m * DPCC_PROPOSALS + p; const size_t plan = slot * H * 6; for(int a = 0; a < 6; a++) proposals[plan + (size_t)u * 6 + a] = action[a]; int duplicate = 1; for(size_t i = 0; i < (size_t)H * 6; i++) if(proposals[plan + i] != current[row + i]) { duplicate = 0; break; } for(uint previous_p = 0; previous_p < p && !duplicate; previous_p++) if(permitted[previous_p]) { duplicate = 1; const size_t previous_plan = ((size_t)m * DPCC_PROPOSALS + previous_p) * H * 6; for(size_t i = 0; i < (size_t)H * 6; i++) if(proposals[plan + i] != proposals[previous_plan + i]) { duplicate = 0; break; } } if(duplicate) continue; permitted[p] = 1; if(emitted < available) { proposal_active[slot] = 1.0f; emitted++; } } control[DPCC_PC_PROPOSED] = (float)emitted; } __kernel void DPCCProjectionSelect(__global const float *proposals, __global const float *proposal_active, __global const float *summaries, __global float *current, __global float *controls, __global float *orders, __global float *final_active, uint M, uint H, uint max_eval, uint max_iter, uint lookback, float improvement_atol, float distance_tie_atol) { const uint m = (uint)get_global_id(0); if(m >= M) return; __global float *control = controls + (size_t)m * DPCC_PC_STRIDE; if(control[DPCC_PC_STAGE] != 3.0f || control[DPCC_PC_SYSTEM] != 0.0f) return; int best = -1; uint evaluated = 0; for(uint p = 0; p < DPCC_PROPOSALS; p++) { const size_t slot = (size_t)m * DPCC_PROPOSALS + p; if(!DPCCBoolean(proposal_active[slot])) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } if(proposal_active[slot] == 0.0f) continue; evaluated++; __global const float *summary = summaries + slot * DPCC_PS_STRIDE; control[DPCC_PC_SYSTEM] = fmax(control[DPCC_PC_SYSTEM], summary[9]); if(summary[11] != 1.0f || summary[10] != (float)p) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; if(control[DPCC_PC_SYSTEM] != 0.0f) continue; if(!DPCCProjectionRiskBetter(summary, control[DPCC_PC_FEASIBLE], control[DPCC_PC_VMAX], control[DPCC_PC_VSUM], improvement_atol)) continue; if(best < 0 || DPCCProjectionPrefer(summary, summaries + ((size_t)m * DPCC_PROPOSALS + (uint)best) * DPCC_PS_STRIDE, improvement_atol, distance_tie_atol)) best = (int)p; } control[DPCC_PC_EVAL] += (float)evaluated; if(control[DPCC_PC_EVAL] > (float)max_eval - 1.0f || control[DPCC_PC_PROPOSED] != (float)evaluated) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; if(control[DPCC_PC_SYSTEM] != 0.0f) return; if(best >= 0) { const size_t slot = (size_t)m * DPCC_PROPOSALS + (uint)best; DPCCProjectionCopy(proposals, current, slot * H * 6, (size_t)m * H * 6, (size_t)H * 6); DPCCProjectionBase(control, summaries + slot * DPCC_PS_STRIDE); control[DPCC_PC_ITER] += 1.0f; control[DPCC_PC_LAST_P] = (float)best; DPCCProjectionOrder(control, orders + (size_t)m * H, H, lookback); } else control[DPCC_PC_CURSOR] += 1.0f; if(control[DPCC_PC_FEASIBLE] == 1.0f || control[DPCC_PC_ITER] >= (float)max_iter || control[DPCC_PC_EVAL] >= (float)max_eval - 1.0f || control[DPCC_PC_CURSOR] >= control[DPCC_PC_COUNT]) { control[DPCC_PC_STAGE] = 4.0f; final_active[m] = 1.0f; } } __kernel void DPCCProjectionPublish(__global const float *original, __global const float *current, __global const float *final_summaries, __global float *controls, __global float *result, __global float *correction_cost, __global float *candidate_status, __global float *diagnostics, uint M, uint H, uint max_eval) { const uint m = (uint)get_global_id(0); if(m >= M) return; __global float *control = controls + (size_t)m * DPCC_PC_STRIDE; const size_t row = (size_t)m * H * 6; const size_t diagnostic = (size_t)m * DPCC_PD_STRIDE; correction_cost[m] = 0.0f; candidate_status[m] = 2.0f; for(int i = 0; i < DPCC_PD_STRIDE; i++) diagnostics[diagnostic + i] = 0.0f; if(control[DPCC_PC_SYSTEM] != 0.0f) return; if(control[DPCC_PC_ACTIVE] == 1.0f && control[DPCC_PC_STAGE] == 4.0f) { __global const float *summary = final_summaries + (size_t)m * DPCC_PS_STRIDE; DPCCProjectionBase(control, summary); if(summary[11] != 1.0f) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; control[DPCC_PC_EVAL] += 1.0f; if(control[DPCC_PC_EVAL] > (float)max_eval) control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; if(control[DPCC_PC_SYSTEM] != 0.0f) return; control[DPCC_PC_STATUS] = summary[0] == 1.0f ? 0.0f : 2.0f; control[DPCC_PC_VERIFIED] = 1.0f; control[DPCC_PC_STAGE] = 5.0f; } else if(control[DPCC_PC_ACTIVE] == 1.0f && control[DPCC_PC_STAGE] != 5.0f) { control[DPCC_PC_SYSTEM] = (float)DPCC_STATUS_NUMERIC; return; } DPCCProjectionCopy(control[DPCC_PC_IDENTITY] == 1.0f ? original : current, result, row, row, (size_t)H * 6); if(control[DPCC_PC_ACTIVE] == 1.0f) { correction_cost[m] = control[DPCC_PC_IDENTITY] == 1.0f ? 0.0f : control[DPCC_PC_DISTANCE]; candidate_status[m] = control[DPCC_PC_STATUS]; } diagnostics[diagnostic] = control[DPCC_PC_EVAL]; diagnostics[diagnostic + 1] = control[DPCC_PC_ITER]; diagnostics[diagnostic + 2] = control[DPCC_PC_VMAX]; diagnostics[diagnostic + 3] = control[DPCC_PC_VSUM]; for(int i = 0; i < 5; i++) diagnostics[diagnostic + 4 + i] = control[DPCC_PC_H + i]; diagnostics[diagnostic + 9] = control[DPCC_PC_LAST_P]; diagnostics[diagnostic + 10] = control[DPCC_PC_ACTIVE]; diagnostics[diagnostic + 11] = control[DPCC_PC_VERIFIED]; } __kernel void DPCCProjectionReduce(__global const float *controls, __global float *aggregate, uint M, uint mode) { if(get_global_id(0) != 0) return; float system = 0.0f; int searching = 0; int exhausted = 0; for(uint m = 0; m < M; m++) { __global const float *control = controls + (size_t)m * DPCC_PC_STRIDE; const float status = control[DPCC_PC_SYSTEM]; if(!isfinite(status) || (status != 0.0f && status != 3.0f && status != 5.0f)) system = (float)DPCC_STATUS_NUMERIC; else system = fmax(system, status); if(control[DPCC_PC_ACTIVE] == 1.0f) { if(control[DPCC_PC_STAGE] == 3.0f) searching = 1; if(control[DPCC_PC_STATUS] == 2.0f) exhausted = 1; if(mode == 1 && control[DPCC_PC_STAGE] != 5.0f) system = (float)DPCC_STATUS_NUMERIC; } } if(mode > 1) system = (float)DPCC_STATUS_INVALID; aggregate[0] = system != 0.0f ? system : (mode == 0 ? (float)searching : (exhausted ? 2.0f : 0.0f)); } //+------------------------------------------------------------------+ //| Sparse branch derivative of the accepted projection operations | //+------------------------------------------------------------------+ typedef struct { int2 index; float2 coefficient; int valid; } DPCCTracePair; DPCCTracePair DPCCTraceZero(void) { DPCCTracePair result; result.index = (int2)(-1, -1); result.coefficient = (float2)(0.0f, 0.0f); result.valid = 1; return result; } DPCCTracePair DPCCTraceRead(__global const int *indices, __global const float *coefficients, size_t scalar) { DPCCTracePair result; result.index = vload2(scalar, indices); result.coefficient = vload2(scalar, coefficients); result.valid = 1; return result; } DPCCTracePair DPCCTraceCombine(DPCCTracePair a, DPCCTracePair b, float sa, float sb) { DPCCTracePair result = DPCCTraceZero(); int source[4] = {a.index.x, a.index.y, b.index.x, b.index.y}; float value[4] = {a.coefficient.x * sa, a.coefficient.y * sa, b.coefficient.x * sb, b.coefficient.y * sb}; result.valid = a.valid && b.valid; for(int i = 0; i < 4; i++) { if(!isfinite(value[i])) result.valid = 0; for(int j = 0; j < i; j++) if(source[i] >= 0 && source[i] == source[j]) { value[j] += value[i]; source[i] = -1; value[i] = 0.0f; break; } } int count = 0; for(int i = 0; i < 4; i++) if(source[i] >= 0 && value[i] != 0.0f) { if(!isfinite(value[i]) || count >= 2) result.valid = 0; else { result.index[count] = source[i]; result.coefficient[count] = value[i]; } count++; } return result; } void DPCCTraceWrite(__global int *indices, __global float *coefficients, size_t scalar, DPCCTracePair value, __global float *error) { if(!value.valid) *error = 1.0f; vstore2(value.index, scalar, indices); vstore2(value.coefficient, scalar, coefficients); } void DPCCTraceEncode(float q, DPCCTracePair dq, DPCCTracePair dtp, DPCCTracePair dsl, __private DPCCTracePair *action) { for(int a = 0; a < 6; a++) action[a] = DPCCTraceZero(); if(q > 0.0f || q < 0.0f) { const int side = q > 0.0f ? 0 : 3; action[side] = DPCCTraceCombine(dq, DPCCTraceZero(), q > 0.0f ? 1.0f : -1.0f, 0.0f); action[side + 1] = dtp; action[side + 2] = dsl; } } __kernel void DPCCProjectionTrace(__global int *current_index, __global float *current_coefficient, __global int *proposal_index, __global float *proposal_coefficient, __global int *frame_index, __global float *frame_coefficient, __global float *wave, __global float *errors, __global float *common, __global const float *current, __global const float *context, __global const float *controls, __global const float *orders, __global const float *summaries, uint M, uint H, uint stage, uint frame, uint off_account, uint off_instrument, uint off_limits, uint max_eval) { const uint m = (uint)get_global_id(0); if(stage == 5) { if(m == 0) { common[0] = 0.0f; for(size_t row = 0; row < (size_t)M * H * 6; row++) if(errors[row] != 0.0f) common[0] = 1.0f; } return; } if(m >= M) return; const uint L = H * 6; const size_t row = (size_t)m * L; __global const float *control = controls + (size_t)m * DPCC_PC_STRIDE; __global float *error = errors + m; if(stage == 0) { *error = 0.0f; for(uint r = 0; r < L; r++) { DPCCTracePair identity = DPCCTraceZero(); identity.index.x = (int)r; identity.coefficient.x = 1.0f; DPCCTraceWrite(current_index, current_coefficient, row + r, identity, error); } return; } if(stage == 1) { __global const float *summary = summaries + (size_t)m * DPCC_PS_STRIDE; if(control[DPCC_PC_ACTIVE] != 1.0f || control[DPCC_PC_SYSTEM] != 0.0f || control[DPCC_PC_STAGE] != 1.0f || summary[11] != 1.0f || summary[0] == 1.0f || (float)max_eval - (control[DPCC_PC_EVAL] + 1.0f) < 2.0f) return; __global const float *instrument = context + off_instrument; __global const float *limits = context + off_limits; if(!DPCCInteger(limits[20], 0.0f, 511.0f)) { *error = 1.0f; return; } const uint mask = (uint)limits[20]; for(uint h = 0; h < H; h++) { const size_t offset = row + (size_t)h * 6; float q = current[offset] - current[offset + 3]; const int side = q > 0.0f ? 0 : 3; float tp = current[offset + side + 1]; float sl = current[offset + side + 2]; DPCCTracePair dq = DPCCTraceCombine( DPCCTraceRead(current_index, current_coefficient, offset), DPCCTraceRead(current_index, current_coefficient, offset + 3), 1.0f, -1.0f); DPCCTracePair dtp = DPCCTraceRead(current_index, current_coefficient, offset + side + 1); DPCCTracePair dsl = DPCCTraceRead(current_index, current_coefficient, offset + side + 2); if(fabs(q) >= DPCC_ACTION_VOLUME_MAX) dq = DPCCTraceZero(); q = copysign(fmin(fabs(q), DPCC_ACTION_VOLUME_MAX), q); if(mask & 1U) { if(fabs(q) >= limits[0]) dq = DPCCTraceZero(); q = copysign(fmin(fabs(q), limits[0]), q); } if(tp <= 0.0f) dtp = DPCCTraceZero(); if(sl <= 0.0f) dsl = DPCCTraceZero(); tp = fmax(0.0f, tp); sl = fmax(0.0f, sl); if(q != 0.0f) { if(mask & 128U) if(!(sl > limits[7] / instrument[7] && sl < limits[8] / instrument[7])) dsl = DPCCTraceZero(); if(mask & 256U) if(!(tp > limits[9] / instrument[6] && tp < limits[10] / instrument[6])) dtp = DPCCTraceZero(); } DPCCTracePair action[6]; DPCCTraceEncode(q, dq, dtp, dsl, action); for(int a = 0; a < 6; a++) DPCCTraceWrite(current_index, current_coefficient, offset + a, action[a], error); } return; } if(stage == 2) { wave[(size_t)m * 2] = -1.0f; wave[(size_t)m * 2 + 1] = control[DPCC_PC_ITER]; if(control[DPCC_PC_STAGE] != 3.0f || control[DPCC_PC_ACTIVE] != 1.0f || control[DPCC_PC_SYSTEM] != 0.0f || control[DPCC_PC_CURSOR] >= control[DPCC_PC_COUNT]) return; const float selected = orders[(size_t)m * H + (uint)control[DPCC_PC_CURSOR]]; if(!DPCCInteger(selected, 0.0f, (float)H - 1.0f)) { *error = 1.0f; return; } const uint u = (uint)selected; wave[(size_t)m * 2] = selected; for(uint p = 0; p < 9; p++) { size_t offset = row + (size_t)u * 6; float q = current[offset] - current[offset + 3]; if(p == 4 && u > 0) { offset -= 6; q = current[offset] - current[offset + 3]; } const int side = q > 0.0f ? 0 : 3; DPCCTracePair dq = DPCCTraceCombine( DPCCTraceRead(current_index, current_coefficient, offset), DPCCTraceRead(current_index, current_coefficient, offset + 3), 1.0f, -1.0f); DPCCTracePair dtp = DPCCTraceRead(current_index, current_coefficient, offset + side + 1); DPCCTracePair dsl = DPCCTraceRead(current_index, current_coefficient, offset + side + 2); if(p < 4) { const float factor = p == 0 ? 0.75f : (p == 1 ? 0.5f : (p == 2 ? 0.25f : 0.0f)); q *= factor; dq = DPCCTraceCombine(dq, DPCCTraceZero(), factor, 0.0f); } if(p == 4 && u == 0) { q = context[off_account + 2]; dq = DPCCTraceZero(); dtp = DPCCTraceZero(); dsl = DPCCTraceZero(); } if(p == 5 || p == 6) dsl = DPCCTraceZero(); if(p == 7 || p == 8) dtp = DPCCTraceZero(); DPCCTracePair action[6]; DPCCTraceEncode(q, dq, dtp, dsl, action); for(uint a = 0; a < 6; a++) DPCCTraceWrite(proposal_index, proposal_coefficient, ((size_t)m * 9 + p) * 6 + a, action[a], error); } return; } if(stage == 3) { const float before = wave[(size_t)m * 2 + 1]; if(control[DPCC_PC_ITER] == before) return; const float u = wave[(size_t)m * 2]; const float p = control[DPCC_PC_LAST_P]; if(control[DPCC_PC_ITER] != before + 1.0f || !DPCCInteger(u, 0.0f, (float)H - 1.0f) || !DPCCInteger(p, 0.0f, 8.0f) || control[DPCC_PC_SYSTEM] != 0.0f) { *error = 1.0f; return; } for(uint a = 0; a < 6; a++) DPCCTraceWrite(current_index, current_coefficient, row + (uint)u * 6 + a, DPCCTraceRead(proposal_index, proposal_coefficient, ((size_t)m * 9 + (uint)p) * 6 + a), error); return; } if(stage == 4) { const size_t destination = ((size_t)frame * M + m) * L; const float status = control[DPCC_PC_STATUS]; const int eligible = control[DPCC_PC_ACTIVE] == 1.0f && (status == 0.0f || status == 2.0f) && control[DPCC_PC_VERIFIED] == 1.0f && control[DPCC_PC_SYSTEM] == 0.0f; for(uint r = 0; r < L; r++) { DPCCTracePair value = DPCCTraceZero(); if(eligible) { if(control[DPCC_PC_IDENTITY] == 1.0f) { value.index.x = (int)r; value.coefficient.x = 1.0f; } else value = DPCCTraceRead(current_index, current_coefficient, row + r); } DPCCTraceWrite(frame_index, frame_coefficient, destination + r, value, error); } return; } *error = 1.0f; } //+------------------------------------------------------------------+ //| Deterministic gather VJP, one independent input scalar per lane | //+------------------------------------------------------------------+ __kernel void DPCCProjectionTraceVJP(__global const int *indices, __global const float *coefficients, __global const float *gradient_out, __global float *gradient_in, __global float *errors, uint M, uint H, uint frame) { const uint i = (uint)get_global_id(0); const uint L = H * 6; if(i >= M * L) return; const uint m = i / L; const int source = (int)(i % L); const size_t base = ((size_t)frame * M + m) * L; float sum = 0.0f; int valid = 1; for(uint r = 0; r < L; r++) { const float upstream = gradient_out[(size_t)m * L + r]; if(!isfinite(upstream)) valid = 0; for(uint s = 0; s < 2; s++) { const size_t position = (base + r) * 2 + s; const int index = indices[position]; const float coefficient = coefficients[position]; if(index < -1 || index >= (int)L || !isfinite(coefficient) || (index == -1 && coefficient != 0.0f)) valid = 0; if(index == source) sum += coefficient * upstream; } } errors[i] = valid && isfinite(sum) ? 0.0f : 1.0f; gradient_in[i] = sum; } //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| DPCC shared exact-chain operators and native affine preflight | //+------------------------------------------------------------------+ inline int DPCCOperatorProductSum(float left, float right, __private float *sum) { if(!isfinite(left) || !isfinite(right)) return(0); const float product = left * right; if(!isfinite(product)) return(0); const float next = *sum + product; if(!isfinite(next)) return(0); *sum = next; return(1); } //+------------------------------------------------------------------+ //| DPCCDecoderPack permutes KVHD and KHVD without host transfer | //+------------------------------------------------------------------+ __kernel void DPCCDecoderPack(__global const float *source, __global float *destination, __global float *flags, uint K, uint V, uint H, uint D, uint inverse) { const size_t index = get_global_id(0); const size_t total = (size_t)K * V * H * D; if(index >= total) return; if(K == 0 || V == 0 || H == 0 || D == 0 || inverse > 1) { flags[index] = (float)DPCC_STATUS_INVALID; return; } const size_t d = index % D; const size_t v = index / D % V; const size_t h = index / ((size_t)V * D) % H; const size_t k = index / ((size_t)V * H * D); const size_t z = (((k * V + v) * H + h) * D + d); const size_t from = inverse == 0 ? z : index; const size_t to = inverse == 0 ? index : z; const float value = source[from]; flags[index] = isfinite(value) ? 0.0f : (float)DPCC_STATUS_NUMERIC; destination[to] = isfinite(value) ? value : 0.0f; } //+------------------------------------------------------------------+ //| DPCCOperatorActivation preserves the admitted clamped forward | //+------------------------------------------------------------------+ inline int DPCCOperatorActivation(float z, int activation, __private float *value, __private float *derivative) { if(!isfinite(z)) return(DPCC_STATUS_NUMERIC); if(activation == ActFunc_None) { *value = z; *derivative = 1.0f; } else if(activation == ActFunc_TANH) { *value = tanh(clamp(z, -20.0f, 20.0f)); *derivative = fabs(z) < 20.0f ? 1.0f - *value * *value : 0.0f; } else if(activation == ActFunc_GELU) { const float az = 1.702f * z; if(!isfinite(az)) return(DPCC_STATUS_NUMERIC); const float denominator = 1.0f + exp(clamp(-az, -20.0f, 20.0f)); const float sigmoid = 1.0f / denominator; *value = z / denominator; *derivative = sigmoid; if(fabs(az) < 20.0f) { const float product = az * sigmoid; const float correction = product * (1.0f - sigmoid); if(!isfinite(product) || !isfinite(correction)) return(DPCC_STATUS_NUMERIC); *derivative += correction; } } else return(DPCC_STATUS_INVALID); return(isfinite(*value) && isfinite(*derivative) ? 0 : DPCC_STATUS_NUMERIC); } //+------------------------------------------------------------------+ //| DPCCActivationForward keeps preactivation separate from output | //+------------------------------------------------------------------+ __kernel void DPCCActivationForward(__global const float *preactivation, __global float *output, __global float *flags, int activation, uint count) { const size_t index = get_global_id(0); if(index >= count) return; float value = 0.0f; float derivative = 0.0f; const int status = DPCCOperatorActivation(preactivation[index], activation, &value, &derivative); flags[index] = (float)status; output[index] = status == 0 ? value : 0.0f; } //+------------------------------------------------------------------+ //| DPCCActivationGradient computes the exact unclipped chain | //+------------------------------------------------------------------+ __kernel void DPCCActivationGradient(__global const float *preactivation, __global const float *upstream, __global float *input_gradient, __global float *flags, int activation, uint count) { const size_t index = get_global_id(0); if(index >= count) return; float value = 0.0f; float derivative = 0.0f; int status = DPCCOperatorActivation(preactivation[index], activation, &value, &derivative); const float gradient = upstream[index] * derivative; if(!isfinite(upstream[index]) || !isfinite(gradient)) status = DPCC_STATUS_NUMERIC; flags[index] = (float)status; input_gradient[index] = status == 0 ? gradient : 0.0f; } //+------------------------------------------------------------------+ //| DPCCSharedAffineInputGradient excludes the last bias coordinate | //+------------------------------------------------------------------+ __kernel void DPCCSharedAffineInputGradient(__global const float *weights, __global const float *upstream, __global float *input_gradient, __global float *flags, uint tokens, uint input_width, uint output_width) { const size_t i = get_global_id(0); const size_t t = get_global_id(1); if(i >= input_width || t >= tokens) return; const size_t index = t * input_width + i; int status = output_width == 0 ? DPCC_STATUS_INVALID : 0; float sum = 0.0f; for(uint o = 0; o < output_width && status == 0; o++) if(!DPCCOperatorProductSum(weights[(size_t)o * (input_width + 1) + i], upstream[t * output_width + o], &sum)) status = DPCC_STATUS_NUMERIC; flags[index] = (float)status; input_gradient[index] = status == 0 ? sum : 0.0f; } //+------------------------------------------------------------------+ //| DPCCSharedAffineForwardPreflight checks native ordered reduction | //+------------------------------------------------------------------+ __kernel void DPCCSharedAffineForwardPreflight(__global const float *weights, __global const float *input, __global float *flags, uint tokens, uint input_width, uint output_width) { const size_t o = get_global_id(0); const size_t t = get_global_id(1); if(o >= output_width || t >= tokens) return; int status = input_width == 0 ? DPCC_STATUS_INVALID : 0; float sum = 0.0f; for(uint i = 0; i < input_width && status == 0; i++) { const float x = input[t * input_width + i]; const float w = weights[o * ((size_t)input_width + 1) + i]; if(!isfinite(x) || !isfinite(w)) status = DPCC_STATUS_NUMERIC; else if(x != 0.0f && !DPCCOperatorProductSum(x, w, &sum)) status = DPCC_STATUS_NUMERIC; } if(status == 0 && !DPCCOperatorProductSum(weights[o * ((size_t)input_width + 1) + input_width], 1.0f, &sum)) status = DPCC_STATUS_NUMERIC; flags[t * output_width + o] = (float)status; } //+------------------------------------------------------------------+ //| DPCCSharedAffineAdamPreflight validates before any native write | //+------------------------------------------------------------------+ __kernel void DPCCSharedAffineAdamPreflight(__global const float *weights, __global const float *gradient, __global const float *input, __global const float *momentum1, __global const float *momentum2, __global float *flags, uint tokens, uint input_width, uint output_width, float learning_rate, float beta1, float beta2) { const size_t index = get_global_id(0); const size_t stride = (size_t)input_width + 1; if(index >= stride * output_width) return; if((tokens & 0x80000000U) != 0) { const uint frame_mode = (tokens >> 29) & 0x3U; const uint frame_tokens = tokens & 0x1FFFFFFFU; int frame_status = frame_tokens == 0 || input_width == 0 || output_width == 0 || frame_mode == 0 || frame_mode > 3 || (frame_mode != 1 && frame_tokens != 1) ? DPCC_STATUS_INVALID : 0; if(!isfinite(learning_rate) || learning_rate <= 0.0f) frame_status = DPCC_STATUS_NUMERIC; const size_t o = index / stride; const size_t i = index % stride; if(frame_mode == 1) { float frame_sum = 0.0f; for(uint t = 0; t < frame_tokens && frame_status == 0; t++) if(!DPCCOperatorProductSum(gradient[(size_t)t * output_width + o], i == input_width ? 1.0f : input[(size_t)t * input_width + i], &frame_sum)) frame_status = DPCC_STATUS_NUMERIC; if(frame_status == 0) { const float old_sum = momentum1[index]; const float next_sum = old_sum + frame_sum; if(!isfinite(old_sum) || !isfinite(next_sum)) frame_status = DPCC_STATUS_NUMERIC; else ((__global float *)momentum1)[index] = next_sum; } } else if(frame_mode == 2 || frame_mode == 3) { const float sum = gradient[index]; const float old_w = weights[index]; const float old_m = momentum1[index]; const float old_v = momentum2[index]; if(!isfinite(sum) || !isfinite(old_w) || !isfinite(old_m) || !isfinite(old_v) || old_v < 0.0f) frame_status = DPCC_STATUS_NUMERIC; if(frame_status == 0) { const float g = clamp(sum, -MAX_GRAD, MAX_GRAD); const float g2 = g * g; const float mt = beta1 * old_m + (1.0f - beta1) * g; const float vt = beta2 * old_v + (1.0f - beta2) * g2; const float numerator = learning_rate * mt; const float denominator = sqrt(vt); const float delta = vt == 0.0f ? 0.0f : numerator / denominator; const float next_w = old_w + delta; if(!isfinite(g2) || !isfinite(mt) || !isfinite(vt) || vt < 0.0f || !isfinite(numerator) || !isfinite(denominator) || !isfinite(delta) || !isfinite(next_w)) frame_status = DPCC_STATUS_NUMERIC; else if(frame_mode == 3) { ((__global float *)weights)[index] = next_w; ((__global float *)momentum1)[index] = mt; ((__global float *)momentum2)[index] = vt; } } } flags[index] = (float)frame_status; return; } const uint mode = 0; int status = tokens == 0 || input_width == 0 || mode > 3 ? DPCC_STATUS_INVALID : 0; if(!isfinite(learning_rate) || learning_rate <= 0.0f) status = DPCC_STATUS_NUMERIC; const size_t o = index / stride; const size_t i = index % stride; float sum = 0.0f; if(mode == 0 || mode == 1) { for(uint t = 0; t < tokens && status == 0; t++) if(!DPCCOperatorProductSum(gradient[(size_t)t * output_width + o], i == input_width ? 1.0f : input[(size_t)t * input_width + i], &sum)) status = DPCC_STATUS_NUMERIC; if(mode == 0) { const float old_w = weights[index]; const float old_m = momentum1[index]; const float old_v = momentum2[index]; if(!isfinite(old_w) || !isfinite(old_m) || !isfinite(old_v) || old_v < 0.0f) status = DPCC_STATUS_NUMERIC; if(status == 0) { const float g = clamp(sum, -MAX_GRAD, MAX_GRAD); const float mt = beta1 * old_m + (1.0f - beta1) * g; const float vt = beta2 * old_v + (1.0f - beta2) * (g * g); const float numerator = learning_rate * mt; const float denominator = sqrt(vt); const float delta = vt == 0.0f ? 0.0f : numerator / denominator; const float next_w = old_w + delta; if(!isfinite(mt) || !isfinite(vt) || vt < 0.0f || !isfinite(numerator) || !isfinite(denominator) || !isfinite(delta) || !isfinite(next_w)) status = DPCC_STATUS_NUMERIC; } } else if(status == 0) { const float old_sum = momentum1[index]; const float next_sum = old_sum + sum; if(!isfinite(old_sum) || !isfinite(next_sum)) status = DPCC_STATUS_NUMERIC; else ((__global float *)momentum1)[index] = next_sum; } } else { sum = gradient[index]; const float old_w = weights[index]; const float old_m = momentum1[index]; const float old_v = momentum2[index]; if(tokens != 1 || !isfinite(sum) || !isfinite(old_w) || !isfinite(old_m) || !isfinite(old_v) || old_v < 0.0f) status = DPCC_STATUS_NUMERIC; if(status == 0) { const float g = clamp(sum, -MAX_GRAD, MAX_GRAD); const float g2 = g * g; const float mt = beta1 * old_m + (1.0f - beta1) * g; const float vt = beta2 * old_v + (1.0f - beta2) * g2; const float numerator = learning_rate * mt; const float denominator = sqrt(vt); const float delta = vt == 0.0f ? 0.0f : numerator / denominator; const float next_w = old_w + delta; if(!isfinite(g2) || !isfinite(mt) || !isfinite(vt) || vt < 0.0f || !isfinite(numerator) || !isfinite(denominator) || !isfinite(delta) || !isfinite(next_w)) status = DPCC_STATUS_NUMERIC; else if(mode == 3) { ((__global float *)weights)[index] = next_w; ((__global float *)momentum1)[index] = mt; ((__global float *)momentum2)[index] = vt; } } } flags[index] = (float)status; } //+------------------------------------------------------------------+ //| DPCCOperatorReduce publishes only complete finite status vectors | //+------------------------------------------------------------------+ __kernel void DPCCOperatorReduce(__global const float *flags, __global float *status, uint count) { if(get_global_id(0) != 0) return; float result = count == 0 ? (float)DPCC_STATUS_INVALID : 0.0f; for(uint i = 0; i < count; i++) { const float code = flags[i]; if(!isfinite(code) || (code != 0.0f && code != (float)DPCC_STATUS_INVALID && code != (float)DPCC_STATUS_NUMERIC)) result = (float)DPCC_STATUS_NUMERIC; else result = fmax(result, code); } status[0] = result; } //+------------------------------------------------------------------+ //| DPCCDecoderSnapshotCheck detects mutation of a live source tensor | //+------------------------------------------------------------------+ __kernel void DPCCDecoderSnapshotCheck(__global const float *sourceKVHD, __global const float *packedKHVD, __global float *flags, uint K, uint V, uint H, uint D) { const size_t index = get_global_id(0); const size_t total = (size_t)K * V * H * D; if(index >= total) return; if(K == 0 || V == 0 || H == 0 || D == 0) { flags[index] = (float)DPCC_STATUS_INVALID; return; } const size_t d = index % D; const size_t v = index / D % V; const size_t h = index / ((size_t)V * D) % H; const size_t k = index / ((size_t)V * H * D); const size_t z = (((k * V + v) * H + h) * D + d); const float source = sourceKVHD[z]; const float snapshot = packedKHVD[index]; if(!isfinite(source) || !isfinite(snapshot)) flags[index] = (float)DPCC_STATUS_NUMERIC; else flags[index] = as_uint(source) == as_uint(snapshot) ? 0.0f : (float)DPCC_STATUS_INVALID; } //+------------------------------------------------------------------+ //| DPCC exact denoiser layouts, attention derivatives and checks | //+------------------------------------------------------------------+ inline float DPCCAttentionNumber(float value, __private int *status) { if(!isfinite(value)) *status = DPCC_STATUS_NUMERIC; return(value); } //+------------------------------------------------------------------+ //| DPCCAttentionProduct checks operands before retaining raw math | //+------------------------------------------------------------------+ inline float DPCCAttentionProduct(float left, float right, __private int *status) { DPCCAttentionNumber(left, status); DPCCAttentionNumber(right, status); return(DPCCAttentionNumber(left * right, status)); } //+------------------------------------------------------------------+ //| DPCCAttentionSum preserves the raw binary reduction tree | //+------------------------------------------------------------------+ inline float DPCCAttentionSum(float value, __local float *scratch, __private int *status) { const uint lane = get_local_id(1); const uint width = get_local_size(1); scratch[lane] = DPCCAttentionNumber(value, status); barrier(CLK_LOCAL_MEM_FENCE); for(uint span = width / 2; span > 0; span /= 2) { if(lane < span) scratch[lane] = DPCCAttentionNumber(scratch[lane] + scratch[lane + span], status); barrier(CLK_LOCAL_MEM_FENCE); } const float result = DPCCAttentionNumber(scratch[0], status); barrier(CLK_LOCAL_MEM_FENCE); return(result); } //+------------------------------------------------------------------+ //| DPCCAttentionMaximum checks every raw reduction input | //+------------------------------------------------------------------+ inline float DPCCAttentionMaximum(float value, __local float *scratch, __private int *status) { const uint lane = get_local_id(1); const uint width = get_local_size(1); scratch[lane] = DPCCAttentionNumber(value, status); barrier(CLK_LOCAL_MEM_FENCE); for(uint span = width / 2; span > 0; span /= 2) { if(lane < span) scratch[lane] = fmax(scratch[lane], scratch[lane + span]); barrier(CLK_LOCAL_MEM_FENCE); } const float result = DPCCAttentionNumber(scratch[0], status); barrier(CLK_LOCAL_MEM_FENCE); return(result); } //+------------------------------------------------------------------+ //| DPCCAttentionNumber4 checks all four raw vector coordinates | //+------------------------------------------------------------------+ inline float4 DPCCAttentionNumber4(float4 value, __private int *status) { if(!all(isfinite(value))) *status = DPCC_STATUS_NUMERIC; return(value); } //+------------------------------------------------------------------+ //| DPCCAttentionSum4 retains every raw vector tree partial | //+------------------------------------------------------------------+ inline float4 DPCCAttentionSum4(float4 value, __local float4 *scratch, __private int *status) { const uint lane = get_local_id(1); const uint width = get_local_size(1); scratch[lane] = DPCCAttentionNumber4(value, status); barrier(CLK_LOCAL_MEM_FENCE); for(uint span = width / 2; span > 0; span /= 2) { if(lane < span) scratch[lane] = DPCCAttentionNumber4(scratch[lane] + scratch[lane + span], status); barrier(CLK_LOCAL_MEM_FENCE); } const float4 result = DPCCAttentionNumber4(scratch[0], status); barrier(CLK_LOCAL_MEM_FENCE); return(result); } //+------------------------------------------------------------------+ //| DPCCAttentionGradRow shares the exact derivative with preflight | //+------------------------------------------------------------------+ inline void DPCCAttentionGradRow(__global const float *query, __global const float *key_value, __global const float *logsumexp, __global const float *output, __global const float *output_gr, int dimension, int total_q, int total_kv, __local float *scratch, __private float *grad_q, __private float *grad_k, __private float *grad_v, __private int *status) { const size_t id = get_global_id(0); const size_t d = get_local_id(1); const size_t head = get_global_id(2); const size_t heads = get_global_size(2); const float scale = 1.0f / sqrt((float)dimension); *grad_q = 0.0f; *grad_k = 0.0f; *grad_v = 0.0f; //--- The query and key branches are uniform within each workgroup. if(id < (size_t)total_q) { const size_t iq = (id * heads + head) * dimension + d; const float lse = DPCCAttentionNumber(logsumexp[id * heads + head], status); const float q = DPCCAttentionNumber(query[iq], status); const float g = DPCCAttentionNumber(output_gr[iq], status); const float out = DPCCAttentionNumber(output[iq], status); const float row_dot = DPCCAttentionSum(DPCCAttentionProduct(g, out, status), scratch, status); for(int k = 0; k < total_kv; k++) { const size_t ik = ((size_t)k * 2 * heads + head) * dimension + d; const size_t iv = ik + heads * dimension; const float key = DPCCAttentionNumber(key_value[ik], status); const float value = DPCCAttentionNumber(key_value[iv], status); const float dot_qk = DPCCAttentionSum(DPCCAttentionProduct(q, key, status), scratch, status); const float score = DPCCAttentionProduct(dot_qk, scale, status); const float exponent = DPCCAttentionNumber(score - lse, status); const float probability = DPCCAttentionNumber(exp(exponent), status); const float dp = DPCCAttentionSum(DPCCAttentionProduct(g, value, status), scratch, status); const float ds = DPCCAttentionProduct(probability, DPCCAttentionNumber(dp - row_dot, status), status); const float term = DPCCAttentionProduct(DPCCAttentionProduct(ds, key, status), scale, status); *grad_q = DPCCAttentionNumber(*grad_q + term, status); } } if(id < (size_t)total_kv) { const size_t ik = (id * 2 * heads + head) * dimension + d; const size_t iv = ik + heads * dimension; const float key = DPCCAttentionNumber(key_value[ik], status); const float value = DPCCAttentionNumber(key_value[iv], status); for(int q_id = 0; q_id < total_q; q_id++) { const size_t iq = ((size_t)q_id * heads + head) * dimension + d; const float lse = DPCCAttentionNumber(logsumexp[(size_t)q_id * heads + head], status); const float q = DPCCAttentionNumber(query[iq], status); const float g = DPCCAttentionNumber(output_gr[iq], status); const float out = DPCCAttentionNumber(output[iq], status); const float row_dot = DPCCAttentionSum(DPCCAttentionProduct(g, out, status), scratch, status); const float dot_qk = DPCCAttentionSum(DPCCAttentionProduct(q, key, status), scratch, status); const float score = DPCCAttentionProduct(dot_qk, scale, status); const float exponent = DPCCAttentionNumber(score - lse, status); const float probability = DPCCAttentionNumber(exp(exponent), status); const float dp = DPCCAttentionSum(DPCCAttentionProduct(g, value, status), scratch, status); const float ds = DPCCAttentionProduct(probability, DPCCAttentionNumber(dp - row_dot, status), status); const float term_k = DPCCAttentionProduct(DPCCAttentionProduct(ds, q, status), scale, status); const float term_v = DPCCAttentionProduct(probability, g, status); *grad_k = DPCCAttentionNumber(*grad_k + term_k, status); *grad_v = DPCCAttentionNumber(*grad_v + term_v, status); } } } //+------------------------------------------------------------------+ //| DPCC_MHFlashAttentionGradient retains exact hidden gradients | //+------------------------------------------------------------------+ __kernel void DPCC_MHFlashAttentionGradient(__global const float *query, __global float *query_gr, __global const float *key_value, __global float *key_value_gr, __global const float *logsumexp, __global const float *output, __global const float *output_gr, int dimension, int total_q, int total_kv, int mask_future) { if(dimension < 4 || dimension > 64 || (dimension & (dimension - 1)) != 0 || total_q <= 0 || total_kv <= 0 || mask_future != 0 || get_local_size(0) != 1 || get_local_size(1) != (size_t)dimension || get_local_size(2) != 1) return; __local float scratch[64]; int status = 0; float grad_q = 0.0f; float grad_k = 0.0f; float grad_v = 0.0f; DPCCAttentionGradRow(query, key_value, logsumexp, output, output_gr, dimension, total_q, total_kv, scratch, &grad_q, &grad_k, &grad_v, &status); const size_t id = get_global_id(0); const size_t d = get_local_id(1); const size_t head = get_global_id(2); const size_t heads = get_global_size(2); if(id < (size_t)total_q) query_gr[(id * heads + head) * dimension + d] = grad_q; if(id < (size_t)total_kv) { const size_t ik = (id * 2 * heads + head) * dimension + d; key_value_gr[ik] = grad_k; key_value_gr[ik + heads * dimension] = grad_v; } } //+------------------------------------------------------------------+ //| DPCCAttentionPreflight checks raw forward and adjoint | //+------------------------------------------------------------------+ __kernel void DPCCAttentionPreflight(__global const float *query, __global const float *key_value, __global const float *logsumexp, __global const float *output, __global const float *output_gr, __global float *flags, int dimension, int total_q, int total_kv, int mask_future, uint mode) { const size_t id = get_global_id(0); const size_t lane = get_local_id(1); const size_t head = get_global_id(2); const size_t heads = get_global_size(2); const size_t index = (id * heads + head) * get_global_size(1) + get_global_id(1); if(dimension < 4 || dimension > 64 || (dimension & (dimension - 1)) != 0 || total_q <= 0 || total_kv <= 0 || mask_future != 0 || mode > 1 || get_local_size(0) != 1 || get_global_size(1) != (size_t)dimension || get_local_size(1) != (size_t)dimension || get_local_size(2) != 1 || get_global_size(0) != (size_t)max(total_q, total_kv)) { flags[index] = (float)DPCC_STATUS_INVALID; return; } __local float scratch[64]; __local float4 scratch4[64]; int status = 0; if(mode == 1) { float grad_q = 0.0f; float grad_k = 0.0f; float grad_v = 0.0f; DPCCAttentionGradRow(query, key_value, logsumexp, output, output_gr, dimension, total_q, total_kv, scratch, &grad_q, &grad_k, &grad_v, &status); } else if(id < (size_t)total_q) { const size_t iq = (id * heads + head) * dimension; float previous_max = MIN_VALUE; float sumexp = 0.0f; float result = 0.0f; for(int tile = 0; tile < total_kv; tile += dimension) { const size_t key_id = (size_t)tile + lane; const size_t ik = (key_id * 2 * heads + head) * dimension; const size_t iv = ik + heads * dimension; float score = MIN_VALUE; if(key_id < (size_t)total_kv) { score = 0.0f; for(int d = 0; d < dimension; d += 4) { const float4 q = DPCCAttentionNumber4(vload4(0, query + iq + d), &status); const float4 k = DPCCAttentionNumber4(vload4(0, key_value + ik + d), &status); DPCCAttentionNumber4(q * k, &status); const float dot4 = DPCCAttentionNumber(dot(q, k), &status); score = DPCCAttentionNumber(score + dot4, &status); } score = DPCCAttentionNumber(score / sqrt((float)dimension), &status); if(score <= MIN_VALUE) status = DPCC_STATUS_NUMERIC; } const float maximum = DPCCAttentionNumber( fmax(previous_max, DPCCAttentionMaximum(score, scratch, &status)), &status); const float probability = score > MIN_VALUE ? DPCCAttentionNumber(exp(DPCCAttentionNumber(score - maximum, &status)), &status) : 0.0f; const float tile_sum = DPCCAttentionSum(probability, scratch, &status); const float rescale = sumexp == 0.0f ? 0.0f : DPCCAttentionNumber( exp(DPCCAttentionNumber(previous_max - maximum, &status)), &status); sumexp = sumexp == 0.0f ? tile_sum : DPCCAttentionNumber(DPCCAttentionProduct(rescale, sumexp, &status) + tile_sum, &status); for(int d = 0; d < dimension; d += 4) { float4 weighted = (float4)0.0f; if(key_id < (size_t)total_kv) { const float4 value = DPCCAttentionNumber4(vload4(0, key_value + iv + d), &status); weighted = DPCCAttentionNumber4(value * probability, &status); } const float4 tile_value = DPCCAttentionSum4(weighted, scratch4, &status); const int component = (int)lane - d; if(component >= 0 && component < 4) { const float value = vget4(tile_value, component); result = result == 0.0f ? value : DPCCAttentionNumber(DPCCAttentionProduct(rescale, result, &status) + value, &status); } } previous_max = maximum; } if(sumexp <= 0.0f || !isfinite(sumexp)) status = DPCC_STATUS_NUMERIC; DPCCAttentionNumber(result / sumexp, &status); DPCCAttentionNumber(previous_max + log(sumexp), &status); } flags[index] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDenoiserSelfLayout maps isolated plan rows to flash heads | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserSelfLayout(__global const float *source0, __global const float *source1, __global const float *source2, __global float *destination0, __global float *destination1, __global float *flags, uint L, uint N, uint heads, uint dimension, uint mode) { const size_t index = get_global_id(0); const size_t width = (size_t)heads * dimension; const size_t total = (size_t)L * N * width; if(index >= total) return; if(L == 0 || N == 0 || heads == 0 || dimension == 0 || mode > 3) { flags[index] = (float)DPCC_STATUS_INVALID; return; } const size_t d = index % dimension; const size_t a = index / dimension % heads; const size_t n = index / width % N; const size_t l = index / (width * N); const size_t group = (size_t)L * heads; const size_t physical = (n * group + l * heads + a) * dimension + d; const size_t physical_k = (n * 2 * group + l * heads + a) * dimension + d; const size_t physical_v = physical_k + group * dimension; int status = 0; if(mode == 0) { destination0[physical] = DPCCAttentionNumber(source0[index], &status); destination1[physical_k] = DPCCAttentionNumber(source1[index], &status); destination1[physical_v] = DPCCAttentionNumber(source2[index], &status); } else if(mode == 1) destination0[index] = DPCCAttentionNumber(source0[physical], &status); else if(mode == 3) destination0[physical] = DPCCAttentionNumber(source0[index], &status); else { destination0[index] = DPCCAttentionNumber(source0[physical_k], &status); destination1[index] = DPCCAttentionNumber(source0[physical_v], &status); } flags[index] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDenoiserCrossLayout preserves scenario concatenation | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserCrossLayout(__global const float *source0, __global const float *source1, __global float *destination0, __global float *destination1, __global float *flags, uint L, uint R, uint H, uint K, uint heads, uint dimension, uint mode) { const size_t index = get_global_id(0); const size_t width = (size_t)heads * dimension; const size_t rows = (size_t)L * R; const size_t total = mode == 0 ? rows * K * width : (mode == 2 ? rows * width : (size_t)K * H * width); if(index >= total) return; if(L == 0 || R == 0 || H == 0 || K == 0 || heads == 0 || dimension == 0 || mode > 3) { flags[index] = (float)DPCC_STATUS_INVALID; return; } const size_t d = index % dimension; const size_t a = index / dimension % heads; const size_t group = (size_t)K * heads; int status = 0; if(mode == 0) { const size_t q = index / ((size_t)K * width); destination0[index] = DPCCAttentionNumber(source0[q * width + a * dimension + d], &status); } else if(mode == 2) { const size_t q = index / width; float sum = 0.0f; for(uint k = 0; k < K; k++) { const float value = DPCCAttentionNumber(source0[(q * group + (size_t)k * heads + a) * dimension + d], &status); sum = DPCCAttentionNumber(sum + value, &status); } destination0[index] = sum; } else { const size_t h = index / width % H; const size_t k = index / (width * H); const size_t physical_k = (h * 2 * group + k * heads + a) * dimension + d; const size_t physical_v = physical_k + group * dimension; if(mode == 1) { destination0[physical_k] = DPCCAttentionNumber(source0[index], &status); destination0[physical_v] = DPCCAttentionNumber(source1[index], &status); } else { destination0[index] = DPCCAttentionNumber(source0[physical_k], &status); destination1[index] = DPCCAttentionNumber(source0[physical_v], &status); } } flags[index] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDenoiserPairMap pads or crops each candidate independently | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserPairMap(__global const float *source, __global float *destination, __global float *flags, uint L, uint H, uint W, uint mode) { const size_t index = get_global_id(0); const size_t padded = ((size_t)H + 1) / 2 * 2 * W; const size_t plain = (size_t)H * W; const size_t stride = mode == 0 || mode == 2 ? padded : plain; const size_t total = (size_t)L * stride; if(index >= total) return; if(L == 0 || H == 0 || W == 0 || mode > 3) { flags[index] = (float)DPCC_STATUS_INVALID; return; } const size_t l = index / stride; const size_t coordinate = index % stride; int status = 0; float value = 0.0f; if(mode == 0 || mode == 2) value = coordinate < plain ? DPCCAttentionNumber(source[l * plain + coordinate], &status) : 0.0f; else value = DPCCAttentionNumber(source[l * padded + coordinate], &status); destination[index] = value; flags[index] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDenoiserBroadcastGradient sums shared row contributions | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserBroadcastGradient(__global const float *source, __global float *destination, __global float *flags, uint rows, uint width) { const size_t coordinate = get_global_id(0); if(coordinate >= width) return; int status = rows == 0 || width == 0 ? DPCC_STATUS_INVALID : 0; float sum = 0.0f; for(uint row = 0; row < rows; row++) { const float value = DPCCAttentionNumber(source[(size_t)row * width + coordinate], &status); sum = DPCCAttentionNumber(sum + value, &status); } destination[coordinate] = sum; flags[coordinate] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDecoderRhoLoss computes one checked scenario partial | //+------------------------------------------------------------------+ __kernel void DPCCDecoderRhoLoss(__global const float *y, __global const float *target, __global const float *rho, __global float *gradient_scratch, __global float *partials, __global float *flags, uint K, uint H, uint prepare_gradient) { const size_t k = get_global_id(0); if(k >= K) return; if(K == 0 || H == 0 || prepare_gradient > 1) { partials[k] = 0.0f; flags[k] = (float)DPCC_STATUS_INVALID; return; } int status = 0; const float mass = rho[k]; if(!isfinite(mass)) status = DPCC_STATUS_NUMERIC; else if(mass < 0.0f) status = DPCC_STATUS_INVALID; const size_t width = (size_t)H * 4; const float denominator = (float)width; float sum = 0.0f; for(size_t coordinate = 0; coordinate < width; coordinate++) { const size_t n = k * width + coordinate; const float prediction = y[n]; const float expected = target[coordinate]; const float error = prediction - expected; const float squared = error * error; const float mean_squared = squared / denominator; const float weighted = mass * mean_squared; const float next = sum + weighted; if(!isfinite(prediction) || !isfinite(expected) || !isfinite(error) || !isfinite(squared) || !isfinite(mean_squared) || !isfinite(weighted) || !isfinite(next)) status = DPCC_STATUS_NUMERIC; sum = next; if(prepare_gradient) { const float factor = -2.0f * mass; const float product = factor * error; const float gradient = product / denominator; if(!isfinite(factor) || !isfinite(product) || !isfinite(gradient)) status = DPCC_STATUS_NUMERIC; gradient_scratch[n] = gradient; } } partials[k] = sum; flags[k] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDecoderRhoLossReduce admits complete rho and loss numerics | //+------------------------------------------------------------------+ __kernel void DPCCDecoderRhoLossReduce(__global const float *rho, __global const float *partials, __global const float *flags, __global float *control, uint K, float tolerance) { if(get_global_id(0) != 0) return; int status = K == 0 || !isfinite(tolerance) || tolerance < 0.0f || tolerance >= 1.0f ? DPCC_STATUS_INVALID : 0; float mass = 0.0f; float loss = 0.0f; for(uint k = 0; k < K; k++) { const float flag = flags[k]; const float r = rho[k]; const float partial = partials[k]; if(!isfinite(flag) || flag == (float)DPCC_STATUS_NUMERIC || !isfinite(r) || !isfinite(partial)) status = DPCC_STATUS_NUMERIC; else if((flag != 0.0f || r < 0.0f || partial < 0.0f) && status != DPCC_STATUS_NUMERIC) status = DPCC_STATUS_INVALID; const float next_mass = mass + r; const float next_loss = loss + partial; if(!isfinite(next_mass) || !isfinite(next_loss)) status = DPCC_STATUS_NUMERIC; mass = next_mass; loss = next_loss; } if(isfinite(mass) && fabs(mass - 1.0f) > tolerance && status != DPCC_STATUS_NUMERIC) status = DPCC_STATUS_INVALID; control[0] = (float)status; control[1] = loss; control[2] = mass; } // Work-only composite operators. No registration or public primitive class. // Host validates physical buffer sizes, disjoint ownership, C/R v2 layout, // integer dimensions and total byte bounds before dispatch. Flags per output // are reduced by existing DPCCOperatorReduce before any dependent launch. // Existing DPCCActivationForward(None) snapshots/prechecks the complete C. // Prepare the existing InitOwned(K*H*4), ADAM RevIN options (9 per element). // Its legacy kernel substitutes NaN, so preflight BOTH variance and restore. __kernel void DPCCCompositeNormOptions(__global const float *context, __global const float *normalized, __global float *options, __global float *flags, uint norm_offset, uint count) { const size_t n = get_global_id(0); if(n >= count) return; const size_t s = norm_offset + (n % 4) * 4; const float mean = context[s], std = context[s + 1]; const float gain = context[s + 2], bias = context[s + 3]; float status = 0.0f; if(!isfinite(mean) || !isfinite(std) || !isfinite(gain) || !isfinite(bias) || !isfinite(normalized[n]) || std <= 0.0f || gain != 1.0f || bias != 0.0f) status = 3.0f; const float variance = std * std; const float centered = normalized[n] - bias; const float scaled = sqrt(variance) * centered; const float restored = scaled + mean; if(status == 0.0f && (!isfinite(variance) || variance <= 0.0f || !isfinite(centered) || !isfinite(scaled) || !isfinite(restored))) status = 5.0f; for(uint i = 0; i < 9; i++) options[n * 9 + i] = 0.0f; if(status == 0.0f) { options[n * 9] = mean; options[n * 9 + 1] = variance; options[n * 9 + 3] = 1.0f; options[n * 9 + 4] = bias; } flags[n] = status; } // Convert existing RevIN native output into X at the start of exact R. // R tail is contiguous C[Account..Calendar]; Active[M] starts all-one. // This packs values, not a second financial/event engine. The existing // rollout validator still checks the complete physical/event contract. __kernel void DPCCCompositeQuoteR(__global const float *context, __global const float *native, __global float *rollout_context, __global float *flags, uint K, uint H, uint M, uint account_offset, uint execution_offset, uint active_offset, uint total, uint price_basis, uint quote_adapter, float max_quote_age, float max_last_age) { const size_t n = get_global_id(0); if(n >= total) return; const size_t prices = (size_t)K * H * 4; __global const float *account = context + account_offset; __global const float *instrument = account + 24; float status = 0.0f, value = 0.0f; if(price_basis > 1 || quote_adapter != price_basis || instrument[31] != (float)price_basis || instrument[32] != (float)quote_adapter || !isfinite(max_quote_age) || max_quote_age < 0.0f || !isfinite(max_last_age) || max_last_age < 0.0f || !isfinite(instrument[36]) || instrument[36] < 0.0f || instrument[36] > max_quote_age || !isfinite(instrument[37]) || instrument[37] < 0.0f || (price_basis == 1 && instrument[37] > max_last_age) || !isfinite(account[15]) || !isfinite(account[16]) || account[16] < account[15] || (price_basis == 1 && !isfinite(instrument[33]))) status = 3.0f; if(n < prices) { const size_t row = n / 4, ch = n % 4, h = row % H; const float mark = price_basis == 0 ? account[15] : instrument[33]; // Preserve the causal host snapshot's frozen float delta. Its agreement // with float Bid-Last uses the existing rollout price_atol convention. const float delta = context[execution_offset + 20]; const float price_atol = context[account_offset + 24 + 48 + 15]; float o = h == 0 ? mark : native[row * 4]; const float hi = native[row * 4 + 1], lo = native[row * 4 + 2]; const float close = native[row * 4 + 3]; for(uint a = 0; a < 4; a++) if(!isfinite(native[row * 4 + a])) status = 3.0f; if(status == 0.0f && (!isfinite(delta) || !isfinite(price_atol) || price_atol < 0.0f)) status = 3.0f; const float difference = account[15] - instrument[33] - delta; if(status == 0.0f && price_basis == 1 && !isfinite(difference)) status = 5.0f; if(status == 0.0f && ((price_basis == 0 && delta != 0.0f) || (price_basis == 1 && fabs(difference) > price_atol))) status = 3.0f; if(status == 0.0f && (hi < o || hi < close || lo > o || lo > close || lo > hi)) status = 3.0f; if(status == 0.0f && context[execution_offset + row * 24 + 20] != delta) status = 3.0f; value = (ch == 0 ? o : native[n]) + delta; // Algebraic mark+delta can lose a bit; first Bid.Open is the snapshot. if(h == 0 && ch == 0) value = account[15]; if(status == 0.0f && !isfinite(value)) status = 5.0f; const float bid_o = h == 0 ? account[15] : o + delta; const float bid_hi = hi + delta, bid_lo = lo + delta, bid_close = close + delta; if(status == 0.0f && (!isfinite(bid_o) || !isfinite(bid_hi) || !isfinite(bid_lo) || !isfinite(bid_close))) status = 5.0f; if(status == 0.0f && (bid_hi < bid_o || bid_hi < bid_close || bid_lo > bid_o || bid_lo > bid_close || bid_lo > bid_hi)) status = 3.0f; } else if(n < active_offset) value = context[account_offset + n - prices]; else value = 1.0f; if(!isfinite(value) && status == 0.0f) status = 3.0f; rollout_context[n] = status == 0.0f ? value : 0.0f; flags[n] = status; } // Exact SP-09 Tokens[K,H,V*(D+1)+1] followed by CTX-03 Global45. // Account13 is an external exact input; it is never rebuilt from Account24. // Step16 is updated separately, leaving market/global bytes immutable. __kernel void DPCCCompositeCondition(__global const float *context, __global const float *account_features, __global float *condition, __global float *flags, uint K, uint V, uint H, uint D, uint u_offset, uint pi_offset, uint norm_offset, uint account_offset, uint limits_offset, float volume_scale, float price_scale, float money_scale, uint price_basis) { const size_t n = get_global_id(0); const size_t F = (size_t)V * (D + 1) + 1; const size_t tokens = (size_t)K * H * F; if(n >= tokens + 45) return; float status = 0.0f, value = 0.0f; if(n < tokens) { const size_t row = n / F, f = n % F, k = row / H, h = row % H; if(f == F - 1) { const float p = context[pi_offset + k]; if(!isfinite(p) || p < 0.0f) status = 3.0f; value = log(fmax(p, 1.0e-6f)); } else { if(f < (size_t)V * D) { const size_t v = f / D, d = f % D; value = context[((k * V + v) * H + h) * D + d]; } else { const size_t v = f - (size_t)V * D; const float u = context[u_offset + (k * V + v) * H + h]; if(!isfinite(u) || u < 0.0f) status = 3.0f; value = log(fmax(u, 1.0e-6f)); } } } else { const size_t i = n - tokens; __global const float *a = context + account_offset; __global const float *ins = a + 24; __global const float *lim = context + limits_offset; const float mask_f = lim[20]; if(!isfinite(volume_scale) || volume_scale <= 0.0f || !isfinite(price_scale) || price_scale <= 0.0f || !isfinite(money_scale) || money_scale <= 0.0f || !isfinite(mask_f) || mask_f < 0.0f || mask_f > 511.0f || floor(mask_f) != mask_f || lim[21] != money_scale || lim[22] != 2.0f || lim[23] != 0.0f || !isfinite(ins[0]) || ins[0] <= 0.0f || !isfinite(ins[1]) || ins[1] <= 0.0f || !isfinite(ins[5]) || ins[5] <= 0.0f || price_basis > 1 || ins[31] != (float)price_basis || ins[32] != (float)price_basis || !isfinite(a[15]) || !isfinite(a[16]) || a[16] < a[15] || !isfinite(ins[29]) || ins[29] < 0.0f || ins[29] > 4.0f || floor(ins[29]) != ins[29]) status = 3.0f; for(uint t = 0; t < 24; t++) if(!isfinite(lim[t])) status = 3.0f; const uint mask = status == 0.0f ? (uint)mask_f : 0; if(i < 13) value = account_features[i]; else if(i < 24) { const uint li = (uint)i - 13; const uint bit = li < 7 ? li : li < 9 ? 7 : 8; value = (mask & (1U << bit)) ? lim[li] : 0.0f; if(li == 0) value /= volume_scale; else if(li == 1 || (li >= 3 && li <= 6)) value /= money_scale; else if(li >= 7) value = value * ins[0] / price_scale; } else if(i == 24 || i == 25) value = lim[i - 13] / money_scale; else if(i == 26) value = lim[13] * ins[0] / price_scale; else if(i == 27) value = lim[14] / volume_scale; else if(i == 28) value = lim[15] / price_scale; else if(i == 29) value = lim[16] / money_scale; else if(i == 30 || i == 31) value = lim[i - 13]; else if(i == 32) value = lim[19] / price_scale; else if(i == 33) value = mask_f / 511.0f; else if(i == 34) value = lim[21] / money_scale; else if(i == 35) value = lim[22] / 2.0f; else if(i == 36) value = 0.0f; else if(i == 37) value = (a[16] - a[15]) / price_scale; else if(i == 38) value = ins[1] / price_scale; else if(i == 39) value = ins[3] / volume_scale; else if(i == 40) value = ins[5] / volume_scale; else if(i == 41) value = ins[15] * ins[14] * volume_scale / money_scale; else if(i == 42) value = ins[21] * volume_scale / money_scale; else if(i == 43) { const float std = context[norm_offset + 13]; if(!isfinite(std) || std <= 0.0f) status = 3.0f; value = ((price_basis == 0 ? a[15] : ins[33]) - context[norm_offset + 12]) / std; } else value = ins[29] / 4.0f; } if(!isfinite(value) && status == 0.0f) status = 5.0f; condition[n] = status == 0.0f ? value : 0.0f; flags[n] = status; } // Step16 uses the existing COpenCL::BufferWrite partial-write API directly; // no additional kernel, flags buffer, reduction or Step device holder. // Work-only kernels356..359. Composite owner validates exact physical sizes, // byte ranges, context/aliasing and snapshot before dispatch. All results remain // private until existing DPCCOperatorReduce accepts every output flag. __kernel void DPCCCompositeInitialNoise(__global const float *context, __global float *rollout_context, __global float *normalized, __global float *candidate_status, __global float *final_validity, __global float *flags, uint M, uint H, uint J, uint noise_offset, uint active_offset) { const size_t n = get_global_id(0); const size_t row_size = (size_t)H * 6; if(n >= (size_t)M * row_size) return; const size_t m = n / row_size, coordinate = n % row_size; const size_t source = noise_offset + (m * ((size_t)J + 1) + J) * row_size + coordinate; const float value = context[source]; flags[n] = isfinite(value) ? 0.0f : 3.0f; normalized[n] = isfinite(value) ? value : 0.0f; if(coordinate == 0) { rollout_context[active_offset + m] = 1.0f; candidate_status[m] = 0.0f; // Final financial validation is still outstanding. final_validity[m] = 0.0f; } } __kernel void DPCCCompositeReverse(__global const float *normalized, __global const float *epsilon, __global const float *context, __global const float *rollout_context, __global float *proposal, __global float *flags, uint M, uint H, uint J, uint j, uint noise_offset, uint active_offset, float beta, float alpha_bar, float sigma) { const size_t n = get_global_id(0), row_size = (size_t)H * 6; if(n >= (size_t)M * row_size) return; const size_t m = n / row_size, coordinate = n % row_size; const float active = rollout_context[active_offset + m], current = normalized[n]; float status = 0.0f, result = 0.0f, projected_value = 0.0f; if(j == 0 || j > J || !isfinite(beta) || !isfinite(alpha_bar) || !isfinite(sigma) || beta <= 0.0f || beta >= 1.0f || alpha_bar <= 0.0f || alpha_bar >= 1.0f || sigma < 0.0f || (j == 1 && sigma != 0.0f) || (active != 0.0f && active != 1.0f) || !isfinite(current)) status = 3.0f; if(status == 0.0f && active == 0.0f) result = current; // No epsilon/noise reads or row compaction for inactive m. if(status == 0.0f && active == 1.0f) { const float eps = epsilon[n]; if(!isfinite(eps)) status = 3.0f; if(status == 0.0f) { const float root_variance = sqrt(1.0f - alpha_bar), root_alpha = sqrt(1.0f - beta); const float product = beta * eps, correction = product / root_variance; const float numerator = current - correction, mu = numerator / root_alpha; if(!isfinite(root_variance) || !isfinite(root_alpha) || root_variance <= 0.0f || root_alpha <= 0.0f || !isfinite(product) || !isfinite(correction) || !isfinite(numerator) || !isfinite(mu)) status = 5.0f; else { result = mu; // DIF-02 deterministic j1: Noise[m,0] is never dereferenced. if(j > 1) { const size_t source = noise_offset + (m * ((size_t)J + 1) + j - 1) * row_size + coordinate; const float noise = context[source]; if(!isfinite(noise)) status = 3.0f; else { const float perturbation = sigma * noise; result = mu + perturbation; if(!isfinite(perturbation) || !isfinite(result)) status = 5.0f; } } } } } proposal[n] = status == 0.0f ? result : 0.0f; flags[n] = status; } // DATA-04 uses the same bounded action space in every projection mode. // direction0 maps denoiser values to probabilities; direction1 maps probabilities back to denoiser values. __kernel void DPCCCompositeActionAffine(__global const float *source, __global float *destination, __global const float *rollout_context, __global float *flags, __global float *probability_stack, __global float *projected_stack, uint M, uint H, uint active_offset, uint direction, uint sigmoid_output, uint probability_offset, float b0, float b1, float b2, float b3, float b4, float b5, float s0, float s1, float s2, float s3, float s4, float s5) { const size_t n = get_global_id(0), row_size = (size_t)H * 6; if(n >= (size_t)M * row_size) return; const float biases[6] = {b0,b1,b2,b3,b4,b5}, scales[6] = {s0,s1,s2,s3,s4,s5}; const float active = rollout_context[active_offset + n / row_size], value = source[n]; const float bias = biases[n % 6], scale = scales[n % 6]; float status = 0.0f, result = 0.0f, projected_value = 0.0f; if(direction > 1 || sigmoid_output != 1 || (active != 0.0f && active != 1.0f) || !isfinite(value) || !isfinite(bias) || !isfinite(scale) || scale <= 0.0f) status = 3.0f; if(status == 0.0f && active == 1.0f) { if(direction == 0) { const float logit = bias + scale * value; if(!isfinite(logit)) status = 5.0f; else result = logit >= 0.0f ? 1.0f / (1.0f + exp(-logit)) : exp(logit) / (1.0f + exp(logit)); } else { projected_value = clamp(value, 0.0f, 1.0f); const float probability = clamp(projected_value, DPCC_ACTION_PROBABILITY_EPS, 1.0f - DPCC_ACTION_PROBABILITY_EPS); const float logit = log(probability / (1.0f - probability)); result = (logit - bias) / scale; if(!isfinite(logit) || !isfinite(result)) status = 5.0f; } if(!isfinite(result)) status = 5.0f; } if(probability_offset != 0xFFFFFFFFu) { if(direction == 0) probability_stack[probability_offset + n] = (status == 0.0f && active == 1.0f ? result : 0.0f); else projected_stack[probability_offset + n] = (status == 0.0f && active == 1.0f ? projected_value : 0.0f); } // Inactive physical/normalized rows are finite service zeros, same index. destination[n] = status == 0.0f ? result : 0.0f; flags[n] = status; } __kernel void DPCCCompositeProjectionMask(__global float *rollout_context, __global float *candidate_status, __global const float *projection_status, __global float *normalized_corrected, __global float *flags, uint M, uint H, uint active_offset) { const size_t m = get_global_id(0), row_size = (size_t)H * 6; if(m >= M) return; const float active = rollout_context[active_offset + m], previous = candidate_status[m]; const float projected = projection_status[m]; float flag = 0.0f; if((active != 0.0f && active != 1.0f) || (previous != 0.0f && previous != 1.0f && previous != 2.0f) || (projected != 0.0f && projected != 1.0f && projected != 2.0f) || (active == 1.0f && previous != 0.0f) || (active == 0.0f && previous == 0.0f)) flag = 3.0f; for(size_t c = 0; c < row_size; c++) if(!isfinite(normalized_corrected[m * row_size + c])) flag = 3.0f; if(flag == 0.0f) { if(active == 1.0f && projected != 0.0f) { rollout_context[active_offset + m] = 0.0f; candidate_status[m] = projected; } // An already failed row keeps its own explicit failure code permanently. if(active == 0.0f || projected != 0.0f) for(size_t c = 0; c < row_size; c++) normalized_corrected[m * row_size + c] = 0.0f; } flags[m] = flag; } // ID360: owner Diagnostics[M,12] summaryV1. Not full proposal history. __kernel void DPCCCompositeSummary(__global const float *projection, __global const float *R,__global const float *eligible,__global float *summary, __global float *flags,const uint M,const uint activeOffset,const uint stage) { const uint m=get_global_id(0);if(m>=M)return; const uint row=m*12;uint error=0; const float active=R[activeOffset+m]; if(stage>2||!isfinite(active)||(active!=0&&active!=1)) error=3; if(stage==0) { summary[row]=0;summary[row+1]=0; for(uint i=2;i<10;i++)summary[row+i]=-1; summary[row+10]=active;summary[row+11]=0; } else if(stage==1 && active==1) { for(uint i=0;i<12;i++)if(!isfinite(projection[row+i]))error=5; const float eval=projection[row],iter=projection[row+1]; if(eval<0||iter<0||eval!=floor(eval)||iter!=floor(iter)||projection[row+10]!=1)error=3; if(!isfinite(summary[row])||!isfinite(summary[row+1])||summary[row]<0||summary[row+1]<0|| eval>16777216-summary[row]||iter>16777216-summary[row+1])error=5; const float evaluations=summary[row]+eval,iterations=summary[row+1]+iter; if(!isfinite(evaluations)||!isfinite(iterations)||evaluations>16777216||iterations>16777216)error=5; if(error==0) { summary[row]=evaluations;summary[row+1]=iterations; for(uint i=2;i<10;i++)summary[row+i]=projection[row+i]; } } else if(stage==2) { const float valid=eligible[m]; if(!isfinite(valid)||(valid!=0&&valid!=1)||(active==0&&valid==1))error=3; if(error==0){summary[row+10]=active;summary[row+11]=valid;} } flags[m]=(float)error; } // ID361; host metadata contains relative seconds only, never absolute UTC. __kernel void DPCCCompositeBindTimes(__global const float *context, __global const long *relative,__global float *flags,const uint K,const uint H, const uint E,const uint executionOffset,const uint countOffset,const uint calendarOffset, const uint firstMarkAtOpen) { const uint row=get_global_id(0);if(row>=K*H)return; const uint h=row%H;uint error=0; const long duration=relative[2*h],gap=relative[2*h+1]; if(duration<=0||gap<0||context[executionOffset+row*24+18]!=(float)duration || context[executionOffset+row*24+19]!=(float)gap)error=3; const float n=context[countOffset+h]; if(!isfinite(n)||n<0||n>E||n!=floor(n)||context[executionOffset+row*24+21]!=n)error=3; if(error==0) for(uint e=0;e=0x1.0p63f||offset< -0x1.0p63f)error=3; else { const long seconds=convert_long_rtz(offset); if(seconds>duration || (h==0?(seconds<0||(firstMarkAtOpen&&seconds<=0)):seconds<=-gap))error=3; } } else for(uint f=0;f<6;f++)if(context[i+f]!=0)error=3; } flags[row]=(float)error; } // ID362: compact FNV mutation identity, not a cryptographic provenance digest. __kernel void DPCCCompositeSnapshotIdentity(__global const float *context, __global const float *account,__global ulong *result,const uint count) { if(get_global_id(0)!=0)return; ulong hash=(ulong)14695981039346656037UL; hash^=(ulong)count;hash*=(ulong)1099511628211UL; for(uint i=0;i=M) return; if(K==0||K>N||N>64||(N&(N-1))!=0||!isfinite(tolerance)||tolerance<0||tolerance>=1) {eligible[m]=0;flags[m]=4;return;} uint error=0; float total=0; for(uint k=0;ktolerance) error=3; const float a=R[activeOffset+m],s=status[m],v=endvalid[m]; if(!isfinite(a)||(a!=0&&a!=1)||!isfinite(s)||(s!=0&&s!=1&&s!=2)|| !isfinite(v)||(v!=0&&v!=1)) error=3; const uint valid=(a==1&&s==0&&v==1); for(uint k=0;k64||(N&(N-1))!=0) {receipt[0]=4;receipt[1]=-1;receipt[2]=0;receipt[3]=0;return;} uint error=0,count=0; int winner=-1; float best=0; for(uint m=0;mtolerance|| fqfdiag[d+3]<0||fqfdiag[d+5]!=0) {error=5;continue;} float prefix[64],next[64],terms[64]; uint seen[64]; for(uint k=0;k=N||q!=floor(q)) error=5; else {const uint original=(uint)q;if(seen[original]) error=5;seen[original]=1;} if(k&& (z[i]tolerance) {error=5;continue;} // Reproduce the native parallel CDF, with explicit overflow checks. for(uint offset=1;offset=offset?prefix[k-offset]:0); for(uint k=0;kbest) {winner=(int)m;best=scores[m];} } receipt[0]=(float)(error?error:(winner<0?1:0));receipt[1]=(float)winner; receipt[2]=(winner<0?0:best);receipt[3]=(float)count; if(error||winner<0) return; for(uint i=0;i= count) return; const float bias[] = {b0,b1,b2,b3,b4,b5}, scale[] = {s0,s1,s2,s3,s4,s5}; const uint a = n % 6; float status = 0.0f; const float raw = actions[n], noise = epsilon[n], b = bias[a], s = scale[a]; if(!isfinite(raw) || !isfinite(noise) || !isfinite(b) || !isfinite(s) || s <= 0.0f || !isfinite(alpha_bar) || alpha_bar <= 0.0f || alpha_bar >= 1.0f) status = 3.0f; const float centered = raw - b, y0 = centered / s; const float mean = sqrt(alpha_bar) * y0, stochastic = sqrt(1.0f-alpha_bar) * noise; const float value = mean + stochastic; if(status == 0.0f && (!isfinite(centered) || !isfinite(y0) || !isfinite(mean) || !isfinite(stochastic) || !isfinite(value))) status = 5.0f; candidate[n] = status == 0.0f ? value : 0.0f; flags[n] = status; } // Causal affine target: no statistics are fitted from future native OHLC. // Existing rho loss broadcasts this H4 target across all K scenarios. __kernel void DPCCDecoderNativeTarget(__global const float *native, __global const float *price_norm, __global float *target, __global float *receipt, const uint count) { if(get_global_id(0) != 0) return; int status = count == 0 || count % 4 != 0 ? DPCC_STATUS_INVALID : 0; for(uint i = 0; i < count; i++) { const uint p = (i % 4) * 4; const float mean = price_norm[p], std = price_norm[p + 1]; const float gain = price_norm[p + 2], bias = price_norm[p + 3]; if(!isfinite(native[i]) || !isfinite(mean) || !isfinite(std) || !isfinite(gain) || !isfinite(bias)) status = DPCC_STATUS_NUMERIC; else if((std <= 0.0f || gain != 1.0f || bias != 0.0f) && status != DPCC_STATUS_NUMERIC) status = DPCC_STATUS_INVALID; const float variance = std * std; const float centered = native[i] - mean; const float normalized = centered / std; if(status == 0 && (!isfinite(variance) || variance <= 0.0f || !isfinite(centered) || !isfinite(normalized))) status = DPCC_STATUS_NUMERIC; } // Complete admission precedes writes into the private normalized candidate. if(status == 0) for(uint i = 0; i < count; i++) { const uint p = (i % 4) * 4; target[i] = (native[i] - price_norm[p]) / price_norm[p + 1]; } receipt[0] = (float)status; receipt[1] = 0.0f; receipt[2] = 0.0f; } //+------------------------------------------------------------------+ //| Price head: causal normalized OHLC from shared raw parameters | //+------------------------------------------------------------------+ __kernel void DPCCDecoderPricePath(__global const float *raw, __global float *normalized, __global float *flags, uint K, uint H) { const size_t k = get_global_id(0); if(k >= K) return; int status = 0; float previous_close = 0.0f; for(uint h = 0; h < H; h++) { const size_t p = (k * H + h) * 4; const float gap = h == 0 ? 0.0f : raw[p]; const float body = raw[p + 1]; const float upper_raw = raw[p + 2]; const float lower_raw = raw[p + 3]; const float upper = fmax(upper_raw, 0.0f) + log1p(exp(-fabs(upper_raw))); const float lower = fmax(lower_raw, 0.0f) + log1p(exp(-fabs(lower_raw))); const float open = h == 0 ? 0.0f : previous_close + gap; const float close = open + body; const float high = open + fmax(body, 0.0f) + upper; const float low = open + fmin(body, 0.0f) - lower; if(!isfinite(gap) || !isfinite(body) || !isfinite(upper_raw) || !isfinite(lower_raw) || !isfinite(upper) || !isfinite(lower) || !isfinite(open) || !isfinite(close) || !isfinite(high) || !isfinite(low) || high < fmax(open, close) || low > fmin(open, close)) status = DPCC_STATUS_NUMERIC; normalized[p] = open; normalized[p + 1] = high; normalized[p + 2] = low; normalized[p + 3] = close; previous_close = close; } flags[k] = (float)status; } //+------------------------------------------------------------------+ //| Price head: immutable affine conversion and explicit H prefix | //+------------------------------------------------------------------+ __kernel void DPCCDecoderPriceNative(__global const float *normalized, __global float *native, __global float *flags, uint K, uint H, uint plan_horizon, float anchor, float scale, float half_tick, float anchor_error, float scale_error) { const size_t row = get_global_id(0); if(row >= (size_t)K * plan_horizon) return; const size_t source = (row / plan_horizon * H + row % plan_horizon) * 4; float result[4]; int status = 0; for(uint c = 0; c < 4; c++) { const float y = normalized[source + c]; const float product = scale * y; const float price = anchor + product; const float uncertainty = anchor_error + scale_error * fabs(y) + 1.1920928955078125e-7f * (fabs(anchor) + fabs(product) + fabs(price)); if(!isfinite(y) || !isfinite(product) || !isfinite(price) || !isfinite(uncertainty) || uncertainty >= half_tick) status = DPCC_STATUS_NUMERIC; result[c] = price; } if(result[1] < fmax(result[0], result[3]) || result[2] > fmin(result[0], result[3])) status = DPCC_STATUS_NUMERIC; for(uint c = 0; c < 4; c++) native[row * 4 + c] = status == 0 ? result[c] : 0.0f; flags[row] = (float)status; } //+------------------------------------------------------------------+ //| Price head: exact reverse of the causal normalized OHLC path | //+------------------------------------------------------------------+ __kernel void DPCCDecoderPriceGradient(__global const float *raw, __global const float *upstream, __global float *raw_gradient, __global float *flags, uint K, uint H) { const size_t k = get_global_id(0); if(k >= K) return; int status = H == 0 ? DPCC_STATUS_INVALID : 0; float next_open_gradient = 0.0f; for(uint reverse_h = H; reverse_h > 0; reverse_h--) { const uint h = reverse_h - 1; const size_t p = (k * H + h) * 4; const float body = raw[p + 1]; const float upper_raw = raw[p + 2]; const float lower_raw = raw[p + 3]; const float upper_exp = exp(-fabs(upper_raw)); const float lower_exp = exp(-fabs(lower_raw)); const float upper_sigmoid = upper_raw >= 0.0f ? 1.0f / (1.0f + upper_exp) : upper_exp / (1.0f + upper_exp); const float lower_sigmoid = lower_raw >= 0.0f ? 1.0f / (1.0f + lower_exp) : lower_exp / (1.0f + lower_exp); const float close_gradient = upstream[p + 3] + next_open_gradient; const float body_gradient = close_gradient + (body > 0.0f ? upstream[p + 1] : 0.0f) + (body < 0.0f ? upstream[p + 2] : 0.0f); const float open_gradient = upstream[p] + upstream[p + 1] + upstream[p + 2] + close_gradient; const float gap_gradient = h == 0 ? 0.0f : open_gradient; const float upper_gradient = upstream[p + 1] * upper_sigmoid; const float lower_gradient = -upstream[p + 2] * lower_sigmoid; if((h > 0 && !isfinite(raw[p])) || !isfinite(body) || !isfinite(upper_raw) || !isfinite(lower_raw) || !isfinite(upstream[p]) || !isfinite(upstream[p + 1]) || !isfinite(upstream[p + 2]) || !isfinite(upstream[p + 3]) || !isfinite(close_gradient) || !isfinite(body_gradient) || !isfinite(open_gradient) || !isfinite(upper_gradient) || !isfinite(lower_gradient)) status = DPCC_STATUS_NUMERIC; raw_gradient[p] = status == 0 ? gap_gradient : 0.0f; raw_gradient[p + 1] = status == 0 ? body_gradient : 0.0f; raw_gradient[p + 2] = status == 0 ? upper_gradient : 0.0f; raw_gradient[p + 3] = status == 0 ? lower_gradient : 0.0f; next_open_gradient = h == 0 || status != 0 ? 0.0f : open_gradient; } flags[k] = (float)status; } //+------------------------------------------------------------------+ //| DPCCDecoderPriceLoss builds detached rho-weighted Huber signals | //+------------------------------------------------------------------+ __kernel void DPCCDecoderPriceLoss(__global const float *y, __global const float *target, __global const float *rho, __global float *gradient_scratch, __global float *partials, __global float *flags, uint K, uint H, uint prepare_gradient, float lambda_price, float delta) { const size_t k = get_global_id(0); if(k >= K) return; if(K == 0 || H == 0 || prepare_gradient > 1 || !isfinite(lambda_price) || lambda_price <= 0.0f || !isfinite(delta) || delta <= 0.0f) { partials[k] = 0.0f; flags[k] = (float)DPCC_STATUS_INVALID; return; } int status = 0; const float mass = rho[k]; if(!isfinite(mass)) status = DPCC_STATUS_NUMERIC; else if(mass < 0.0f) status = DPCC_STATUS_INVALID; const size_t width = (size_t)H * 4; const float factor = lambda_price * mass / (float)width; if(!isfinite(factor)) status = DPCC_STATUS_NUMERIC; float sum = 0.0f; for(size_t coordinate = 0; coordinate < width; coordinate++) { const size_t n = k * width + coordinate; const float prediction = y[n]; const float expected = target[coordinate]; const float error = prediction - expected; const float magnitude = fabs(error); const float huber = magnitude <= delta ? 0.5f * error * error : delta * (magnitude - 0.5f * delta); const float weighted = factor * huber; const float next = sum + weighted; if(!isfinite(prediction) || !isfinite(expected) || !isfinite(error) || !isfinite(huber) || !isfinite(weighted) || !isfinite(next)) status = DPCC_STATUS_NUMERIC; sum = next; if(prepare_gradient) { const float gradient = -factor * clamp(error, -delta, delta); if(!isfinite(gradient)) status = DPCC_STATUS_NUMERIC; gradient_scratch[n] = gradient; } } partials[k] = sum; flags[k] = (float)status; } //+------------------------------------------------------------------+ //| Exact ForecastPrice branch in the native shared Conv layout | //+------------------------------------------------------------------+ __kernel void DPCCForecastPriceBridge(__global const float *price_z, __global const float *weights, __global const float *native_generator, __global const float *native_e, __global float *combined_generator, __global float *price_e, __global float *scratch, __global float *control, const uint scenarios, const uint variables, const uint horizon, const uint dimension, const uint phase, __global const float *delta_target) { const uint id = get_global_id(0); const uint state = variables * dimension; if(phase == 1) { if(id != 0) return; float invalid = 0.0f; float nonzero = 0.0f; for(uint q = 0; q < state; q++) { invalid += scratch[2 * q]; nonzero += scratch[2 * q + 1]; } control[0] = invalid; control[1] = nonzero; return; } if(id >= state || dimension == 0 || variables == 0 || horizon == 0 || scenarios == 0) return; const uint v = id / dimension; const uint d = id % dimension; float invalid = 0.0f; float nonzero = 0.0f; if(phase == 0) { float aggregate = 0.0f; for(uint k = 0; k < scenarios; k++) for(uint h = 0; h < horizon; h++) { const uint token = ((k * variables + v) * horizon + h) * dimension; const float direct = price_z[token + d]; if(!isfinite(direct)) invalid = 1.0f; if(direct != 0.0f) nonzero = 1.0f; float projected = 0.0f; //--- Native Conv variables=1; filter-major rows of width 2D+1. //--- Only E columns [0,D) propagate; prototype columns stay detached. for(uint out = 0; out < dimension; out++) { const float g = price_z[token + out]; const float w = weights[out * (2 * dimension + 1) + d]; const float product = w * g; if(!isfinite(g) || !isfinite(w) || !isfinite(product)) invalid = 1.0f; projected += product; } aggregate += direct + projected; } if(!isfinite(aggregate)) invalid = 1.0f; price_e[id] = aggregate; } else if(phase == 2) { //--- Preserve native generator signals until the native upstream is complete. for(uint k = 0; k < scenarios; k++) for(uint h = 0; h < horizon; h++) { const uint index = ((k * variables + v) * horizon + h) * dimension + d; const float native = native_generator[index]; const float price = price_z[index]; const float sum = (price == 0.0f ? native : native + price); if(!isfinite(native) || !isfinite(price) || !isfinite(sum)) invalid = 1.0f; combined_generator[index] = sum; } const float native = native_e[id]; const float price = price_e[id]; const float sum = (price == 0.0f ? native : native + price); if(!isfinite(native) || !isfinite(price) || !isfinite(sum)) invalid = 1.0f; price_e[id] = sum; } else if(phase == 3) { for(uint h = 0; h < horizon; h++) if(!isfinite(delta_target[(v * horizon + h) * dimension + d])) invalid = 1.0f; } else invalid = 1.0f; scratch[2 * id] = invalid; scratch[2 * id + 1] = nonzero; } //+------------------------------------------------------------------+ //| Prepares the immutable Action6 condition and canonical Pi | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserFrameCondition(__global const float *source, __global const float *profile, __global float *account, __global float *tokens, __global float *pi, __global float *flags, uint K, uint token_count, uint dimension, float probability_tolerance, uint mode) { const size_t n = get_global_id(0); const size_t token_scalars = (size_t)K * token_count * (dimension + 1); const size_t total = 109 + token_scalars + K; if(n >= total) return; if(mode == 1) { uint error = 0; float contribution = 0.0f; if(n < 109) contribution = account[n] * profile[109 + n]; else if(n < 109 + token_scalars) contribution = tokens[n - 109]; else contribution = pi[n - 109 - token_scalars]; const float next = source[n] + contribution; if(!isfinite(source[n]) || !isfinite(contribution) || !isfinite(next)) error = DPCC_STATUS_NUMERIC; ((__global float *)source)[n] = error == 0 ? next : 0.0f; flags[n] = (float)error; return; } if(mode != 0) { flags[n] = (float)DPCC_STATUS_INVALID; return; } uint error = 0; const float raw = source[n]; if(!isfinite(raw)) error = DPCC_STATUS_NUMERIC; if(n < 109) { const float bias = profile[n], scale = profile[109 + n]; if(!isfinite(bias) || !isfinite(scale) || scale <= 0.0f) error = DPCC_STATUS_INVALID; const float value = (raw - bias) / scale; if(!isfinite(value)) error = DPCC_STATUS_NUMERIC; account[n] = error == 0 ? value : 0.0f; } else if(n < 109 + token_scalars) tokens[n - 109] = error == 0 ? raw : 0.0f; else { if(raw < 0.0f) error = DPCC_STATUS_INVALID; pi[n - 109 - token_scalars] = error == 0 ? raw : 0.0f; } if(n == 0) { float mass = 0.0f; for(uint k = 0; k < K; k++) { const float value = source[109 + token_scalars + k]; if(!isfinite(value) || value < 0.0f) error = DPCC_STATUS_INVALID; mass += value; } if(!isfinite(mass) || mass <= 0.0f || fabs(mass - 1.0f) > probability_tolerance) error = DPCC_STATUS_INVALID; } flags[n] = (float)error; } //+------------------------------------------------------------------+ //| Packs per-scenario attention and applies the original Market Pi | //+------------------------------------------------------------------+ __kernel void DPCCDenoiserFrameLayout(__global const float *key, __global const float *value, __global const float *query, __global const float *pi, __global float *packed_query, __global float *packed_kv, __global float *weighted, __global float *flags, uint K, uint tokens, uint width, uint heads, uint mode) { const size_t n = get_global_id(0); if(mode == 0) { if(n >= (size_t)K * tokens * width) return; const uint dh = width / heads; const size_t d = n % width, token = n / width % tokens, k = n / ((size_t)tokens * width); const size_t group = k * heads + d / dh; const size_t physical = (token * 2 * K * heads + group) * dh + d % dh; uint error = 0; const float kval = key[n], vval = value[n]; if(!isfinite(kval) || !isfinite(vval)) error = DPCC_STATUS_NUMERIC; packed_kv[physical] = error == 0 ? kval : 0.0f; packed_kv[physical + (size_t)K * width] = error == 0 ? vval : 0.0f; if(token == 0) { const float q = query[d]; if(!isfinite(q)) error = DPCC_STATUS_NUMERIC; packed_query[k * width + d] = error == 0 ? q : 0.0f; } flags[n] = (float)error; } else if(mode == 1) { if(n >= width) return; uint error = 0; float result = 0.0f; for(uint k = 0; k < K; k++) { const float p = pi[k], q = query[(size_t)k * width + n]; const float term = p * q; if(!isfinite(p) || p < 0.0f || !isfinite(q) || !isfinite(term)) error = DPCC_STATUS_NUMERIC; result += term; } if(!isfinite(result)) error = DPCC_STATUS_NUMERIC; weighted[n] = error == 0 ? result : 0.0f; flags[n] = (float)error; } else if(mode == 2) { if(n >= tokens) return; const float result = n < K ? key[n] : 0.0f; const uint error = isfinite(result) ? 0 : DPCC_STATUS_NUMERIC; weighted[n] = error == 0 ? result : 0.0f; flags[n] = (float)error; } else if(mode == 3) { if(n >= (size_t)K * width) return; const uint k = (uint)(n / width); const uint d = (uint)(n % width); uint error = 0; const float upstream = key[d]; const float probability = pi[k]; const float value = query[n]; const float grad = upstream * probability; if(!isfinite(upstream) || !isfinite(probability) || probability < 0.0f || !isfinite(value) || !isfinite(grad)) error = DPCC_STATUS_NUMERIC; packed_query[n] = error == 0 ? grad : 0.0f; if(d == 0) { float probability_gradient = 0.0f; for(uint i = 0; i < width && error == 0; i++) { const float term = key[i] * query[(size_t)k * width + i]; if(!isfinite(term)) error = DPCC_STATUS_NUMERIC; else probability_gradient += term; } if(!isfinite(probability_gradient)) error = DPCC_STATUS_NUMERIC; packed_kv[k] = error == 0 ? probability_gradient : 0.0f; } flags[n] = (float)error; } else if(mode == 4) { if(n >= (size_t)K * tokens * width) return; const uint dh = width / heads; const uint d = (uint)(n % width); const uint token = (uint)(n / width % tokens); const uint k = (uint)(n / ((size_t)tokens * width)); const uint group = k * heads + d / dh; const size_t physical = ((size_t)token * 2 * K * heads + group) * dh + d % dh; const float grad_key = value[physical]; const float grad_value = value[physical + (size_t)K * width]; uint error = 0; if(!isfinite(grad_key) || !isfinite(grad_value)) error = DPCC_STATUS_NUMERIC; packed_kv[n] = error == 0 ? grad_key : 0.0f; weighted[n] = error == 0 ? grad_value : 0.0f; if(k == 0 && token == 0) { float grad_query = 0.0f; for(uint scenario = 0; scenario < K && error == 0; scenario++) { const float term = key[(size_t)scenario * width + d]; if(!isfinite(term)) error = DPCC_STATUS_NUMERIC; else grad_query += term; } if(!isfinite(grad_query)) error = DPCC_STATUS_NUMERIC; packed_query[d] = error == 0 ? grad_query : 0.0f; } flags[n] = (float)error; } else if(mode == 5) { if(n >= tokens) return; const float result = n < K ? key[n] : 0.0f; const uint error = K > tokens ? DPCC_STATUS_INVALID : (isfinite(result) ? 0 : DPCC_STATUS_NUMERIC); weighted[n] = error == 0 ? result : 0.0f; flags[n] = (float)error; } else { if(n < (size_t)K * width * tokens) flags[n] = (float)DPCC_STATUS_INVALID; } } //+------------------------------------------------------------------+ //| Read-only AC-SRM candidate rows. | //+------------------------------------------------------------------+ inline void DPCCQBad(__global float *f,uint r){atomic_or((volatile __global uint*)(f+r),0x3f800000U);} inline float DPCCQFinite(float v,__global float *f,uint r){if(!isfinite(v)){DPCCQBad(f,r);return 0.0f;}return v;} __kernel void DPCCQRowsDense(__global const float*w,__global const float*x,__global float*y,__global float*f, const uint ni,const uint no,const uint shared,const int act) { const uint n=get_global_id(0),r=get_global_id(1);float s=0.0f; for(uint i=0;i0.0f)||gate<0.0f){DPCCQBad(f,r);y[qb+d]=0.0f;} else y[qb+d]=DPCCQFinite(v/mass*gate,f,r);if(d==0)lse[(r*queries+h)*scenarios+k]=DPCCQFinite(log(mass)+maximum,f,r); } __kernel void DPCCQRowsSoftmax(__global const float*x,__global float*y,__global float*f,const uint width) { const uint n=get_global_id(0),r=get_global_id(1);float maximum=-INFINITY,mass=0.0f; for(uint i=0;i=tolerance)DPCCQBad(f,r); } __kernel void DPCCQRowsJoin(__global const float*a,__global const float*p,__global float*q,__global float*f,const uint horizon) { const uint n=get_global_id(0),r=get_global_id(1);q[r*19+n]=DPCCQFinite(n<13?a[n]:p[r*horizon*6+n-13],f,r); } __kernel void DPCCQRowsRank(__global const float*q1,__global const float*q2,__global const float*allowed, __global const float*f,__global float*scores,__global float*selected,const uint rows) { uint best=rows;float value=-INFINITY;for(uint m=0;mvalue){best=m;value=s;}} selected[0]=(best==rows?-1.0f:(float)best); } // ABI3 Action6 condition: I09 raw query followed by Z/logU tokens and Pi. __kernel void DPCCAction6PackCondition(__global const float *account13, __global const float *physical, __global const float *z, __global const float *logu, __global const float *pi, __global float *condition, __global float *flags, const uint K,const uint V,const uint Hm,const uint D, const uint account_offset,const uint instrument_offset, const uint limits_offset,const uint condition_count) { const uint i=get_global_id(0); if(i>=condition_count)return; const uint tokens=K*V*Hm, token_width=D+1, token_values=tokens*token_width; float value=0.0f; uint error=0; if(i<13) value=account13[i]; else if(i<37) value=physical[account_offset+i-13]; else if(i<85) value=physical[instrument_offset+i-37]; else if(i<109) value=physical[limits_offset+i-85]; else if(i<109+token_values) { const uint at=i-109, token=at/token_width, feature=at%token_width; value=(feature=M*row)return; const uint m=n/row,coordinate=n%row; const float active=context[active_offset+m]; const uint source=noise_offset+(m*(J+1)+J)*row+coordinate; const float value=noise[source]; uint error=(!isfinite(active)||(active!=0.0f&&active!=1.0f)||!isfinite(value)?3:0); current[n]=(error==0&&active==1.0f?value:0.0f); if(coordinate==0) { status[m]=(error==0?0.0f:3.0f); // Initial inactivity is a candidate disposition, not a numerical failure. candidate_status[m]=(error==0&&active==1.0f?0.0f:1.0f); } flags[n]=(float)error; } __kernel void DPCCAction6Reverse(__global const float *current, __global const float *epsilon, __global const float *noise, __global const float *context, __global float *proposal, __global const float *status, __global float *flags, const uint M,const uint H,const uint J,const uint j, const uint noise_offset,const uint active_offset, const float beta,const float alpha_bar,const float sigma) { const uint n=get_global_id(0),row=H*6; if(n>=M*row)return; const uint m=n/row,coordinate=n%row; const float active=context[active_offset+m],x=current[n]; uint error=0;float value=0.0f; if(j==0||j>J||!isfinite(beta)||beta<=0.0f||beta>=1.0f|| !isfinite(alpha_bar)||alpha_bar<=0.0f||alpha_bar>=1.0f|| !isfinite(sigma)||sigma<0.0f||(j==1&&sigma!=0.0f)|| !isfinite(active)||(active!=0.0f&&active!=1.0f)||!isfinite(x)|| !isfinite(status[m])||(status[m]!=0.0f&&status[m]!=3.0f))error=3; if(error==0&&active==0.0f)value=x; if(error==0&&active==1.0f&&status[m]==0.0f) { const float eps=epsilon[n],root_var=sqrt(1.0f-alpha_bar),root_alpha=sqrt(1.0f-beta); const float correction=beta*eps/root_var,mu=(x-correction)/root_alpha; if(!isfinite(eps)||!isfinite(root_var)||root_var<=0.0f||!isfinite(root_alpha)||root_alpha<=0.0f||!isfinite(mu))error=5; else { value=mu; if(j>1) { const uint at=noise_offset+(m*(J+1)+j-1)*row+coordinate; const float perturbation=sigma*noise[at];value=mu+perturbation; if(!isfinite(noise[at])||!isfinite(perturbation)||!isfinite(value))error=5; } } } proposal[n]=(error==0?value:0.0f);flags[n]=(float)error; } // Candidate flags consumed by DPCCQRowsRank: zero is allowed, nonzero excluded. // Keep the projector's detailed reason in candidate_status; publish the merged // Full-H validity is required for training; inference ranks the current Action6. __kernel void DPCCAction6CandidateMask(__global const float *candidate_status, __global const float *rollout_valid, __global float *candidate_flags, const uint M, const uint require_rollout_valid) { const uint m=get_global_id(0); if(m>=M) return; const float projected=candidate_status[m],valid=rollout_valid[m]; const uint rollout_allowed = (require_rollout_valid == 0U || valid == 1.0f); candidate_flags[m] = (isfinite(projected) && isfinite(valid) && projected == 0.0f && rollout_allowed) ? 0.0f : 1.0f; } // Seed a physical Action6 VJP at only (selected,h=0). No sigmoid or row reduction. __kernel void DPCCAction6GradientSeed(__global const float *gradient6, __global const float *projection_status, __global const float *rollout_valid, __global const float *context, __global float *plan_gradient, __global float *flags, const uint M,const uint H,const uint selected, const uint active_offset) { const uint n=get_global_id(0),row=H*6; if(n>=M*row)return; const uint m=n/row,c=n%6,h=(n%row)/6; uint error=0; float value=0.0f; if(selected>=M)error=3; else { const float projected=projection_status[selected],valid=rollout_valid[selected]; const float active=context[active_offset+selected]; if(!isfinite(projected)||projected!=0.0f||!isfinite(valid)||valid!=1.0f|| !isfinite(active)||active!=1.0f)error=3; if(error==0&&m==selected&&h==0) { value=gradient6[c]; if(!isfinite(value))error=5; } } plan_gradient[n]=(error==0?value:0.0f); flags[n]=(float)error; } // Seed all H physical derivatives for only the selected candidate. __kernel void DPCCFullPlanGradientSeed(__global const float *gradient_h6, __global const float *projection_status, __global const float *rollout_valid, __global const float *context, __global float *plan_gradient, __global float *flags, const uint M, const uint H, const uint selected, const uint active_offset, const uint allow_invalid_rollout) { const uint n = get_global_id(0), row = H * 6; if(n >= M * row) return; const uint m = n / row, coordinate = n % row; uint error = 0; float value = 0.0f; if(selected >= M || allow_invalid_rollout > 1U) error = 3; else { const float projected = projection_status[selected], valid = rollout_valid[selected]; const float active = context[active_offset + selected]; if(!isfinite(projected) || projected != 0.0f || !isfinite(valid) || (valid != 1.0f && !(allow_invalid_rollout == 1U && valid == 0.0f)) || !isfinite(active) || active != 1.0f) error = 3; if(error == 0 && m == selected) { value = gradient_h6[coordinate]; if(!isfinite(value)) error = 5; } } plan_gradient[n] = (error == 0 ? value : 0.0f); flags[n] = (float)error; } // mode0: affine adjoint; mode1: inverse affine adjoint; mode2: reverse-DDPM; // mode3: branch sum; mode4: sigmoid pullback using the saved bounded action. // All four plan operands and flags have disjoint device ranges at the caller. __kernel void DPCCAction6GradientStep(__global const float *source, __global const float *other, __global float *result, __global float *epsilon_gradient, __global float *flags, const uint count,const uint mode,const uint other_offset, const float beta,const float alpha_bar, const float s0,const float s1,const float s2, const float s3,const float s4,const float s5) { const uint n=get_global_id(0); if(n>=count)return; const float scales[6]={s0,s1,s2,s3,s4,s5}; const float g=source[n],scale=scales[n%6]; float value=0.0f,epsilon=0.0f; uint error=(!isfinite(g)||!isfinite(scale)||scale<=0.0f||mode>5?3:0); if(error==0) { if(mode==0)value=g*scale; else if(mode==1 || mode==5) { const float probability=other[other_offset+n]; if(!isfinite(probability)||probability<0.0f||probability>1.0f)error=5; else if(probability<=DPCC_ACTION_PROBABILITY_EPS|| probability>=1.0f-DPCC_ACTION_PROBABILITY_EPS)value=0.0f; else if(mode==1)value=g/(scale*probability*(1.0f-probability)); else value=g; } else if(mode==2) { if(!isfinite(beta)||beta<=0.0f||beta>=1.0f|| !isfinite(alpha_bar)||alpha_bar<=0.0f||alpha_bar>=1.0f)error=3; else { const float root_alpha=sqrt(1.0f-beta),root_variance=sqrt(1.0f-alpha_bar); value=g/root_alpha; epsilon=-beta*g/(root_alpha*root_variance); } } else { if(mode==3) { const float branch=other[other_offset+n]; if(!isfinite(branch))error=5; else value=g+branch; } else { const float probability=other[other_offset+n]; if(!isfinite(probability)||probability<0.0f||probability>1.0f)error=5; else value=g*scale*probability*(1.0f-probability); } } } if(!isfinite(value)||(mode==2&&!isfinite(epsilon)))error=5; result[n]=(error==0?value:0.0f); if(mode==2)epsilon_gradient[n]=(error==0?epsilon:0.0f); flags[n]=(float)error; }