//+------------------------------------------------------------------+ //| KronosPredictorS1.mqh | //| MMQ — Muhammad Minhas Qamar | //| www.mql5.com | //+------------------------------------------------------------------+ #property copyright "MMQ — Muhammad Minhas Qamar" #property link "https://www.mql5.com" #property version "1.00" #ifndef KRONOS_PREDICTOR_S1_MQH #define KRONOS_PREDICTOR_S1_MQH #include // loaders, KR_S1_BITS/KR_S2_BITS #include // LinearT, AddRowBias, RMSNorm, TransformerBlock #define KR_PRED_VOCAB 1024 // 2^10, s1 and s2 vocab //+------------------------------------------------------------------+ //| Predictor, decode_s1 stage. | //+------------------------------------------------------------------+ class CKronosPredictorS1 { protected: int m_blocks; // n_layers (8) -- predictor uses the FULL count, no n-1 int m_heads; // 8 ulong m_dm; // 512 ulong m_ff; // 1024 string m_dir; //--- embeddings matrix m_embS1; // emb_s1.weight : [vocab, d_model] matrix m_embS2; // emb_s2.weight : [vocab, d_model] matrix m_fusionW; vector m_fusionB; // fusion_proj : [d_model, 2*d_model] matrix m_teMin, m_teHour, m_teWday, m_teDay, m_teMon; // [size, d_model] each //--- 8 transformer blocks vector m_n1[]; matrix m_Wq[]; vector m_bq[]; matrix m_Wk[]; vector m_bk[]; matrix m_Wv[]; vector m_bv[]; matrix m_Wo[]; vector m_bo[]; vector m_n2[]; matrix m_W1[]; matrix m_W3[]; matrix m_W2[]; //--- final norm + s1 head vector m_normW; // norm.weight : [d_model] matrix m_projS1W; vector m_projS1B; // head.proj_s1 : [vocab, d_model] //--- KV-cache (grow-phase only) //--- Per block: the projected K (already RoPE-rotated, heads concatenated) and //--- raw V over all cached positions, each (m_cacheLen, d_model). In the grow //--- phase a token's absolute position never changes, so a K rotated once at //--- write-time stays valid; only the new last row is computed per step. matrix m_cacheK[]; // [m_blocks] of (T, d_model) matrix m_cacheV[]; // [m_blocks] of (T, d_model) matrix m_cacheContext; // (T, d_model) post-final-norm context // for every position; decode_s2 cross- // attn keys/values run over ALL of it int m_cacheLen; // number of cached positions bool m_cacheValid; // guard: primed and grow-phase only string F(const string name) { return m_dir + name + ".bin"; } //+---------------------------------------------------------------+ //| Apply RoPE per head over a (T, d_model) projection, using a | //| cos/sin table of width head_dim. Rotates each head's column | //| slice independently, matching MHA's per-head ApplyRoPE. | //| posOffset shifts the row->position mapping (row r uses | //| position posOffset+r) so a single appended row can be rotated | //| at its true absolute position. | //+---------------------------------------------------------------+ matrix RoPEAllHeads(const matrix &M, const matrix &cosT, const matrix &sinT, int posOffset) { ulong T = M.Rows(); ulong hd = m_dm / (ulong)m_heads; matrix outM = matrix::Zeros(T, m_dm); for(int h = 0; h < m_heads; h++) { ulong c0 = (ulong)h * hd; matrix Ms = SliceCols(M, c0, hd); //--- rotate slice; cos/sin row for output row r comes from posOffset+r matrix Mr = matrix::Zeros(T, hd); ulong half = hd / 2; for(ulong r = 0; r < T; r++) { ulong p = (ulong)posOffset + r; for(ulong k = 0; k < hd; k++) { double rh = (k < half) ? -Ms[r][k + half] : Ms[r][k - half]; Mr[r][k] = Ms[r][k] * cosT[p][k] + rh * sinT[p][k]; } } WriteCols(outM, Mr, c0); } return outM; } //+---------------------------------------------------------------+ //| Embedding lookup with scaling: row `id` of the table times | //| the given scale (sqrt(d_model) for the hierarchical embed). | //+---------------------------------------------------------------+ void EmbedTokens(const int &ids[], const matrix &table, double scale, matrix &out) { int L = ArraySize(ids); out = matrix::Zeros((ulong)L, m_dm); for(int t = 0; t < L; t++) { ulong id = (ulong)ids[t]; for(ulong j = 0; j < m_dm; j++) out[t][j] = table[id][j] * scale; } } //+---------------------------------------------------------------+ //| Add one temporal table's contribution: for each row t, add | //| table row stamp[t][col] to x. | //+---------------------------------------------------------------+ void AddTimeEmbed(const matrix &stamp, int col, const matrix &table, matrix &x) { ulong L = x.Rows(); for(ulong t = 0; t < L; t++) { ulong idx = (ulong)MathRound(stamp[t][col]); // stamp stored as float for(ulong j = 0; j < m_dm; j++) x[t][j] += table[idx][j]; } } //+---------------------------------------------------------------+ //| Build the transformer input rows for the given tokens/stamp: | //| HierarchicalEmbedding( fusion_proj(cat(emb_s1*sqrt d, | //| emb_s2*sqrt d)) ) + TemporalEmbedding(stamp). Shared by the | //| full DecodeS1 and the cached prime/step paths so they embed | //| identically. out is (L, d_model). | //+---------------------------------------------------------------+ void EmbedRows(const int &s1_ids[], const int &s2_ids[], const matrix &stamp, matrix &out) { int L = ArraySize(s1_ids); double scale = MathSqrt((double)m_dm); matrix e1, e2; EmbedTokens(s1_ids, m_embS1, scale, e1); EmbedTokens(s2_ids, m_embS2, scale, e2); matrix cat = matrix::Zeros((ulong)L, 2 * m_dm); for(int t = 0; t < L; t++) { for(ulong j = 0; j < m_dm; j++) cat[t][j] = e1[t][j]; for(ulong j = 0; j < m_dm; j++) cat[t][m_dm + j] = e2[t][j]; } out = LinearT(cat, m_fusionW); AddRowBias(out, m_fusionB); if(stamp.Rows() == (ulong)L && stamp.Cols() >= 5) { AddTimeEmbed(stamp, 0, m_teMin, out); AddTimeEmbed(stamp, 1, m_teHour, out); AddTimeEmbed(stamp, 2, m_teWday, out); AddTimeEmbed(stamp, 3, m_teDay, out); AddTimeEmbed(stamp, 4, m_teMon, out); } } public: //+---------------------------------------------------------------+ //| Load embeddings, the 8 transformer blocks, the final norm and | //| the s1 head. The predictor uses the FULL n_layers, not n-1. | //+---------------------------------------------------------------+ bool Init(const string weight_dir, int n_layers, ulong d_model, int n_heads, ulong ff_dim) { m_dir = weight_dir; m_dm = d_model; m_heads = n_heads; m_ff = ff_dim; m_blocks = n_layers; // predictor: full n_layers (NOT n-1) if(d_model == 0 || n_heads == 0 || n_layers == 0 || ff_dim == 0) { Print("CKronosPredictorS1::Init: invalid config (zero dimension)"); return false; } bool ok = true; //--- embeddings //--- emb_s1/emb_s2 are lookup tables (indexed by token id) -> keep row-major ok &= KronosLoadMatrix(F("embedding_emb_s1_weight"), KR_PRED_VOCAB, m_dm, m_embS1); ok &= KronosLoadMatrix(F("embedding_emb_s2_weight"), KR_PRED_VOCAB, m_dm, m_embS2); //--- fusion_proj is a LinearT weight -> store transposed ok &= KronosLoadMatrixT(F("embedding_fusion_proj_weight"), m_dm, 2 * m_dm, m_fusionW); ok &= KronosLoadVector(F("embedding_fusion_proj_bias"), m_dm, m_fusionB); ok &= KronosLoadMatrix(F("time_emb_minute_embed_weight"), 60, m_dm, m_teMin); ok &= KronosLoadMatrix(F("time_emb_hour_embed_weight"), 24, m_dm, m_teHour); ok &= KronosLoadMatrix(F("time_emb_weekday_embed_weight"), 7, m_dm, m_teWday); ok &= KronosLoadMatrix(F("time_emb_day_embed_weight"), 32, m_dm, m_teDay); ok &= KronosLoadMatrix(F("time_emb_month_embed_weight"), 13, m_dm, m_teMon); //--- final norm + s1 head ok &= KronosLoadVector(F("norm_weight"), m_dm, m_normW); ok &= KronosLoadMatrixT(F("head_proj_s1_weight"), KR_PRED_VOCAB, m_dm, m_projS1W); ok &= KronosLoadVector(F("head_proj_s1_bias"), KR_PRED_VOCAB, m_projS1B); if(!ok) { Print("CKronosPredictorS1::Init: embedding/head load failed"); return false; } ArrayResize(m_n1, m_blocks); ArrayResize(m_n2, m_blocks); ArrayResize(m_Wq, m_blocks); ArrayResize(m_bq, m_blocks); ArrayResize(m_Wk, m_blocks); ArrayResize(m_bk, m_blocks); ArrayResize(m_Wv, m_blocks); ArrayResize(m_bv, m_blocks); ArrayResize(m_Wo, m_blocks); ArrayResize(m_bo, m_blocks); ArrayResize(m_W1, m_blocks); ArrayResize(m_W3, m_blocks); ArrayResize(m_W2, m_blocks); for(int b = 0; b < m_blocks; b++) { string p = StringFormat("transformer_%d_", b); ok &= KronosLoadVector(F(p + "norm1_weight"), m_dm, m_n1[b]); ok &= KronosLoadMatrixT(F(p + "self_attn_q_proj_weight"), m_dm, m_dm, m_Wq[b]); ok &= KronosLoadVector(F(p + "self_attn_q_proj_bias"), m_dm, m_bq[b]); ok &= KronosLoadMatrixT(F(p + "self_attn_k_proj_weight"), m_dm, m_dm, m_Wk[b]); ok &= KronosLoadVector(F(p + "self_attn_k_proj_bias"), m_dm, m_bk[b]); ok &= KronosLoadMatrixT(F(p + "self_attn_v_proj_weight"), m_dm, m_dm, m_Wv[b]); ok &= KronosLoadVector(F(p + "self_attn_v_proj_bias"), m_dm, m_bv[b]); ok &= KronosLoadMatrixT(F(p + "self_attn_out_proj_weight"), m_dm, m_dm, m_Wo[b]); ok &= KronosLoadVector(F(p + "self_attn_out_proj_bias"), m_dm, m_bo[b]); ok &= KronosLoadVector(F(p + "norm2_weight"), m_dm, m_n2[b]); ok &= KronosLoadMatrixT(F(p + "ffn_w1_weight"), m_ff, m_dm, m_W1[b]); ok &= KronosLoadMatrixT(F(p + "ffn_w3_weight"), m_ff, m_dm, m_W3[b]); ok &= KronosLoadMatrixT(F(p + "ffn_w2_weight"), m_dm, m_ff, m_W2[b]); if(!ok) { PrintFormat("CKronosPredictorS1::Init failed at block %d", b); return false; } } return ok; } //+---------------------------------------------------------------+ //| decode_s1: returns s1_logits (L, 1024) and context | //| (L, d_model). stamp is (L, 5) float with columns | //| [minute,hour,weekday,day,month], weekday already in the | //| pandas convention (Mon=0..Sun=6). | //+---------------------------------------------------------------+ bool DecodeS1(const int &s1_ids[], const int &s2_ids[], const matrix &stamp, matrix &s1_logits, matrix &context) { int L = ArraySize(s1_ids); if(L == 0 || ArraySize(s2_ids) != L) { PrintFormat("DecodeS1: bad id lengths (%d / %d)", L, ArraySize(s2_ids)); return false; } matrix x; EmbedRows(s1_ids, s2_ids, stamp, x); // HierarchicalEmbedding + TemporalEmbedding //--- 8 causal pre-norm transformer blocks for(int b = 0; b < m_blocks; b++) x = TransformerBlock(x, m_n1[b], m_Wq[b], m_bq[b], m_Wk[b], m_bk[b], m_Wv[b], m_bv[b], m_Wo[b], m_bo[b], m_heads, m_n2[b], m_W1[b], m_W3[b], m_W2[b]); //--- final RMSNorm -> this is "context" returned for decode_s2 context = RMSNorm(x, m_normW); //--- s1 head: d_model -> 1024 s1_logits = LinearT(context, m_projS1W); AddRowBias(s1_logits, m_projS1B); return true; } //+---------------------------------------------------------------+ //| Drop the KV-cache. Call before priming a fresh AR path. | //+---------------------------------------------------------------+ void ResetCache() { ArrayResize(m_cacheK, m_blocks); ArrayResize(m_cacheV, m_blocks); for(int b = 0; b < m_blocks; b++) { m_cacheK[b] = matrix::Zeros(0, m_dm); m_cacheV[b] = matrix::Zeros(0, m_dm); } m_cacheContext = matrix::Zeros(0, m_dm); m_cacheLen = 0; m_cacheValid = false; } //+---------------------------------------------------------------+ //| The running full context (T, d_model) accumulated across the | //| cached steps. decode_s2's cross-attention keys/values must | //| span ALL positions, so the caller passes this (not just the | //| last row) to DecodeS2. | //+---------------------------------------------------------------+ void GetContext(matrix &out) const { out = m_cacheContext; } //+---------------------------------------------------------------+ //| Prime the cache over the initial context window. Runs the | //| same block stack as DecodeS1 but captures, per block, the | //| post-RoPE K and the raw V for every position (the exact | //| tensors MHA forms internally). Leaves the cache holding L | //| positions so DecodeS1Step can extend it one row at a time. | //| Returns s1_logits (L,1024) and context (L,d_model) so the | //| caller can reuse them for the very first step's decode_s2 | //| (identical to what the full first-call DecodeS1 would give). | //+---------------------------------------------------------------+ bool PrimeCache(const int &s1_ids[], const int &s2_ids[], const matrix &stamp, matrix &s1_logits, matrix &context) { int L = ArraySize(s1_ids); if(L == 0 || ArraySize(s2_ids) != L) { PrintFormat("PrimeCache: bad id lengths (%d / %d)", L, ArraySize(s2_ids)); return false; } ResetCache(); matrix x; EmbedRows(s1_ids, s2_ids, stamp, x); ulong hd = m_dm / (ulong)m_heads; matrix cosT, sinT; RoPETables((ulong)L, hd, cosT, sinT); // table over the L context positions for(int b = 0; b < m_blocks; b++) { //--- pre-norm + projections (mirrors MHA on RMSNorm(x)) matrix n1 = RMSNorm(x, m_n1[b]); matrix Q = LinearT(n1, m_Wq[b]); AddRowBias(Q, m_bq[b]); matrix K = LinearT(n1, m_Wk[b]); AddRowBias(K, m_bk[b]); matrix V = LinearT(n1, m_Wv[b]); AddRowBias(V, m_bv[b]); //--- RoPE the full Q and K per head at their own positions (posOffset 0) matrix Qr = RoPEAllHeads(Q, cosT, sinT, 0); matrix Kr = RoPEAllHeads(K, cosT, sinT, 0); //--- cache the post-RoPE K and raw V for this block m_cacheK[b] = Kr; m_cacheV[b] = V; //--- attention per head (causal), then out-proj + residual + FFN matrix ctx = matrix::Zeros((ulong)L, m_dm); for(int h = 0; h < m_heads; h++) { ulong c0 = (ulong)h * hd; matrix Qh = SliceCols(Qr, c0, hd); matrix Kh = SliceCols(Kr, c0, hd); matrix Vh = SliceCols(V, c0, hd); matrix Oh = SDPA(Qh, Kh, Vh, true); // causal, matches MHA WriteCols(ctx, Oh, c0); } matrix a = LinearT(ctx, m_Wo[b]); AddRowBias(a, m_bo[b]); matrix x1 = x + a; matrix n2 = RMSNorm(x1, m_n2[b]); matrix f = SwiGLU(n2, m_W1[b], m_W3[b], m_W2[b]); x = x1 + f; } m_cacheLen = L; m_cacheValid = true; context = RMSNorm(x, m_normW); m_cacheContext = context; // full context for decode_s2 s1_logits = LinearT(context, m_projS1W); AddRowBias(s1_logits, m_projS1B); return true; } //+---------------------------------------------------------------+ //| One cached AR step (grow phase only). Embeds the single new | //| token row, threads it up through the 8 blocks reusing the | //| cached K/V, appends its K/V at absolute position m_cacheLen, | //| and returns the new row's s1_logits (1,1024) and context_row | //| (1,d_model). Requires m_cacheValid; the caller must keep the | //| sequence in the grow phase (cacheLen < max_context). | //+---------------------------------------------------------------+ bool DecodeS1Step(int s1_id, int s2_id, const vector &stamp_row, matrix &s1_logits, matrix &context_row) { if(!m_cacheValid) { Print("DecodeS1Step: cache not primed"); return false; } //--- embed the single new row (1, d_model) int s1a[], s2a[]; ArrayResize(s1a, 1); s1a[0] = s1_id; ArrayResize(s2a, 1); s2a[0] = s2_id; matrix stamp1 = matrix::Zeros(1, 5); for(int j = 0; j < 5; j++) stamp1[0][j] = stamp_row[j]; matrix x; EmbedRows(s1a, s2a, stamp1, x); // (1, d_model) int pos = m_cacheLen; // absolute position of the new row ulong hd = m_dm / (ulong)m_heads; //--- RoPE table sized to cover position `pos` (rows 0..pos), query reads row pos matrix cosT, sinT; RoPETables((ulong)(pos + 1), hd, cosT, sinT); for(int b = 0; b < m_blocks; b++) { matrix n1 = RMSNorm(x, m_n1[b]); // (1, d_model) matrix Q = LinearT(n1, m_Wq[b]); AddRowBias(Q, m_bq[b]); matrix K = LinearT(n1, m_Wk[b]); AddRowBias(K, m_bk[b]); matrix V = LinearT(n1, m_Wv[b]); AddRowBias(V, m_bv[b]); //--- rotate the single new row at its absolute position `pos` matrix Qr = RoPEAllHeads(Q, cosT, sinT, pos); matrix Kr = RoPEAllHeads(K, cosT, sinT, pos); //--- append new K/V to the cache for this block -> (pos+1, d_model) matrix Kall = AppendRow(m_cacheK[b], Kr); matrix Vall = AppendRow(m_cacheV[b], V); m_cacheK[b] = Kall; m_cacheV[b] = Vall; //--- attention: single query (row pos) over all cached keys (non-causal: //--- the new row is the latest position, so it attends to everything) matrix ctx = matrix::Zeros(1, m_dm); for(int h = 0; h < m_heads; h++) { ulong c0 = (ulong)h * hd; matrix Qh = SliceCols(Qr, c0, hd); // (1, hd) matrix Kh = SliceCols(Kall, c0, hd); // (pos+1, hd) matrix Vh = SliceCols(Vall, c0, hd); matrix Oh = SDPA(Qh, Kh, Vh, false); // (1, hd) WriteCols(ctx, Oh, c0); } matrix a = LinearT(ctx, m_Wo[b]); AddRowBias(a, m_bo[b]); matrix x1 = x + a; matrix n2 = RMSNorm(x1, m_n2[b]); matrix f = SwiGLU(n2, m_W1[b], m_W3[b], m_W2[b]); x = x1 + f; } m_cacheLen = pos + 1; context_row = RMSNorm(x, m_normW); m_cacheContext = AppendRow(m_cacheContext, context_row); // grow full context s1_logits = LinearT(context_row, m_projS1W); AddRowBias(s1_logits, m_projS1B); return true; } private: //+---------------------------------------------------------------+ //| Return M with one extra row `row` (1, cols) appended at the | //| bottom. M may have 0 rows. | //+---------------------------------------------------------------+ matrix AppendRow(const matrix &M, const matrix &row) { ulong T = M.Rows(), C = M.Cols() > 0 ? M.Cols() : row.Cols(); matrix outM = matrix::Zeros(T + 1, C); for(ulong r = 0; r < T; r++) for(ulong c = 0; c < C; c++) outM[r][c] = M[r][c]; for(ulong c = 0; c < C; c++) outM[T][c] = row[0][c]; return outM; } }; #endif // KRONOS_PREDICTOR_S1_MQH //+------------------------------------------------------------------+