kronos-mql5/Include/Kronos/KronosPredictorS1.mqh
2026-07-05 05:03:49 +00:00

472 lines
21 KiB
MQL5

//+------------------------------------------------------------------+
//| KronosPredictorS1.mqh |
//| MMQ — Muhammad Minhas Qamar |
//| www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "MMQ — Muhammad Minhas Qamar"
#property link "https://www.mql5.com"
#property version "1.00"
#ifndef KRONOS_PREDICTOR_S1_MQH
#define KRONOS_PREDICTOR_S1_MQH
#include <Kronos\KronosTokenizerMath.mqh> // loaders, KR_S1_BITS/KR_S2_BITS
#include <Kronos\KronosTransformerCore.mqh> // LinearT, AddRowBias, RMSNorm, TransformerBlock
#define KR_PRED_VOCAB 1024 // 2^10, s1 and s2 vocab
//+------------------------------------------------------------------+
//| Predictor, decode_s1 stage. |
//+------------------------------------------------------------------+
class CKronosPredictorS1
{
protected:
int m_blocks; // n_layers (8) -- predictor uses the FULL count, no n-1
int m_heads; // 8
ulong m_dm; // 512
ulong m_ff; // 1024
string m_dir;
//--- embeddings
matrix m_embS1; // emb_s1.weight : [vocab, d_model]
matrix m_embS2; // emb_s2.weight : [vocab, d_model]
matrix m_fusionW;
vector m_fusionB; // fusion_proj : [d_model, 2*d_model]
matrix m_teMin, m_teHour, m_teWday, m_teDay, m_teMon; // [size, d_model] each
//--- 8 transformer blocks
vector m_n1[];
matrix m_Wq[];
vector m_bq[];
matrix m_Wk[];
vector m_bk[];
matrix m_Wv[];
vector m_bv[];
matrix m_Wo[];
vector m_bo[];
vector m_n2[];
matrix m_W1[];
matrix m_W3[];
matrix m_W2[];
//--- final norm + s1 head
vector m_normW; // norm.weight : [d_model]
matrix m_projS1W;
vector m_projS1B; // head.proj_s1 : [vocab, d_model]
//--- KV-cache (grow-phase only)
//--- Per block: the projected K (already RoPE-rotated, heads concatenated) and
//--- raw V over all cached positions, each (m_cacheLen, d_model). In the grow
//--- phase a token's absolute position never changes, so a K rotated once at
//--- write-time stays valid; only the new last row is computed per step.
matrix m_cacheK[]; // [m_blocks] of (T, d_model)
matrix m_cacheV[]; // [m_blocks] of (T, d_model)
matrix m_cacheContext; // (T, d_model) post-final-norm context
// for every position; decode_s2 cross-
// attn keys/values run over ALL of it
int m_cacheLen; // number of cached positions
bool m_cacheValid; // guard: primed and grow-phase only
string F(const string name) { return m_dir + name + ".bin"; }
//+---------------------------------------------------------------+
//| Apply RoPE per head over a (T, d_model) projection, using a |
//| cos/sin table of width head_dim. Rotates each head's column |
//| slice independently, matching MHA's per-head ApplyRoPE. |
//| posOffset shifts the row->position mapping (row r uses |
//| position posOffset+r) so a single appended row can be rotated |
//| at its true absolute position. |
//+---------------------------------------------------------------+
matrix RoPEAllHeads(const matrix &M, const matrix &cosT, const matrix &sinT, int posOffset)
{
ulong T = M.Rows();
ulong hd = m_dm / (ulong)m_heads;
matrix outM = matrix::Zeros(T, m_dm);
for(int h = 0; h < m_heads; h++)
{
ulong c0 = (ulong)h * hd;
matrix Ms = SliceCols(M, c0, hd);
//--- rotate slice; cos/sin row for output row r comes from posOffset+r
matrix Mr = matrix::Zeros(T, hd);
ulong half = hd / 2;
for(ulong r = 0; r < T; r++)
{
ulong p = (ulong)posOffset + r;
for(ulong k = 0; k < hd; k++)
{
double rh = (k < half) ? -Ms[r][k + half] : Ms[r][k - half];
Mr[r][k] = Ms[r][k] * cosT[p][k] + rh * sinT[p][k];
}
}
WriteCols(outM, Mr, c0);
}
return outM;
}
//+---------------------------------------------------------------+
//| Embedding lookup with scaling: row `id` of the table times |
//| the given scale (sqrt(d_model) for the hierarchical embed). |
//+---------------------------------------------------------------+
void EmbedTokens(const int &ids[], const matrix &table, double scale, matrix &out)
{
int L = ArraySize(ids);
out = matrix::Zeros((ulong)L, m_dm);
for(int t = 0; t < L; t++)
{
ulong id = (ulong)ids[t];
for(ulong j = 0; j < m_dm; j++)
out[t][j] = table[id][j] * scale;
}
}
//+---------------------------------------------------------------+
//| Add one temporal table's contribution: for each row t, add |
//| table row stamp[t][col] to x. |
//+---------------------------------------------------------------+
void AddTimeEmbed(const matrix &stamp, int col, const matrix &table, matrix &x)
{
ulong L = x.Rows();
for(ulong t = 0; t < L; t++)
{
ulong idx = (ulong)MathRound(stamp[t][col]); // stamp stored as float
for(ulong j = 0; j < m_dm; j++)
x[t][j] += table[idx][j];
}
}
//+---------------------------------------------------------------+
//| Build the transformer input rows for the given tokens/stamp: |
//| HierarchicalEmbedding( fusion_proj(cat(emb_s1*sqrt d, |
//| emb_s2*sqrt d)) ) + TemporalEmbedding(stamp). Shared by the |
//| full DecodeS1 and the cached prime/step paths so they embed |
//| identically. out is (L, d_model). |
//+---------------------------------------------------------------+
void EmbedRows(const int &s1_ids[], const int &s2_ids[], const matrix &stamp, matrix &out)
{
int L = ArraySize(s1_ids);
double scale = MathSqrt((double)m_dm);
matrix e1, e2;
EmbedTokens(s1_ids, m_embS1, scale, e1);
EmbedTokens(s2_ids, m_embS2, scale, e2);
matrix cat = matrix::Zeros((ulong)L, 2 * m_dm);
for(int t = 0; t < L; t++)
{
for(ulong j = 0; j < m_dm; j++)
cat[t][j] = e1[t][j];
for(ulong j = 0; j < m_dm; j++)
cat[t][m_dm + j] = e2[t][j];
}
out = LinearT(cat, m_fusionW);
AddRowBias(out, m_fusionB);
if(stamp.Rows() == (ulong)L && stamp.Cols() >= 5)
{
AddTimeEmbed(stamp, 0, m_teMin, out);
AddTimeEmbed(stamp, 1, m_teHour, out);
AddTimeEmbed(stamp, 2, m_teWday, out);
AddTimeEmbed(stamp, 3, m_teDay, out);
AddTimeEmbed(stamp, 4, m_teMon, out);
}
}
public:
//+---------------------------------------------------------------+
//| Load embeddings, the 8 transformer blocks, the final norm and |
//| the s1 head. The predictor uses the FULL n_layers, not n-1. |
//+---------------------------------------------------------------+
bool Init(const string weight_dir, int n_layers, ulong d_model, int n_heads, ulong ff_dim)
{
m_dir = weight_dir;
m_dm = d_model;
m_heads = n_heads;
m_ff = ff_dim;
m_blocks = n_layers; // predictor: full n_layers (NOT n-1)
if(d_model == 0 || n_heads == 0 || n_layers == 0 || ff_dim == 0)
{ Print("CKronosPredictorS1::Init: invalid config (zero dimension)"); return false; }
bool ok = true;
//--- embeddings
//--- emb_s1/emb_s2 are lookup tables (indexed by token id) -> keep row-major
ok &= KronosLoadMatrix(F("embedding_emb_s1_weight"), KR_PRED_VOCAB, m_dm, m_embS1);
ok &= KronosLoadMatrix(F("embedding_emb_s2_weight"), KR_PRED_VOCAB, m_dm, m_embS2);
//--- fusion_proj is a LinearT weight -> store transposed
ok &= KronosLoadMatrixT(F("embedding_fusion_proj_weight"), m_dm, 2 * m_dm, m_fusionW);
ok &= KronosLoadVector(F("embedding_fusion_proj_bias"), m_dm, m_fusionB);
ok &= KronosLoadMatrix(F("time_emb_minute_embed_weight"), 60, m_dm, m_teMin);
ok &= KronosLoadMatrix(F("time_emb_hour_embed_weight"), 24, m_dm, m_teHour);
ok &= KronosLoadMatrix(F("time_emb_weekday_embed_weight"), 7, m_dm, m_teWday);
ok &= KronosLoadMatrix(F("time_emb_day_embed_weight"), 32, m_dm, m_teDay);
ok &= KronosLoadMatrix(F("time_emb_month_embed_weight"), 13, m_dm, m_teMon);
//--- final norm + s1 head
ok &= KronosLoadVector(F("norm_weight"), m_dm, m_normW);
ok &= KronosLoadMatrixT(F("head_proj_s1_weight"), KR_PRED_VOCAB, m_dm, m_projS1W);
ok &= KronosLoadVector(F("head_proj_s1_bias"), KR_PRED_VOCAB, m_projS1B);
if(!ok)
{
Print("CKronosPredictorS1::Init: embedding/head load failed");
return false;
}
ArrayResize(m_n1, m_blocks);
ArrayResize(m_n2, m_blocks);
ArrayResize(m_Wq, m_blocks);
ArrayResize(m_bq, m_blocks);
ArrayResize(m_Wk, m_blocks);
ArrayResize(m_bk, m_blocks);
ArrayResize(m_Wv, m_blocks);
ArrayResize(m_bv, m_blocks);
ArrayResize(m_Wo, m_blocks);
ArrayResize(m_bo, m_blocks);
ArrayResize(m_W1, m_blocks);
ArrayResize(m_W3, m_blocks);
ArrayResize(m_W2, m_blocks);
for(int b = 0; b < m_blocks; b++)
{
string p = StringFormat("transformer_%d_", b);
ok &= KronosLoadVector(F(p + "norm1_weight"), m_dm, m_n1[b]);
ok &= KronosLoadMatrixT(F(p + "self_attn_q_proj_weight"), m_dm, m_dm, m_Wq[b]);
ok &= KronosLoadVector(F(p + "self_attn_q_proj_bias"), m_dm, m_bq[b]);
ok &= KronosLoadMatrixT(F(p + "self_attn_k_proj_weight"), m_dm, m_dm, m_Wk[b]);
ok &= KronosLoadVector(F(p + "self_attn_k_proj_bias"), m_dm, m_bk[b]);
ok &= KronosLoadMatrixT(F(p + "self_attn_v_proj_weight"), m_dm, m_dm, m_Wv[b]);
ok &= KronosLoadVector(F(p + "self_attn_v_proj_bias"), m_dm, m_bv[b]);
ok &= KronosLoadMatrixT(F(p + "self_attn_out_proj_weight"), m_dm, m_dm, m_Wo[b]);
ok &= KronosLoadVector(F(p + "self_attn_out_proj_bias"), m_dm, m_bo[b]);
ok &= KronosLoadVector(F(p + "norm2_weight"), m_dm, m_n2[b]);
ok &= KronosLoadMatrixT(F(p + "ffn_w1_weight"), m_ff, m_dm, m_W1[b]);
ok &= KronosLoadMatrixT(F(p + "ffn_w3_weight"), m_ff, m_dm, m_W3[b]);
ok &= KronosLoadMatrixT(F(p + "ffn_w2_weight"), m_dm, m_ff, m_W2[b]);
if(!ok)
{
PrintFormat("CKronosPredictorS1::Init failed at block %d", b);
return false;
}
}
return ok;
}
//+---------------------------------------------------------------+
//| decode_s1: returns s1_logits (L, 1024) and context |
//| (L, d_model). stamp is (L, 5) float with columns |
//| [minute,hour,weekday,day,month], weekday already in the |
//| pandas convention (Mon=0..Sun=6). |
//+---------------------------------------------------------------+
bool DecodeS1(const int &s1_ids[], const int &s2_ids[], const matrix &stamp,
matrix &s1_logits, matrix &context)
{
int L = ArraySize(s1_ids);
if(L == 0 || ArraySize(s2_ids) != L)
{ PrintFormat("DecodeS1: bad id lengths (%d / %d)", L, ArraySize(s2_ids)); return false; }
matrix x;
EmbedRows(s1_ids, s2_ids, stamp, x); // HierarchicalEmbedding + TemporalEmbedding
//--- 8 causal pre-norm transformer blocks
for(int b = 0; b < m_blocks; b++)
x = TransformerBlock(x, m_n1[b],
m_Wq[b], m_bq[b], m_Wk[b], m_bk[b],
m_Wv[b], m_bv[b], m_Wo[b], m_bo[b],
m_heads, m_n2[b], m_W1[b], m_W3[b], m_W2[b]);
//--- final RMSNorm -> this is "context" returned for decode_s2
context = RMSNorm(x, m_normW);
//--- s1 head: d_model -> 1024
s1_logits = LinearT(context, m_projS1W);
AddRowBias(s1_logits, m_projS1B);
return true;
}
//+---------------------------------------------------------------+
//| Drop the KV-cache. Call before priming a fresh AR path. |
//+---------------------------------------------------------------+
void ResetCache()
{
ArrayResize(m_cacheK, m_blocks);
ArrayResize(m_cacheV, m_blocks);
for(int b = 0; b < m_blocks; b++)
{
m_cacheK[b] = matrix::Zeros(0, m_dm);
m_cacheV[b] = matrix::Zeros(0, m_dm);
}
m_cacheContext = matrix::Zeros(0, m_dm);
m_cacheLen = 0;
m_cacheValid = false;
}
//+---------------------------------------------------------------+
//| The running full context (T, d_model) accumulated across the |
//| cached steps. decode_s2's cross-attention keys/values must |
//| span ALL positions, so the caller passes this (not just the |
//| last row) to DecodeS2. |
//+---------------------------------------------------------------+
void GetContext(matrix &out) const { out = m_cacheContext; }
//+---------------------------------------------------------------+
//| Prime the cache over the initial context window. Runs the |
//| same block stack as DecodeS1 but captures, per block, the |
//| post-RoPE K and the raw V for every position (the exact |
//| tensors MHA forms internally). Leaves the cache holding L |
//| positions so DecodeS1Step can extend it one row at a time. |
//| Returns s1_logits (L,1024) and context (L,d_model) so the |
//| caller can reuse them for the very first step's decode_s2 |
//| (identical to what the full first-call DecodeS1 would give). |
//+---------------------------------------------------------------+
bool PrimeCache(const int &s1_ids[], const int &s2_ids[], const matrix &stamp,
matrix &s1_logits, matrix &context)
{
int L = ArraySize(s1_ids);
if(L == 0 || ArraySize(s2_ids) != L)
{ PrintFormat("PrimeCache: bad id lengths (%d / %d)", L, ArraySize(s2_ids)); return false; }
ResetCache();
matrix x;
EmbedRows(s1_ids, s2_ids, stamp, x);
ulong hd = m_dm / (ulong)m_heads;
matrix cosT, sinT;
RoPETables((ulong)L, hd, cosT, sinT); // table over the L context positions
for(int b = 0; b < m_blocks; b++)
{
//--- pre-norm + projections (mirrors MHA on RMSNorm(x))
matrix n1 = RMSNorm(x, m_n1[b]);
matrix Q = LinearT(n1, m_Wq[b]);
AddRowBias(Q, m_bq[b]);
matrix K = LinearT(n1, m_Wk[b]);
AddRowBias(K, m_bk[b]);
matrix V = LinearT(n1, m_Wv[b]);
AddRowBias(V, m_bv[b]);
//--- RoPE the full Q and K per head at their own positions (posOffset 0)
matrix Qr = RoPEAllHeads(Q, cosT, sinT, 0);
matrix Kr = RoPEAllHeads(K, cosT, sinT, 0);
//--- cache the post-RoPE K and raw V for this block
m_cacheK[b] = Kr;
m_cacheV[b] = V;
//--- attention per head (causal), then out-proj + residual + FFN
matrix ctx = matrix::Zeros((ulong)L, m_dm);
for(int h = 0; h < m_heads; h++)
{
ulong c0 = (ulong)h * hd;
matrix Qh = SliceCols(Qr, c0, hd);
matrix Kh = SliceCols(Kr, c0, hd);
matrix Vh = SliceCols(V, c0, hd);
matrix Oh = SDPA(Qh, Kh, Vh, true); // causal, matches MHA
WriteCols(ctx, Oh, c0);
}
matrix a = LinearT(ctx, m_Wo[b]);
AddRowBias(a, m_bo[b]);
matrix x1 = x + a;
matrix n2 = RMSNorm(x1, m_n2[b]);
matrix f = SwiGLU(n2, m_W1[b], m_W3[b], m_W2[b]);
x = x1 + f;
}
m_cacheLen = L;
m_cacheValid = true;
context = RMSNorm(x, m_normW);
m_cacheContext = context; // full context for decode_s2
s1_logits = LinearT(context, m_projS1W);
AddRowBias(s1_logits, m_projS1B);
return true;
}
//+---------------------------------------------------------------+
//| One cached AR step (grow phase only). Embeds the single new |
//| token row, threads it up through the 8 blocks reusing the |
//| cached K/V, appends its K/V at absolute position m_cacheLen, |
//| and returns the new row's s1_logits (1,1024) and context_row |
//| (1,d_model). Requires m_cacheValid; the caller must keep the |
//| sequence in the grow phase (cacheLen < max_context). |
//+---------------------------------------------------------------+
bool DecodeS1Step(int s1_id, int s2_id, const vector &stamp_row,
matrix &s1_logits, matrix &context_row)
{
if(!m_cacheValid)
{ Print("DecodeS1Step: cache not primed"); return false; }
//--- embed the single new row (1, d_model)
int s1a[], s2a[];
ArrayResize(s1a, 1); s1a[0] = s1_id;
ArrayResize(s2a, 1); s2a[0] = s2_id;
matrix stamp1 = matrix::Zeros(1, 5);
for(int j = 0; j < 5; j++)
stamp1[0][j] = stamp_row[j];
matrix x;
EmbedRows(s1a, s2a, stamp1, x); // (1, d_model)
int pos = m_cacheLen; // absolute position of the new row
ulong hd = m_dm / (ulong)m_heads;
//--- RoPE table sized to cover position `pos` (rows 0..pos), query reads row pos
matrix cosT, sinT;
RoPETables((ulong)(pos + 1), hd, cosT, sinT);
for(int b = 0; b < m_blocks; b++)
{
matrix n1 = RMSNorm(x, m_n1[b]); // (1, d_model)
matrix Q = LinearT(n1, m_Wq[b]);
AddRowBias(Q, m_bq[b]);
matrix K = LinearT(n1, m_Wk[b]);
AddRowBias(K, m_bk[b]);
matrix V = LinearT(n1, m_Wv[b]);
AddRowBias(V, m_bv[b]);
//--- rotate the single new row at its absolute position `pos`
matrix Qr = RoPEAllHeads(Q, cosT, sinT, pos);
matrix Kr = RoPEAllHeads(K, cosT, sinT, pos);
//--- append new K/V to the cache for this block -> (pos+1, d_model)
matrix Kall = AppendRow(m_cacheK[b], Kr);
matrix Vall = AppendRow(m_cacheV[b], V);
m_cacheK[b] = Kall;
m_cacheV[b] = Vall;
//--- attention: single query (row pos) over all cached keys (non-causal:
//--- the new row is the latest position, so it attends to everything)
matrix ctx = matrix::Zeros(1, m_dm);
for(int h = 0; h < m_heads; h++)
{
ulong c0 = (ulong)h * hd;
matrix Qh = SliceCols(Qr, c0, hd); // (1, hd)
matrix Kh = SliceCols(Kall, c0, hd); // (pos+1, hd)
matrix Vh = SliceCols(Vall, c0, hd);
matrix Oh = SDPA(Qh, Kh, Vh, false); // (1, hd)
WriteCols(ctx, Oh, c0);
}
matrix a = LinearT(ctx, m_Wo[b]);
AddRowBias(a, m_bo[b]);
matrix x1 = x + a;
matrix n2 = RMSNorm(x1, m_n2[b]);
matrix f = SwiGLU(n2, m_W1[b], m_W3[b], m_W2[b]);
x = x1 + f;
}
m_cacheLen = pos + 1;
context_row = RMSNorm(x, m_normW);
m_cacheContext = AppendRow(m_cacheContext, context_row); // grow full context
s1_logits = LinearT(context_row, m_projS1W);
AddRowBias(s1_logits, m_projS1B);
return true;
}
private:
//+---------------------------------------------------------------+
//| Return M with one extra row `row` (1, cols) appended at the |
//| bottom. M may have 0 rows. |
//+---------------------------------------------------------------+
matrix AppendRow(const matrix &M, const matrix &row)
{
ulong T = M.Rows(), C = M.Cols() > 0 ? M.Cols() : row.Cols();
matrix outM = matrix::Zeros(T + 1, C);
for(ulong r = 0; r < T; r++)
for(ulong c = 0; c < C; c++)
outM[r][c] = M[r][c];
for(ulong c = 0; c < C; c++)
outM[T][c] = row[0][c];
return outM;
}
};
#endif // KRONOS_PREDICTOR_S1_MQH
//+------------------------------------------------------------------+