219 lines
9.4 KiB
MQL5
219 lines
9.4 KiB
MQL5
|
//+------------------------------------------------------------------+
|
||
|
|
//| KronosVerifyKVCache.mq5 |
|
||
|
|
//| MMQ — Muhammad Minhas Qamar |
|
||
|
|
//| www.mql5.com |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Development scaffold for the decode_s1 KV-cache (grow phase). |
|
||
|
|
//| For the first few AR steps it runs BOTH paths and asserts they |
|
||
|
|
//| agree to floating-point round-off: |
|
||
|
|
//| - cached: PrimeCache once, then DecodeS1Step per new token |
|
||
|
|
//| - full: DecodeS1 over the growing window each step |
|
||
|
|
//| The last-row s1_logits must match to ~1e-9 (different summation |
|
||
|
|
//| order only). This localizes any RoPE-position, append-order, or |
|
||
|
|
//| block-threading bug at its source, before the integer-match |
|
||
|
|
//| harnesses (VerifyInference / VerifySlide). Greedy picks keep the |
|
||
|
|
//| generated token sequence deterministic across both paths. |
|
||
|
|
//| |
|
||
|
|
//| Remove this scaffold once green (the full DecodeS1 stays in the |
|
||
|
|
//| library as the slide-phase path and reference math). |
|
||
|
|
//| |
|
||
|
|
//| Placement: kronos_refs/{x_norm,x_stamp,y_stamp}.bin and |
|
||
|
|
//| kronos_weights/{tokenizer,predictor}/*.bin |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
#property copyright "MMQ — Muhammad Minhas Qamar"
|
||
|
|
#property link "https://www.mql5.com"
|
||
|
|
#property version "1.00"
|
||
|
|
#property script_show_inputs
|
||
|
|
#property strict
|
||
|
|
|
||
|
|
#include "KronosInference.mqh"
|
||
|
|
|
||
|
|
#define KR_TOK_DIR "kronos_weights\\tokenizer\\"
|
||
|
|
#define KR_TOK_DM 256
|
||
|
|
#define KR_TOK_HEADS 4
|
||
|
|
#define KR_TOK_ENC 4
|
||
|
|
#define KR_TOK_DEC 4
|
||
|
|
#define KR_TOK_FF 512
|
||
|
|
#define KR_PRED_DIR "kronos_weights\\predictor\\"
|
||
|
|
#define KR_PRED_DM 512
|
||
|
|
#define KR_PRED_HEADS 8
|
||
|
|
#define KR_PRED_LAYERS 8
|
||
|
|
#define KR_PRED_FF 1024
|
||
|
|
|
||
|
|
input string InpRefDir = "kronos_refs\\";
|
||
|
|
input int InpLookback = 256;
|
||
|
|
input int InpSteps = 6; // AR steps to cross-check
|
||
|
|
input double InpTol = 1e-7; // max abs logit delta tolerated
|
||
|
|
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Read a [rows,cols] float32 .bin into a matrix(double). |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
bool LoadF32Matrix(const string fname, ulong rows, ulong cols, matrix &out)
|
||
|
|
{
|
||
|
|
int h = FileOpen(fname, FILE_READ | FILE_BIN);
|
||
|
|
if(h == INVALID_HANDLE)
|
||
|
|
{ PrintFormat("open fail %s (%d)", fname, GetLastError()); return false; }
|
||
|
|
ulong n = rows * cols;
|
||
|
|
float buf[];
|
||
|
|
ArrayResize(buf, (int)n);
|
||
|
|
uint got = FileReadArray(h, buf, 0, (int)n);
|
||
|
|
FileClose(h);
|
||
|
|
if(got != n)
|
||
|
|
{ PrintFormat("short read %s", fname); return false; }
|
||
|
|
out = matrix::Zeros(rows, cols);
|
||
|
|
for(ulong r = 0; r < rows; r++)
|
||
|
|
for(ulong c = 0; c < cols; c++)
|
||
|
|
out[r][c] = (double)buf[r*cols+c];
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Greedy argmax of a logits row (matches SampleFromLogits greedy). |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
int ArgmaxRow(const matrix &logits, int row)
|
||
|
|
{
|
||
|
|
int best = 0;
|
||
|
|
double bv = logits[row][0];
|
||
|
|
for(int j = 1; j < (int)logits.Cols(); j++)
|
||
|
|
if(logits[row][j] > bv) { bv = logits[row][j]; best = j; }
|
||
|
|
return best;
|
||
|
|
}
|
||
|
|
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Max abs delta between the last row of two logit matrices. |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
double LastRowMaxDelta(const matrix &A, int ra, const matrix &B, int rb)
|
||
|
|
{
|
||
|
|
double m = 0.0;
|
||
|
|
for(int j = 0; j < (int)A.Cols(); j++)
|
||
|
|
{
|
||
|
|
double d = MathAbs(A[ra][j] - B[rb][j]);
|
||
|
|
if(d > m) m = d;
|
||
|
|
}
|
||
|
|
return m;
|
||
|
|
}
|
||
|
|
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
//| Script entry point. |
|
||
|
|
//+------------------------------------------------------------------+
|
||
|
|
void OnStart()
|
||
|
|
{
|
||
|
|
Print("============ Kronos KV-cache cross-check (cached vs full decode_s1) ============");
|
||
|
|
const int L = InpLookback;
|
||
|
|
|
||
|
|
matrix x_norm, x_stamp, y_stamp;
|
||
|
|
if(!LoadF32Matrix(InpRefDir + "x_norm.bin", (ulong)L, KR_NFEAT, x_norm)) { Print("ABORT x_norm"); return; }
|
||
|
|
if(!LoadF32Matrix(InpRefDir + "x_stamp.bin", (ulong)L, 5, x_stamp)) { Print("ABORT x_stamp"); return; }
|
||
|
|
if(!LoadF32Matrix(InpRefDir + "y_stamp.bin", (ulong)InpSteps, 5, y_stamp)){ Print("ABORT y_stamp"); return; }
|
||
|
|
|
||
|
|
matrix full_stamp = matrix::Zeros((ulong)(L + InpSteps), 5);
|
||
|
|
for(int i = 0; i < L; i++)
|
||
|
|
for(int j = 0; j < 5; j++)
|
||
|
|
full_stamp[i][j] = x_stamp[i][j];
|
||
|
|
for(int i = 0; i < InpSteps; i++)
|
||
|
|
for(int j = 0; j < 5; j++)
|
||
|
|
full_stamp[L + i][j] = y_stamp[i][j];
|
||
|
|
|
||
|
|
// We need access to the predictor/encoder stages directly. CKronosModel
|
||
|
|
// wraps them privately, so build the encoder + predictor S1/S2 here.
|
||
|
|
CKronosEncoder enc;
|
||
|
|
CKronosPredictorS1 p1;
|
||
|
|
CKronosPredictorS2 p2;
|
||
|
|
bool ok = true;
|
||
|
|
ok &= enc.Init(KR_TOK_DIR, KR_TOK_ENC, KR_TOK_DM, KR_TOK_HEADS, KR_TOK_FF);
|
||
|
|
ok &= p1.Init(KR_PRED_DIR, KR_PRED_LAYERS, KR_PRED_DM, KR_PRED_HEADS, KR_PRED_FF);
|
||
|
|
ok &= p2.Init(KR_PRED_DIR, KR_PRED_DM);
|
||
|
|
if(!ok) { Print("ABORT: component init failed"); return; }
|
||
|
|
Print("Components loaded (encoder, predictor s1/s2).");
|
||
|
|
|
||
|
|
// encode context -> base tokens
|
||
|
|
int base_s1[], base_s2[];
|
||
|
|
if(!enc.Encode(x_norm, base_s1, base_s2)) { Print("ABORT: encode failed"); return; }
|
||
|
|
|
||
|
|
// ---- prime the cache over the L base tokens ----
|
||
|
|
int win0_s1[], win0_s2[];
|
||
|
|
ArrayResize(win0_s1, L); ArrayResize(win0_s2, L);
|
||
|
|
for(int t = 0; t < L; t++) { win0_s1[t] = base_s1[t]; win0_s2[t] = base_s2[t]; }
|
||
|
|
matrix stamp0 = matrix::Zeros((ulong)L, 5);
|
||
|
|
for(int t = 0; t < L; t++) for(int j = 0; j < 5; j++) stamp0[t][j] = full_stamp[t][j];
|
||
|
|
|
||
|
|
matrix prime_logits, prime_ctx;
|
||
|
|
if(!p1.PrimeCache(win0_s1, win0_s2, stamp0, prime_logits, prime_ctx))
|
||
|
|
{ Print("ABORT: PrimeCache failed"); return; }
|
||
|
|
|
||
|
|
// running token buffers for the FULL-path window
|
||
|
|
int pre[], post[];
|
||
|
|
ArrayCopy(pre, base_s1); ArrayCopy(post, base_s2);
|
||
|
|
|
||
|
|
double worst = 0.0;
|
||
|
|
int fails = 0;
|
||
|
|
|
||
|
|
for(int i = 0; i < InpSteps; i++)
|
||
|
|
{
|
||
|
|
int seqlen = L + i; // window length for the full path this step
|
||
|
|
|
||
|
|
// ----- cached path: step-0 reuses primed logits, else DecodeS1Step -----
|
||
|
|
matrix cached_logits, cached_ctx;
|
||
|
|
int cl_row;
|
||
|
|
if(i == 0)
|
||
|
|
{
|
||
|
|
cached_logits = prime_logits;
|
||
|
|
p1.GetContext(cached_ctx);
|
||
|
|
cl_row = (int)cached_logits.Rows() - 1;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
vector srow = vector::Zeros(5);
|
||
|
|
for(int j = 0; j < 5; j++) srow[j] = full_stamp[L + i - 1][j];
|
||
|
|
if(!p1.DecodeS1Step(pre[ArraySize(pre)-1], post[ArraySize(post)-1], srow, cached_logits, cached_ctx))
|
||
|
|
{ Print("ABORT: DecodeS1Step failed"); return; }
|
||
|
|
cl_row = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
// ----- full path: DecodeS1 over the current window -----
|
||
|
|
int win_s1[], win_s2[];
|
||
|
|
ArrayResize(win_s1, seqlen); ArrayResize(win_s2, seqlen);
|
||
|
|
for(int t = 0; t < seqlen; t++) { win_s1[t] = pre[t]; win_s2[t] = post[t]; }
|
||
|
|
matrix stamp_win = matrix::Zeros((ulong)seqlen, 5);
|
||
|
|
for(int t = 0; t < seqlen; t++) for(int j = 0; j < 5; j++) stamp_win[t][j] = full_stamp[t][j];
|
||
|
|
|
||
|
|
matrix full_logits, full_ctx;
|
||
|
|
if(!p1.DecodeS1(win_s1, win_s2, stamp_win, full_logits, full_ctx))
|
||
|
|
{ Print("ABORT: DecodeS1 failed"); return; }
|
||
|
|
int fl_row = seqlen - 1;
|
||
|
|
|
||
|
|
// ----- compare last-row s1 logits -----
|
||
|
|
double d = LastRowMaxDelta(cached_logits, cl_row, full_logits, fl_row);
|
||
|
|
if(d > worst) worst = d;
|
||
|
|
int a_cached = ArgmaxRow(cached_logits, cl_row);
|
||
|
|
int a_full = ArgmaxRow(full_logits, fl_row);
|
||
|
|
bool argok = (a_cached == a_full);
|
||
|
|
bool tolok = (d <= InpTol);
|
||
|
|
if(!argok || !tolok) fails++;
|
||
|
|
PrintFormat("step %2d: max|dlogit|=%.3e argmax cached=%d full=%d %s",
|
||
|
|
i, d, a_cached, a_full,
|
||
|
|
(argok && tolok) ? "OK" : "FAIL");
|
||
|
|
|
||
|
|
// advance the FULL-path buffer by the greedy s1 pick (deterministic).
|
||
|
|
// s2 pick irrelevant to decode_s1, but we must append a token; use the
|
||
|
|
// s2 argmax conditioned via decode_s2 over the cached full context so
|
||
|
|
// both paths consume identical tokens.
|
||
|
|
int s1_pick = a_full; // greedy
|
||
|
|
int pick_arr[]; ArrayResize(pick_arr, 1); pick_arr[0] = s1_pick;
|
||
|
|
matrix s2_logits;
|
||
|
|
if(!p2.DecodeS2(full_ctx, pick_arr, s2_logits)) { Print("ABORT: DecodeS2"); return; }
|
||
|
|
int s2_pick = ArgmaxRow(s2_logits, (int)s2_logits.Rows() - 1);
|
||
|
|
|
||
|
|
int n = ArraySize(pre);
|
||
|
|
ArrayResize(pre, n + 1); pre[n] = s1_pick;
|
||
|
|
ArrayResize(post, n + 1); post[n] = s2_pick;
|
||
|
|
}
|
||
|
|
|
||
|
|
PrintFormat("worst max|dlogit| over %d steps = %.3e (tol %.1e)", InpSteps, worst, InpTol);
|
||
|
|
if(fails == 0)
|
||
|
|
Print(">>> KV-CACHE CROSS-CHECK PASSED: cached decode_s1 == full path within tolerance. <<<");
|
||
|
|
else
|
||
|
|
PrintFormat(">>> KV-CACHE CROSS-CHECK FAILED: %d step(s) diverged. <<<", fails);
|
||
|
|
Print("================================================================================");
|
||
|
|
}
|
||
|
|
//+------------------------------------------------------------------+
|