//+------------------------------------------------------------------+ //| KronosVerifyKVCache.mq5 | //| MMQ — Muhammad Minhas Qamar | //| www.mql5.com | //+------------------------------------------------------------------+ //| Development scaffold for the decode_s1 KV-cache (grow phase). | //| For the first few AR steps it runs BOTH paths and asserts they | //| agree to floating-point round-off: | //| - cached: PrimeCache once, then DecodeS1Step per new token | //| - full: DecodeS1 over the growing window each step | //| The last-row s1_logits must match to ~1e-9 (different summation | //| order only). This localizes any RoPE-position, append-order, or | //| block-threading bug at its source, before the integer-match | //| harnesses (VerifyInference / VerifySlide). Greedy picks keep the | //| generated token sequence deterministic across both paths. | //| | //| Remove this scaffold once green (the full DecodeS1 stays in the | //| library as the slide-phase path and reference math). | //| | //| Placement: kronos_refs/{x_norm,x_stamp,y_stamp}.bin and | //| kronos_weights/{tokenizer,predictor}/*.bin | //+------------------------------------------------------------------+ #property copyright "MMQ — Muhammad Minhas Qamar" #property link "https://www.mql5.com" #property version "1.00" #property script_show_inputs #property strict #include "KronosInference.mqh" #define KR_TOK_DIR "kronos_weights\\tokenizer\\" #define KR_TOK_DM 256 #define KR_TOK_HEADS 4 #define KR_TOK_ENC 4 #define KR_TOK_DEC 4 #define KR_TOK_FF 512 #define KR_PRED_DIR "kronos_weights\\predictor\\" #define KR_PRED_DM 512 #define KR_PRED_HEADS 8 #define KR_PRED_LAYERS 8 #define KR_PRED_FF 1024 input string InpRefDir = "kronos_refs\\"; input int InpLookback = 256; input int InpSteps = 6; // AR steps to cross-check input double InpTol = 1e-7; // max abs logit delta tolerated //+------------------------------------------------------------------+ //| Read a [rows,cols] float32 .bin into a matrix(double). | //+------------------------------------------------------------------+ bool LoadF32Matrix(const string fname, ulong rows, ulong cols, matrix &out) { int h = FileOpen(fname, FILE_READ | FILE_BIN); if(h == INVALID_HANDLE) { PrintFormat("open fail %s (%d)", fname, GetLastError()); return false; } ulong n = rows * cols; float buf[]; ArrayResize(buf, (int)n); uint got = FileReadArray(h, buf, 0, (int)n); FileClose(h); if(got != n) { PrintFormat("short read %s", fname); return false; } out = matrix::Zeros(rows, cols); for(ulong r = 0; r < rows; r++) for(ulong c = 0; c < cols; c++) out[r][c] = (double)buf[r*cols+c]; return true; } //+------------------------------------------------------------------+ //| Greedy argmax of a logits row (matches SampleFromLogits greedy). | //+------------------------------------------------------------------+ int ArgmaxRow(const matrix &logits, int row) { int best = 0; double bv = logits[row][0]; for(int j = 1; j < (int)logits.Cols(); j++) if(logits[row][j] > bv) { bv = logits[row][j]; best = j; } return best; } //+------------------------------------------------------------------+ //| Max abs delta between the last row of two logit matrices. | //+------------------------------------------------------------------+ double LastRowMaxDelta(const matrix &A, int ra, const matrix &B, int rb) { double m = 0.0; for(int j = 0; j < (int)A.Cols(); j++) { double d = MathAbs(A[ra][j] - B[rb][j]); if(d > m) m = d; } return m; } //+------------------------------------------------------------------+ //| Script entry point. | //+------------------------------------------------------------------+ void OnStart() { Print("============ Kronos KV-cache cross-check (cached vs full decode_s1) ============"); const int L = InpLookback; matrix x_norm, x_stamp, y_stamp; if(!LoadF32Matrix(InpRefDir + "x_norm.bin", (ulong)L, KR_NFEAT, x_norm)) { Print("ABORT x_norm"); return; } if(!LoadF32Matrix(InpRefDir + "x_stamp.bin", (ulong)L, 5, x_stamp)) { Print("ABORT x_stamp"); return; } if(!LoadF32Matrix(InpRefDir + "y_stamp.bin", (ulong)InpSteps, 5, y_stamp)){ Print("ABORT y_stamp"); return; } matrix full_stamp = matrix::Zeros((ulong)(L + InpSteps), 5); for(int i = 0; i < L; i++) for(int j = 0; j < 5; j++) full_stamp[i][j] = x_stamp[i][j]; for(int i = 0; i < InpSteps; i++) for(int j = 0; j < 5; j++) full_stamp[L + i][j] = y_stamp[i][j]; // We need access to the predictor/encoder stages directly. CKronosModel // wraps them privately, so build the encoder + predictor S1/S2 here. CKronosEncoder enc; CKronosPredictorS1 p1; CKronosPredictorS2 p2; bool ok = true; ok &= enc.Init(KR_TOK_DIR, KR_TOK_ENC, KR_TOK_DM, KR_TOK_HEADS, KR_TOK_FF); ok &= p1.Init(KR_PRED_DIR, KR_PRED_LAYERS, KR_PRED_DM, KR_PRED_HEADS, KR_PRED_FF); ok &= p2.Init(KR_PRED_DIR, KR_PRED_DM); if(!ok) { Print("ABORT: component init failed"); return; } Print("Components loaded (encoder, predictor s1/s2)."); // encode context -> base tokens int base_s1[], base_s2[]; if(!enc.Encode(x_norm, base_s1, base_s2)) { Print("ABORT: encode failed"); return; } // ---- prime the cache over the L base tokens ---- int win0_s1[], win0_s2[]; ArrayResize(win0_s1, L); ArrayResize(win0_s2, L); for(int t = 0; t < L; t++) { win0_s1[t] = base_s1[t]; win0_s2[t] = base_s2[t]; } matrix stamp0 = matrix::Zeros((ulong)L, 5); for(int t = 0; t < L; t++) for(int j = 0; j < 5; j++) stamp0[t][j] = full_stamp[t][j]; matrix prime_logits, prime_ctx; if(!p1.PrimeCache(win0_s1, win0_s2, stamp0, prime_logits, prime_ctx)) { Print("ABORT: PrimeCache failed"); return; } // running token buffers for the FULL-path window int pre[], post[]; ArrayCopy(pre, base_s1); ArrayCopy(post, base_s2); double worst = 0.0; int fails = 0; for(int i = 0; i < InpSteps; i++) { int seqlen = L + i; // window length for the full path this step // ----- cached path: step-0 reuses primed logits, else DecodeS1Step ----- matrix cached_logits, cached_ctx; int cl_row; if(i == 0) { cached_logits = prime_logits; p1.GetContext(cached_ctx); cl_row = (int)cached_logits.Rows() - 1; } else { vector srow = vector::Zeros(5); for(int j = 0; j < 5; j++) srow[j] = full_stamp[L + i - 1][j]; if(!p1.DecodeS1Step(pre[ArraySize(pre)-1], post[ArraySize(post)-1], srow, cached_logits, cached_ctx)) { Print("ABORT: DecodeS1Step failed"); return; } cl_row = 0; } // ----- full path: DecodeS1 over the current window ----- int win_s1[], win_s2[]; ArrayResize(win_s1, seqlen); ArrayResize(win_s2, seqlen); for(int t = 0; t < seqlen; t++) { win_s1[t] = pre[t]; win_s2[t] = post[t]; } matrix stamp_win = matrix::Zeros((ulong)seqlen, 5); for(int t = 0; t < seqlen; t++) for(int j = 0; j < 5; j++) stamp_win[t][j] = full_stamp[t][j]; matrix full_logits, full_ctx; if(!p1.DecodeS1(win_s1, win_s2, stamp_win, full_logits, full_ctx)) { Print("ABORT: DecodeS1 failed"); return; } int fl_row = seqlen - 1; // ----- compare last-row s1 logits ----- double d = LastRowMaxDelta(cached_logits, cl_row, full_logits, fl_row); if(d > worst) worst = d; int a_cached = ArgmaxRow(cached_logits, cl_row); int a_full = ArgmaxRow(full_logits, fl_row); bool argok = (a_cached == a_full); bool tolok = (d <= InpTol); if(!argok || !tolok) fails++; PrintFormat("step %2d: max|dlogit|=%.3e argmax cached=%d full=%d %s", i, d, a_cached, a_full, (argok && tolok) ? "OK" : "FAIL"); // advance the FULL-path buffer by the greedy s1 pick (deterministic). // s2 pick irrelevant to decode_s1, but we must append a token; use the // s2 argmax conditioned via decode_s2 over the cached full context so // both paths consume identical tokens. int s1_pick = a_full; // greedy int pick_arr[]; ArrayResize(pick_arr, 1); pick_arr[0] = s1_pick; matrix s2_logits; if(!p2.DecodeS2(full_ctx, pick_arr, s2_logits)) { Print("ABORT: DecodeS2"); return; } int s2_pick = ArgmaxRow(s2_logits, (int)s2_logits.Rows() - 1); int n = ArraySize(pre); ArrayResize(pre, n + 1); pre[n] = s1_pick; ArrayResize(post, n + 1); post[n] = s2_pick; } PrintFormat("worst max|dlogit| over %d steps = %.3e (tol %.1e)", InpSteps, worst, InpTol); if(fails == 0) Print(">>> KV-CACHE CROSS-CHECK PASSED: cached decode_s1 == full path within tolerance. <<<"); else PrintFormat(">>> KV-CACHE CROSS-CHECK FAILED: %d step(s) diverged. <<<", fails); Print("================================================================================"); } //+------------------------------------------------------------------+