kronos-mql5/Include/Kronos/KronosTokenizerMath.mqh

//+------------------------------------------------------------------+
//|                                          KronosTokenizerMath.mqh |
//|                                      MMQ — Muhammad Minhas Qamar |
//|                                   www.mql5.com/en/articles/23304 |
//+------------------------------------------------------------------+
#property copyright "MMQ — Muhammad Minhas Qamar"
#property link      "https://www.mql5.com/en/articles/23304"
#property version   "1.00"

#ifndef KRONOS_TOKENIZER_MATH_MQH
#define KRONOS_TOKENIZER_MATH_MQH

//--- tokenizer math constants
#define KR_S1_BITS        10
#define KR_S2_BITS        10
#define KR_CODEBOOK_DIM   (KR_S1_BITS + KR_S2_BITS)   // 20
#define KR_NFEAT          6                            // open,high,low,close,volume,amount
#define KR_CLIP           5.0
#define KR_EPS            1e-5

//+------------------------------------------------------------------+
//| Per-column z-score over the lookback rows, then clip.            |
//| raw[L][6] in feature order open,high,low,close,volume,amount.    |
//| mean/std are kept for the inverse transform. Population std      |
//| (divide by L) to match numpy ddof=0.                             |
//+------------------------------------------------------------------+
void KronosNormalize(const matrix &raw, matrix &norm, vector &mean, vector &stdv)
  {
   ulong L = raw.Rows();
   ulong F = raw.Cols();
   mean = vector::Zeros(F);
   stdv = vector::Zeros(F);
//--- per-column mean
   for(ulong j = 0; j < F; j++)
     {
      double s = 0.0;
      for(ulong i = 0; i < L; i++)
         s += raw[i][j];
      mean[j] = s / (double)L;
     }
//--- per-column population std (ddof = 0, matches numpy)
   for(ulong j = 0; j < F; j++)
     {
      double s = 0.0;
      for(ulong i = 0; i < L; i++)
        {
         double d = raw[i][j] - mean[j];
         s += d * d;
        }
      stdv[j] = MathSqrt(s / (double)L);
     }
//--- z-score, then clip to +/-KR_CLIP
   norm = matrix::Zeros(L, F);
   for(ulong j = 0; j < F; j++)
     {
      double denom = stdv[j] + KR_EPS;
      for(ulong i = 0; i < L; i++)
        {
         double v = (raw[i][j] - mean[j]) / denom;
         if(v >  KR_CLIP)
            v =  KR_CLIP;
         if(v < -KR_CLIP)
            v = -KR_CLIP;
         norm[i][j] = v;
        }
     }
  }

//+------------------------------------------------------------------+
//| Inverse transform for the model output, using the SAME window    |
//| statistics produced by KronosNormalize.                          |
//+------------------------------------------------------------------+
void KronosDenormalize(const matrix &norm, const vector &mean, const vector &stdv, matrix &out)
  {
   ulong L = norm.Rows(), F = norm.Cols();
   out = matrix::Zeros(L, F);
   for(ulong j = 0; j < F; j++)
     {
      double denom = stdv[j] + KR_EPS;
      for(ulong i = 0; i < L; i++)
         out[i][j] = norm[i][j] * denom + mean[j];
     }
  }

//+------------------------------------------------------------------+
//| Timestamp features for one bar: [minute,hour,weekday,day,month]. |
//| Weekday is remapped from MQL5 (Sun=0..Sat=6) to the pandas       |
//| convention (Mon=0..Sun=6) the model was trained on.              |
//+------------------------------------------------------------------+
void KronosStamp(datetime t, int &out[])
  {
   MqlDateTime s;
   TimeToStruct(t, s);
   ArrayResize(out, 5);
   out[0] = s.min;
   out[1] = s.hour;
   out[2] = (s.day_of_week + 6) % 7;
   out[3] = s.day;
   out[4] = s.mon;
  }

//+------------------------------------------------------------------+
//| BSQ encode: the s1/s2 indices depend only on the sign of each of |
//| the 20 z components (the L2-norm and scale before quantization   |
//| preserve sign). bit_i = 1 iff z_i > 0; index = sum bit_i * 2^i   |
//| (LSB-first).                                                     |
//+------------------------------------------------------------------+
void KronosBSQ_SignsToIndices(const double &z[], int &s1_id, int &s2_id)
  {
   s1_id = 0;
   s2_id = 0;
//--- s1: first KR_S1_BITS components (LSB-first)
   for(int i = 0; i < KR_S1_BITS; i++)
      if(z[i]              > 0.0)
         s1_id |= (1 << i);
//--- s2: next KR_S2_BITS components (LSB-first)
   for(int i = 0; i < KR_S2_BITS; i++)
      if(z[KR_S1_BITS + i] > 0.0)
         s2_id |= (1 << i);
  }

//+------------------------------------------------------------------+
//| BSQ decode: rebuild the 20-dim bipolar code from the indices.    |
//| code_i = (bit_i * 2 - 1) / sqrt(20).                             |
//+------------------------------------------------------------------+
void KronosBSQ_IndicesToCode(int s1_id, int s2_id, double &code[])
  {
   ArrayResize(code, KR_CODEBOOK_DIM);
   double q = 1.0 / MathSqrt((double)KR_CODEBOOK_DIM);
//--- s1 bits -> first KR_S1_BITS code entries
   for(int i = 0; i < KR_S1_BITS; i++)
     {
      int b = (s1_id >> i) & 1;
      code[i]              = (b * 2 - 1) * q;
     }
//--- s2 bits -> next KR_S2_BITS code entries
   for(int i = 0; i < KR_S2_BITS; i++)
     {
      int b = (s2_id >> i) & 1;
      code[KR_S1_BITS + i] = (b * 2 - 1) * q;
     }
  }

//+------------------------------------------------------------------+
//| Load a [rows,cols] float32 .bin (row-major) into a matrix.       |
//| Weights are stored [out,in], the PyTorch nn.Linear convention.   |
//| Shared by the encoder and decoder, hence this lowest header.     |
//+------------------------------------------------------------------+
bool KronosLoadMatrix(const string fname, ulong rows, ulong cols, matrix &out)
  {
//--- open the binary weight file
   int h = FileOpen(fname, FILE_READ | FILE_BIN);
   if(h == INVALID_HANDLE)
     { PrintFormat("KronosLoadMatrix: cannot open %s (err %d)", fname, GetLastError()); return false; }
//--- read the raw float32 buffer
   ulong n = rows * cols;
   float buf[];
   ArrayResize(buf, (int)n);
   uint got = FileReadArray(h, buf, 0, (int)n);
   FileClose(h);
   if(got != n)
     { PrintFormat("KronosLoadMatrix: short read %s (%u of %I64u)", fname, got, n); return false; }
//--- copy into the matrix as-is
   out = matrix::Zeros(rows, cols);                 // stored [out,in], row-major
   for(ulong r = 0; r < rows; r++)
      for(ulong c = 0; c < cols; c++)
         out[r][c] = (double)buf[r * cols + c];
   return true;
  }

//+------------------------------------------------------------------+
//| Load a [rows,cols] float32 .bin (row-major) but store it         |
//| TRANSPOSED as [cols,rows]. The file is still the PyTorch         |
//| nn.Linear [out,in] weight; storing W^T lets LinearT compute      |
//| y = X @ W^T as a plain X.MatMul(stored) with NO per-call         |
//| transpose. Profiling showed matrix::Transpose() was ~60% of the  |
//| forward pass because LinearT re-transposed constant weights on   |
//| every call; doing it once here (free during the copy) removes it.|
//| Use this for every LinearT weight; keep KronosLoadMatrix for     |
//| lookup tables (embeddings, temporal) that are indexed by row.    |
//+------------------------------------------------------------------+
bool KronosLoadMatrixT(const string fname, ulong rows, ulong cols, matrix &out)
  {
//--- open the binary weight file
   int h = FileOpen(fname, FILE_READ | FILE_BIN);
   if(h == INVALID_HANDLE)
     { PrintFormat("KronosLoadMatrixT: cannot open %s (err %d)", fname, GetLastError()); return false; }
//--- read the raw float32 buffer
   ulong n = rows * cols;
   float buf[];
   ArrayResize(buf, (int)n);
   uint got = FileReadArray(h, buf, 0, (int)n);
   FileClose(h);
   if(got != n)
     { PrintFormat("KronosLoadMatrixT: short read %s (%u of %I64u)", fname, got, n); return false; }
//--- copy transposed so the stored matrix is [in,out] == W^T
   out = matrix::Zeros(cols, rows);                 // stored [in,out] == W^T
   for(ulong r = 0; r < rows; r++)
      for(ulong c = 0; c < cols; c++)
         out[c][r] = (double)buf[r * cols + c];      // transpose during the copy
   return true;
  }

//+------------------------------------------------------------------+
//| Load an n-element float32 vector from a .bin file.               |
//+------------------------------------------------------------------+
bool KronosLoadVector(const string fname, ulong n, vector &out)
  {
//--- open the binary weight file
   int h = FileOpen(fname, FILE_READ | FILE_BIN);
   if(h == INVALID_HANDLE)
     { PrintFormat("KronosLoadVector: cannot open %s (err %d)", fname, GetLastError()); return false; }
//--- read the raw float32 buffer
   float buf[];
   ArrayResize(buf, (int)n);
   uint got = FileReadArray(h, buf, 0, (int)n);
   FileClose(h);
   if(got != n)
     { PrintFormat("KronosLoadVector: short read %s (%u of %I64u)", fname, got, n); return false; }
//--- copy into the vector
   out = vector::Zeros(n);
   for(ulong i = 0; i < n; i++)
      out[i] = (double)buf[i];
   return true;
  }

#endif // KRONOS_TOKENIZER_MATH_MQH
//+------------------------------------------------------------------+