171 lines
8.2 KiB
MQL5
171 lines
8.2 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| KronosEncoder.mqh |
|
|
//| MMQ — Muhammad Minhas Qamar |
|
|
//| www.mql5.com/en/articles/23304 |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "MMQ — Muhammad Minhas Qamar"
|
|
#property link "https://www.mql5.com/en/articles/23304"
|
|
#property version "1.00"
|
|
|
|
#ifndef KRONOS_ENCODER_MQH
|
|
#define KRONOS_ENCODER_MQH
|
|
|
|
#include <Kronos\KronosTokenizerMath.mqh> // BSQ math, KR_NFEAT(6), KR_CODEBOOK_DIM(20)
|
|
#include <Kronos\KronosTransformerCore.mqh> // LinearT, AddRowBias, TransformerBlock
|
|
|
|
//--- tokenizer architecture (config_tokenizer.json)
|
|
#define KR_TOK_D_MODEL 256 // d_model
|
|
#define KR_TOK_N_HEADS 4 // n_heads -> head_dim = 256/4 = 64
|
|
#define KR_TOK_N_ENC_LAYERS 4 // n_enc_layers (BLOCKS = this - 1 = 3)
|
|
#define KR_TOK_FF_DIM 512 // ff_dim
|
|
|
|
//--- weight folder, relative to the terminal's MQL5/Files/ sandbox
|
|
#define KR_TOK_WEIGHT_DIR "kronos_weights\\tokenizer\\"
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Tokenizer encoder. The weight loaders (KronosLoadMatrixT for |
|
|
//| LinearT weights, KronosLoadVector for biases) live in |
|
|
//| KronosTokenizerMath.mqh, shared with the decoder. |
|
|
//+------------------------------------------------------------------+
|
|
class CKronosEncoder
|
|
{
|
|
private:
|
|
int m_blocks; // n_enc_layers - 1
|
|
int m_heads;
|
|
ulong m_dm; // d_model
|
|
ulong m_ff; // ff_dim
|
|
string m_dir;
|
|
|
|
//--- single linears
|
|
matrix m_embedW;
|
|
vector m_embedB; // embed : [d_model, 6]
|
|
matrix m_quantW;
|
|
vector m_quantB; // quant_embed: [20, d_model]
|
|
|
|
//--- per-block weights (arrays indexed by block 0..m_blocks-1)
|
|
vector m_n1[]; // norm1.weight
|
|
matrix m_Wq[];
|
|
vector m_bq[]; // self_attn.q_proj.weight/bias
|
|
matrix m_Wk[];
|
|
vector m_bk[]; // self_attn.k_proj.weight/bias
|
|
matrix m_Wv[];
|
|
vector m_bv[]; // self_attn.v_proj.weight/bias
|
|
matrix m_Wo[];
|
|
vector m_bo[]; // self_attn.out_proj.weight/bias
|
|
vector m_n2[]; // norm2.weight
|
|
matrix m_W1[]; // ffn.w1.weight : [ff, d_model]
|
|
matrix m_W3[]; // ffn.w3.weight : [ff, d_model]
|
|
matrix m_W2[]; // ffn.w2.weight : [d_model, ff]
|
|
|
|
string F(const string name) { return m_dir + name + ".bin"; }
|
|
|
|
public:
|
|
//+---------------------------------------------------------------+
|
|
//| Load all encoder weights. Pass the config_tokenizer.json |
|
|
//| values (or rely on the KR_TOK_* defines above). Tensor file |
|
|
//| names follow PyTorch named_parameters() for KronosTokenizer, |
|
|
//| with every non-alphanumeric char mapped to '_' by the |
|
|
//| exporter, as recorded in the tokenizer manifest. |
|
|
//+---------------------------------------------------------------+
|
|
bool Init(const string weight_dir, int n_enc_layers, ulong d_model, int n_heads, ulong ff_dim)
|
|
{
|
|
m_dir = weight_dir;
|
|
m_dm = d_model;
|
|
m_heads = n_heads;
|
|
m_ff = ff_dim;
|
|
m_blocks = n_enc_layers - 1; // the (n-1) block-count quirk from the source
|
|
|
|
if(d_model == 0 || n_heads == 0 || n_enc_layers == 0 || ff_dim == 0)
|
|
{ Print("CKronosEncoder::Init: invalid config (zero dimension)"); return false; }
|
|
|
|
bool ok = true;
|
|
//--- embed (6 -> d_model) and quant_embed (d_model -> 20); LinearT weights
|
|
//--- stored transposed (W^T) so LinearT is a plain MatMul
|
|
ok &= KronosLoadMatrixT(F("embed_weight"), m_dm, KR_NFEAT, m_embedW);
|
|
ok &= KronosLoadVector(F("embed_bias"), m_dm, m_embedB);
|
|
ok &= KronosLoadMatrixT(F("quant_embed_weight"), KR_CODEBOOK_DIM, m_dm, m_quantW);
|
|
ok &= KronosLoadVector(F("quant_embed_bias"), KR_CODEBOOK_DIM, m_quantB);
|
|
|
|
ArrayResize(m_n1, m_blocks);
|
|
ArrayResize(m_n2, m_blocks);
|
|
ArrayResize(m_Wq, m_blocks);
|
|
ArrayResize(m_bq, m_blocks);
|
|
ArrayResize(m_Wk, m_blocks);
|
|
ArrayResize(m_bk, m_blocks);
|
|
ArrayResize(m_Wv, m_blocks);
|
|
ArrayResize(m_bv, m_blocks);
|
|
ArrayResize(m_Wo, m_blocks);
|
|
ArrayResize(m_bo, m_blocks);
|
|
ArrayResize(m_W1, m_blocks);
|
|
ArrayResize(m_W3, m_blocks);
|
|
ArrayResize(m_W2, m_blocks);
|
|
|
|
for(int b = 0; b < m_blocks; b++)
|
|
{
|
|
string p = StringFormat("encoder_%d_", b); // ModuleList index
|
|
ok &= KronosLoadVector(F(p + "norm1_weight"), m_dm, m_n1[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "self_attn_q_proj_weight"), m_dm, m_dm, m_Wq[b]);
|
|
ok &= KronosLoadVector(F(p + "self_attn_q_proj_bias"), m_dm, m_bq[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "self_attn_k_proj_weight"), m_dm, m_dm, m_Wk[b]);
|
|
ok &= KronosLoadVector(F(p + "self_attn_k_proj_bias"), m_dm, m_bk[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "self_attn_v_proj_weight"), m_dm, m_dm, m_Wv[b]);
|
|
ok &= KronosLoadVector(F(p + "self_attn_v_proj_bias"), m_dm, m_bv[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "self_attn_out_proj_weight"), m_dm, m_dm, m_Wo[b]);
|
|
ok &= KronosLoadVector(F(p + "self_attn_out_proj_bias"), m_dm, m_bo[b]);
|
|
ok &= KronosLoadVector(F(p + "norm2_weight"), m_dm, m_n2[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "ffn_w1_weight"), m_ff, m_dm, m_W1[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "ffn_w3_weight"), m_ff, m_dm, m_W3[b]);
|
|
ok &= KronosLoadMatrixT(F(p + "ffn_w2_weight"), m_dm, m_ff, m_W2[b]);
|
|
if(!ok)
|
|
{
|
|
PrintFormat("CKronosEncoder::Init failed at block %d", b);
|
|
return false;
|
|
}
|
|
}
|
|
return ok;
|
|
}
|
|
|
|
//+---------------------------------------------------------------+
|
|
//| Encode a normalized window into hierarchical token ids. |
|
|
//| x_norm : (L, 6), preprocessed by KronosNormalize |
|
|
//| s1_ids,s2_ids : output token ids, length L |
|
|
//+---------------------------------------------------------------+
|
|
bool Encode(const matrix &x_norm, int &s1_ids[], int &s2_ids[])
|
|
{
|
|
if(x_norm.Cols() != KR_NFEAT)
|
|
{ PrintFormat("CKronosEncoder::Encode: expected %d feature cols, got %I64u", KR_NFEAT, x_norm.Cols()); return false; }
|
|
|
|
//--- embed: 6 -> d_model
|
|
matrix z = LinearT(x_norm, m_embedW);
|
|
AddRowBias(z, m_embedB);
|
|
//--- (n_enc_layers - 1) causal pre-norm blocks
|
|
for(int b = 0; b < m_blocks; b++)
|
|
z = TransformerBlock(z, m_n1[b],
|
|
m_Wq[b], m_bq[b], m_Wk[b], m_bk[b],
|
|
m_Wv[b], m_bv[b], m_Wo[b], m_bo[b],
|
|
m_heads, m_n2[b], m_W1[b], m_W3[b], m_W2[b]);
|
|
//--- quant_embed: d_model -> 20 (this is the BSQ input z)
|
|
matrix zq = LinearT(z, m_quantW);
|
|
AddRowBias(zq, m_quantB);
|
|
//--- BSQ: indices depend ONLY on sign of each of the 20 components, so the
|
|
//--- L2-normalize + scale inside BinarySphericalQuantizer are skipped here
|
|
ulong L = zq.Rows();
|
|
ArrayResize(s1_ids, (int)L);
|
|
ArrayResize(s2_ids, (int)L);
|
|
double row[];
|
|
ArrayResize(row, KR_CODEBOOK_DIM);
|
|
for(ulong t = 0; t < L; t++)
|
|
{
|
|
for(int k = 0; k < KR_CODEBOOK_DIM; k++)
|
|
row[k] = zq[t][k];
|
|
int s1, s2;
|
|
KronosBSQ_SignsToIndices(row, s1, s2);
|
|
s1_ids[(int)t] = s1;
|
|
s2_ids[(int)t] = s2;
|
|
}
|
|
return true;
|
|
}
|
|
};
|
|
|
|
#endif // KRONOS_ENCODER_MQH
|
|
//+------------------------------------------------------------------+
|