BasesParserSLan/Src/Main.mqh

//+------------------------------------------------------------------+
//|                                                         Main.mqh |
//|                                  Copyright 2026, Niquel Mendoza. |
//|                                            https://www.mql5.com/ |
//+------------------------------------------------------------------+
#property copyright "Copyright 2026, Niquel Mendoza."
#property link      "https://www.mql5.com/"
#property strict

#ifndef BASESPARSERSLAN_SRC_MAIN_MQH
#define BASESPARSERSLAN_SRC_MAIN_MQH

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "Def.mqh"

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
namespace TSN
{
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CBaseStructuredLan
 {
public:
  //--- Data del documento
  long               m_cinta[];
  uchar              m_raw[];
  int                m_cinta_pos;
  int                m_len;
  int                m_offset;
  int                m_pos;
  int                m_cinta_reserve;

  //--- Perfect hash
  //- Pf
  CPerfectHashByLeo<ulong, CSBLHasherGenUl, CSBLHasherGenUl1> m_pfh;

  //- Structs
  struct PerfectHash
   {
    ulong            seeds[];     // Seeds..
    ulong            key_hash[];  // Key hash
    int              pos_index[]; // [h] (hash) = devuevle indice del objeto
    int              pos_arr[];   // [i] (index de objeto) = devuelve indice en la cinta
    ulong            bucket_size; // Buckets
    ulong            final_table_size;  // Tamaño de pos index (tabla final)
    int              start;       // Start en cinta
    int              table_size;  // Tamaño de tabla
   };
  PerfectHash        m_tables[];
  int                m_tables_size;
  int                m_tables_reserve;
  ulong              m_max_att;

protected:
  //--- Del parseo
  uchar              m_next;

  //---
  virtual int        CalcRegType(const string& file_name) = 0;
  int                FindNodePosOrCreate(const int start);
  void               CorrectPadingInternal(const uchar c);

public:
                     CBaseStructuredLan(void);
                    ~CBaseStructuredLan(void) {}

  //---
  __forceinline void CalcLen() { m_len = ArraySize(m_raw); }
  inline void        Assing(const string& raw);
  int                AssingFile(const string& file_name, const bool comon_flag);

  //---
  bool               SaveCompiled(const string & file_name_out, const bool out_common_flag) const;
  bool               SaveCompiled(const string & file_name_out, const bool out_common_flag, const string & file_name_json, const bool json_common_flag) const;
  void               SaveTables(const int fh) const;
  void               LoadTables(const int fh);

  //---
  int                PerfectHashComputeHash(const string& word, const int node_pos) const;
  int                PerfectHashComputeHashIndex(const string& word, const int node_pos) const;

  //---
  int                FindNodePos(const int start) const;
  void               NormalizeUft();

  //---
  void               GetLastErrorLocation(int &col, int &line) const;
 };

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CBaseStructuredLan::CBaseStructuredLan()
  : m_cinta_reserve(0), m_len(0), m_offset(0), m_tables_size(0), m_max_att(1000)
 {
  m_tables_reserve = ArrayResize(m_tables, 16, 16);
  m_cinta_reserve = ArrayResize(m_cinta, 4096, 4096);
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
inline void CBaseStructuredLan::Assing(const string& raw)
 {
  m_len = StringToCharArray(raw, m_raw);
  m_offset = 0;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CBaseStructuredLan::AssingFile(const string& file_name, const bool comon_flag)
 {
//---
  const int reg_type = CalcRegType(file_name);

//---
  if(reg_type == TSN_SLF_EXT_TYPE_CACHE_FILE) // jsontp tape cache
   {
    ::ResetLastError();
    const int fh = FileOpen(file_name, FILE_BIN | FILE_ANSI | (comon_flag ?  FILE_COMMON : 0) | FILE_READ);
    if(fh == INVALID_HANDLE)
     {
      return -1;
     }
    // Metadata (cinta\yaml)
    m_cinta_pos = FileReadInteger(fh); // Nota que cuadno se guarda se hace con el tamaño exacto
    m_cinta_reserve = m_cinta_pos; // Mismo tamaño
    const int type = FileReadInteger(fh); // Tipo ()

    //---
    if(type == TSN_SLF_RAW_EMBEBED)
     {
      m_len = FileReadInteger(fh);
      m_offset = FileReadInteger(fh);
      FileReadArray(fh, m_cinta, 0, m_cinta_pos); // Total
      // Yaml embebido (aqui no hace falta hacer nada ya todo esta hecho.. osea lo de encoding etc..)
      FileReadArray(fh, m_raw, 0, m_len);
      LoadTables(fh); // Perfect hash tables
     }
    else // Dir
     {
      const bool common = bool(FileReadInteger(fh, CHAR_VALUE));
      const int file_name_len = FileReadInteger(fh);
      if(AssingFile(FileReadString(fh, file_name_len), common) == TSN_SLF_EXT_TYPE_CACHE_FILE)
       {
        FileClose(fh);
        return -1; // fallo
       }
      FileReadArray(fh, m_cinta, 0, m_cinta_pos);
      LoadTables(fh); // Perfect hash tables
     }

    //---
    FileClose(fh);
    return TSN_SLF_EXT_TYPE_CACHE_FILE;
   }
  else // Ninguno de los dos... json normal O jsonasm
   {
    const int l = (int)FileLoad(file_name, m_raw, (comon_flag ? FILE_COMMON : 0));
    if(l == -1)
      return -1;

    //---
    m_len = l;

    //---
    NormalizeUft();

    //---
    return reg_type;
   }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CBaseStructuredLan::NormalizeUft()
 {
  m_offset = 0;

//---
  if(m_len >= 2)
   {
    // UTF16 BE
    if(m_raw[0] == 0xFF && m_raw[1] == 0xFE)
     {
      const int n = (m_len - 2) >> 1;
      for(int i = 0; i < n; i++)
        m_raw[i] = m_raw[2 + (i * 2)]; // byte bajo (el char real)
      m_len = ArrayResize(m_raw, n);
     }
    // UTF16 LE
    else
      if(m_raw[0] == 0xFE && m_raw[1] == 0xFF)
       {
        const int n = (m_len - 2) >> 1;
        for(int i = 0; i < n; i++)
          m_raw[i] = m_raw[3 + (i * 2)]; // byte alto (invertido)
        m_len = ArrayResize(m_raw, n);
       }
   }
  else
    if(m_len >= 3 && m_raw[0] == 0xEF && m_raw[1] == 0xBB && m_raw[2] == 0xBF)
     {
      // UTF-8 BOM
      m_offset = 3; // simplemente saltear
     }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CBaseStructuredLan::FindNodePosOrCreate(const int start)
 {
  for(int i = 0; i < m_tables_size; i++)
   {
    if(m_tables[i].start == start)
      return i;
   }

//---
  if(m_tables_size + 1 >= m_tables_reserve)
   {
    m_tables_reserve += 32;
    ArrayResize(m_tables, m_tables_reserve, m_tables_reserve);
   }

//---
  m_tables[m_tables_size++].start = start;

//---
  return m_tables_size - 1;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CBaseStructuredLan::FindNodePos(const int start) const
 {
  for(int i = 0; i < m_tables_size; i++)
   {
    if(m_tables[i].start == start)
      return i;
   }
  return -1;
 }


//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CBaseStructuredLan::PerfectHashComputeHash(const string& word, const int node_pos) const
 {
//--- Hash 64
  ulong key_hash = FNV_OFFSET_BASIS;
  const int len = StringLen(word);
  for(int i = 0; i < len; i++)
   {
    key_hash ^= (uchar)word[i];
    key_hash *= FNV_PRIME;
   }
//PrintFormat("Hash para %s = %I64u", word, key_hash);

//----
  const int seed_index = int(key_hash % m_tables[node_pos].bucket_size);

//--- Alg
  ulong h = key_hash + m_tables[node_pos].seeds[seed_index] * 0x9e3779b97f4a7c15;
  h = (h ^ (h >> 30)) * 0xbf58476d1ce4e5b9;
  h = (h ^ (h >> 27)) * 0x94d049bb133111eb;
  h ^= (h >> 31);

//--- Pos
  const int fi = int(h % m_tables[node_pos].final_table_size);
  return m_tables[node_pos].key_hash[fi] == key_hash ? m_tables[node_pos].pos_arr[m_tables[node_pos].pos_index[fi]] : -1;
 }

//+------------------------------------------------------------------+
int CBaseStructuredLan::PerfectHashComputeHashIndex(const string& word, const int node_pos) const
 {
//--- Hash 64
  ulong key_hash = FNV_OFFSET_BASIS;
  const int len = StringLen(word);
  for(int i = 0; i < len; i++)
   {
    key_hash ^= (uchar)word[i];
    key_hash *= FNV_PRIME;
   }
//PrintFormat("Hash para %s = %I64u", word, key_hash);

//----
  const int seed_index = int(key_hash % m_tables[node_pos].bucket_size);

//--- Alg
  ulong h = key_hash + m_tables[node_pos].seeds[seed_index] * 0x9e3779b97f4a7c15;
  h = (h ^ (h >> 30)) * 0xbf58476d1ce4e5b9;
  h = (h ^ (h >> 27)) * 0x94d049bb133111eb;
  h ^= (h >> 31);

//--- Pos
  const int fi = int(h % m_tables[node_pos].final_table_size);
  return m_tables[node_pos].key_hash[fi] == key_hash ? m_tables[node_pos].pos_index[fi] : -1;
 }


//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
//--- Ambos
// 4 bytes: m_cinta_pos
// 4 bytes: type


//---
bool CBaseStructuredLan::SaveCompiled(const string & file_name_out, const bool out_common_flag) const
 {
  ::ResetLastError();
  const int fh = FileOpen(file_name_out, FILE_BIN | (out_common_flag ?  FILE_COMMON : 0) | FILE_WRITE | FILE_ANSI);
  if(fh == INVALID_HANDLE)
   {
    return false;
   }

//--- Comun
  FileWriteInteger(fh, m_cinta_pos);
  FileWriteInteger(fh, TSN_SLF_RAW_EMBEBED);

//---
// 4 bytes: len
// 4 bytes: offset
// array: cinta
// array yaml

//---
  FileWriteInteger(fh, m_len);
  FileWriteInteger(fh, m_offset);
  FileWriteArray(fh, m_cinta, 0, m_cinta_pos);
  FileWriteArray(fh, m_raw, 0, m_len);

//--- Perfect hash tables (m_tables)
  SaveTables(fh);

//---
  FileClose(fh);

//---
  return true;
 }

//+------------------------------------------------------------------+
bool CBaseStructuredLan::SaveCompiled(const string & file_name_out, const bool out_common_flag,
                                      const string & file_name_data, const bool data_common_flag) const
 {
//---
  ::ResetLastError();
  const int fh = FileOpen(file_name_out, FILE_BIN | (out_common_flag ?  FILE_COMMON : 0) | FILE_WRITE | FILE_ANSI);
  if(fh == INVALID_HANDLE)
   {
    return false;
   }

//--- Comun
  FileWriteInteger(fh, m_cinta_pos);
  FileWriteInteger(fh, TSN_SLF_RAW_FILE);

//---
// 1 bytes: comon
// 4 bytes: len
// string: name_file
// array: cinta


//---
  FileWriteInteger(fh, data_common_flag, CHAR_VALUE);
  const int len = StringLen(file_name_data);
  FileWriteInteger(fh, len);
  FileWriteString(fh, file_name_data, len);
  FileWriteArray(fh, m_cinta, 0, m_cinta_pos);

//--- Perfect hash tables (m_tables)
  SaveTables(fh);

//---
  FileClose(fh);

//---
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CBaseStructuredLan::SaveTables(const int fh) const
 {
//---
  FileWriteInteger(fh, m_tables_size);

//---
  for(int i = 0; i < m_tables_size; i++)
   {
    const int table_size =  m_tables[i].table_size;
    const int bucket_size = (int)m_tables[i].bucket_size;
    const int final_size = (int)m_tables[i].final_table_size;

    FileWriteInteger(fh, m_tables[i].start);
    FileWriteInteger(fh, table_size);
    FileWriteInteger(fh, bucket_size);
    FileWriteInteger(fh, final_size);

    //---
    FileWriteArray(fh, m_tables[i].seeds, 0, bucket_size);
    FileWriteArray(fh, m_tables[i].pos_index, 0, final_size);
    FileWriteArray(fh, m_tables[i].key_hash, 0, final_size);
    FileWriteArray(fh, m_tables[i].pos_arr, 0, table_size);
   }
 }

//+------------------------------------------------------------------+
void CBaseStructuredLan::LoadTables(const int fh)
 {
//---
  m_tables_size = FileReadInteger(fh);
  m_tables_reserve = m_tables_size;
  ArrayResize(m_tables, m_tables_size, m_tables_size);

//---
  for(int i = 0; i < m_tables_size; i++)
   {
    m_tables[i].start = FileReadInteger(fh);
    const int table_size = FileReadInteger(fh);
    m_tables[i].table_size = table_size;
    const int bucket_size =  FileReadInteger(fh);
    m_tables[i].bucket_size = ulong(bucket_size);
    const int final_size =  FileReadInteger(fh);
    m_tables[i].final_table_size = ulong(final_size);

    //---
    FileReadArray(fh, m_tables[i].seeds, 0, bucket_size);
    FileReadArray(fh, m_tables[i].pos_index, 0, final_size);
    FileReadArray(fh, m_tables[i].key_hash, 0, final_size);
    FileReadArray(fh, m_tables[i].pos_arr, 0, table_size);
   }
 }


//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CBaseStructuredLan::CorrectPadingInternal(const uchar c)
 {
//---
  const int prev = m_len;
  ArrayResize(m_raw, (m_len += 8));

//--- Rellenamos
  for(int i = prev; i < m_len; i++)
   {
    m_raw[i] = c;
   }

//--- Chekeamos
  if(m_len > m_cinta_reserve)
   {
    m_cinta_reserve = m_len;
    ArrayResize(m_cinta, m_len, m_len);
   }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CBaseStructuredLan::GetLastErrorLocation(int &col, int &line) const
 {
  col = 1;
  line = 1;

//Print("err:");
//PrintFormat("Val = '%s'", CharArrayToString(m_raw, 0, m_pos));

  for(int i = 0; i <= m_pos && i < m_len; i++)
   {
    if(m_raw[i] == '\n')
     {
      line++;
      col = 1;
     }
    else
      col++;
   }
 }
}
//+------------------------------------------------------------------+
#endif // BASESPARSERSLAN_SRC_MAIN_MQH