SetFileByLeo/Src/Parser.mqh

//+------------------------------------------------------------------+
//|                                                       Parser.mqh |
//|                                  Copyright 2026, Niquel Mendoza. |
//|                                            https://www.mql5.com/ |
//+------------------------------------------------------------------+
#property copyright "Copyright 2026, Niquel Mendoza."
#property link      "https://www.mql5.com/"
#property strict

#ifndef SETFILEBYLEO_SRC_PARSER_MQH
#define SETFILEBYLEO_SRC_PARSER_MQH

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "Def.mqh"

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
namespace TSN
{
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CSetFileParser
 {
public:
  uchar              m_data[];
  long               m_cinta[];
  int                m_cinta_pos;
  int                m_cinta_reserve;
  int                m_offset;
  int                m_pos;
  int                m_len;

  //---
  CPerfectHashByLeo<ulong> m_pfh;

  //---
  ulong              m_table_seeds[];     // Seeds..
  ulong              m_table_key_hash[];  // Key hash
  int                m_table_pos_index[]; // [h] (hash) = devuelve indice del objeto
  int                m_table_pos_arr[];   // [i] (index de objeto) = devuelve indice en la cinta (posicion del KEY)
  ulong              m_table_bucket_size; // Buckets
  ulong              m_table_final_table_size; // Tamaño de pos_index (tabla final)
  int                m_table_size;        // Cantidad de keys en la tabla
  bool               m_table_ready;       // true una vez construida con exito
  ulong              m_max_att;

private:
  uchar              m_next;
  int                m_prev_start;
  ENUM_SETFILE_PARSER_ERROR m_last_err;
  long               m_mask;
  int                m_pares;
  int                m_last_lock;
  int                m_root_c;

  //---
  bool               ProccesDecimal(double val);


public:
                     CSetFileParser(void);
                    ~CSetFileParser(void) {}

  //--- Utils.
  bool               StrEquals(const string &str, const int curr) const;
  bool               StrEquals(const int other, const int curr) const;
  __forceinline int  GetStep(const int idx) const;

  //--- Load..
  inline void        Assing(const string& set);
  int                AssingFile(const string& file_name, const bool comon_flag);
  bool               AutoParseByFileExt(const int file_ext_type);

  //--- Save\Load
  bool               SaveCompiled(const string & file_name_out, const bool out_common_flag) const;
  bool               SaveCompiled(const string & file_name_out, const bool out_common_flag, const string & file_name_set, const bool json_common_flag) const;
  void               SaveTables(const int fh) const;
  void               LoadTables(const int fh);

  //--- Extra
  __forceinline void CalcLen() { m_len = ArraySize(m_data); }
  void               CorrectPading();

  //--- Last err
  __forceinline ENUM_SETFILE_PARSER_ERROR LastErr() const { return m_last_err; }

  //--- Info
  void               PrintCintaTypes(const int start, const int num) const;

  //--- Base
  bool               Parse();
  CSetNode           GetRoot();

  //--- Perfect hash
  int                PerfectHashComputeHash(const string& word) const;
  int                PerfectHashComputeHashIndex(const string& word) const;
  bool               NodeHashing(int curr, const int obj_count, const int bend);
 };

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CSetFileParser::CSetFileParser(void)
  : m_table_size(0), m_table_ready(false), m_max_att(1000)
 {
//---
  m_cinta_reserve = ArrayResize(m_cinta, 4096, 4096);

//---
  m_pfh.DeleteHashers(true);
  m_pfh.Hasher0(new CSetFHasherGenUl());
  m_pfh.Hasher1(new CSetFHasherGenUl1());
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
inline void CSetFileParser::Assing(const string& set)
 {
  m_len = StringToCharArray(set, m_data);
  m_offset = 0;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
__forceinline int CSetFileParser::GetStep(const int idx) const
 {
  const int tipo = int(m_cinta[idx] & 0xF);
  switch(tipo)
   {
    case SETFILE_TOK_ROOT:
      return 1;
    case SETFILE_TOK_INTEGER:
    case SETFILE_TOK_REAL:
      return 2;
    case SETFILE_TOK_BOOLEAN:
      return 1;
    case SETFILE_TOK_STRING:
      return 1;
    case SETFILE_TOK_KEY:
      return 1;
    case SETFILE_TOK_META:
      return 1;
   }
  return 1;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::StrEquals(const string &str, const int curr) const
 {
//---
// m_cinta[m_cinta_pos++] = long(m_pos - 1) | long(start) << TSN_JSON_BIT_STR_START_END;
  const int start = int(m_cinta[curr] >> TSN_SETFILE_BIT_START_STR);
  const int len = int((m_cinta[curr] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF);
//PrintFormat("Exact = '%s', other = '%s'", CharArrayToString(m_data, start, len), str);

  const int l = StringLen(str);

//---
  if(l != len)
    return false;
  for(int i = 0; i < l; i++)
   {
    if(str[i] != m_data[start + i])
      return false;
   }
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::StrEquals(const int other, const int curr) const
 {
//---
// m_cinta[m_cinta_pos++] = long(m_pos - 1) | long(start) << TSN_JSON_BIT_STR_START_END;
  const int start = int(m_cinta[curr] >> TSN_SETFILE_BIT_START_STR);
  const int len = int((m_cinta[curr] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF);
//PrintFormat("Exact = '%s', other = '%s'", CharArrayToString(m_data, start, len), str);
  const int s = int(m_cinta[other] >> TSN_SETFILE_BIT_START_STR);
  const int l = int((m_cinta[other] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF);

//---
  if(l != len)
    return false;
  for(int i = 0; i < l; i++)
   {
    if(m_data[s + i] != m_data[start + i])
      return false;
   }
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
// luego de un valor buscar key
#define TSN_SETFILE_NEXT_V \
        while(true) \
         { \
          if(m_pos >= m_len) \
           { \
            m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;\
            return true; \
           } \
          const uchar cfasd = m_data[m_pos]; \
          if(cfasd < 33) \
           { \
            m_pos++; \
            continue; \
           } \
          if(cfasd == ';') \
           { \
            m_next = SETFILE_TOKENS_INT_COMENTARIO; \
           } \
          else \
           { \
            m_next = SETFILE_TOKENS_INT_KEY; \
           } \
           break; \
         } \




//----
#define TSN_SETFILE_CHECKNUMBER \
        while(true) \
         { \
          const uchar f = m_data[m_pos]; \
          if(f < 33) \
           { \
            if(f == '\n') \
             { \
              if(m_pares>0) \
               { \
               m_cinta_pos = m_last_lock + 1; \
               m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(m_pos - m_prev_start) << TSN_SETFILE_BIT_START_LEN_PLAIN \
                                   | long(m_prev_start) << TSN_SETFILE_BIT_START_STR; \
               m_pos++; \
              TSN_SETFILE_NEXT_V \
               } \
              else \
              { \
                m_cinta[m_last_lock] |= long(SETFILE_TOK_OP_TYPE_VAL) << TSN_SETFILE_BIT_END_TYPE; \
                m_pos++;\
               TSN_SETFILE_NEXT_V\
              } \
              break; \
             } \
            m_pos++; \
            if(m_pos >= m_len) \
             { \
              m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;\
              return true; \
             } \
            continue; \
           } \
          if(f == '|') \
           { \
            if(m_data[m_pos+1] == '|') \
             { \
              m_pos+=2; \
              while(true) \
               { \
                const uchar fca = m_data[m_pos]; \
                if(fca < 33) \
                 { \
                  m_pos++; \
                  if(m_pos >= m_len) \
                   { \
                    m_cinta_pos = m_last_lock + 1; \
                    m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(m_pos - m_prev_start) << TSN_SETFILE_BIT_START_LEN_PLAIN \
                                   | long(m_prev_start) << TSN_SETFILE_BIT_START_STR; \
                     m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;\
                    return true; \
                   } \
                  continue; \
                 } \
                if(m_pares >= 3) \
                 { \
                  m_cinta[m_last_lock] |= long((fca == 'Y') ? SETFILE_TOK_OP_TYPE_Y : SETFILE_TOK_OP_TYPE_N) << TSN_SETFILE_BIT_END_TYPE; \
                  m_pos++;\
                  TSN_SETFILE_NEXT_V \
                 } \
                else \
                 { \
                  if((fca ^ '0') < 10 || fca == '-') \
                   { \
                    m_next = SETFILE_TOKENS_INT_NUMBER; \
                    m_pares++; \
                   } \
                  else \
                   { \
                    m_pos++;\
                    m_cinta_pos = m_last_lock + 1; \
                    m_next = SETFILE_TOKENS_INT_STRING; \
                   } \
                 }\
                break;\
               }\
              break;\
             } \
           }\
          m_pos++;\
          m_cinta_pos = m_last_lock + 1;   \
          m_next = SETFILE_TOKENS_INT_STRING; \
          break; \
         } \


// 0-1 | 1-2 | 2-3 |
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::ProccesDecimal(double val)
 {
//---
  m_pos++;

//---
  long frac_int = 0;
  int fract_digits = 0;

//---
  while(true)
   {
    const uchar c = m_data[m_pos] ^ '0';
    if(c > 9)
     {
      if(c == 76 || c == 58) // es | o \n
        break;
      else
        return false; // plain
     }
    frac_int = (frac_int << 3) + (frac_int << 1) + c;
    fract_digits++;
    m_pos++;
   }

//---
  val += frac_int * g_Exp10[fract_digits];

//---
  static BitInterpreter bit;
  bit.double_value = val;
  bit.long_value ^= (m_mask & (1LL << 63));
  m_cinta[m_cinta_pos++] = SETFILE_TOK_REAL;
  m_cinta[m_cinta_pos++] = bit.long_value;
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CSetFileParser::CorrectPading(void)
 {
//---
  const int extra = 1 + (m_len % 8); // Resto
  const int prev = m_len;
  ArrayResize(m_data, (m_len += extra));

//---
  for(int i = prev; i < m_len; i++)
   {
    m_data[i] = '\n';
   }
// ArrayPrint(m_data);
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::Parse()
 {
//---
  m_pos = m_offset;
  m_cinta_pos = 0;
  m_root_c = 0;
  if(m_len > m_cinta_reserve)
   {
    m_cinta_reserve = m_len;
    ArrayResize(m_cinta, m_len, m_len);
   }

//---
  m_cinta[m_cinta_pos++] = SETFILE_TOK_ROOT;
  while(true)
   {
    const uchar f = m_data[m_pos];
    if(f < 33)
     {
      m_pos++;
      continue;
     }
    if(f == ';')
     {
      m_next = SETFILE_TOKENS_INT_COMENTARIO;
     }
    else
     {
      m_next = SETFILE_TOKENS_INT_KEY;
     }
    break;
   }

//---
  while(true)
   {
    switch(m_next)
     {
      case SETFILE_TOKENS_INT_NUMBER:
       {
        //---
        long v = 0;

        //--- Validamos signo
        m_mask = 0;
        uchar ch = m_data[m_pos];
        if(ch == '-')
         {
          m_mask = -1;
          m_pos++; // Ahora en numero
          ch = m_data[m_pos];
         }

        //---
        ch ^= '0';

        //---
        while(true)
         {
          //---
          v = (v << 3) + (v << 1) + ch;

          //--- *
          m_pos++;

          //---
          ch = m_data[m_pos]  ^ '0';

          //---
          if(ch < 10)
           {
            continue;
           }
          else
            if(ch == 30)
             {
              if(!ProccesDecimal(double(v)))
               {
                m_next = SETFILE_TOKENS_INT_STRING;
               }
              else
               {
                TSN_SETFILE_CHECKNUMBER
               }
              break;
             }
            else
             {
              m_cinta[m_cinta_pos++] = SETFILE_TOK_INTEGER;
              m_cinta[m_cinta_pos++] = (v ^ m_mask) - m_mask;
              // Print("c: " , CharToString(m_data[m_pos]), " pos: " , m_pos, " len: " , m_len);
              TSN_SETFILE_CHECKNUMBER
              // Print(EnumToString((ENUM_SETFILE_TOKENS_INT)m_next));
              break; // Fin
             }
         }

        //---
        break;
       }

      //--- string
      case SETFILE_TOKENS_INT_STRING:
       {
        const int start = m_prev_start == -1 ? m_pos++ : m_prev_start;

#ifndef SETFILEBYLEO_DESACTIVE_SWAR
        int curr_end = m_pos + 8;
        if(curr_end <= m_len)
         {
          while(true)
           {
            const ulong chunk = (ulong)m_data[m_pos]
                                | (ulong)m_data[m_pos + 1] << 8
                                | (ulong)m_data[m_pos + 2] << 16
                                | (ulong)m_data[m_pos + 3] << 24
                                | (ulong)m_data[m_pos + 4] << 32
                                | (ulong)m_data[m_pos + 5] << 40
                                | (ulong)m_data[m_pos + 6] << 48
                                | (ulong)m_data[m_pos + 7] << 56;

            //---
            const ulong x = chunk ^ TSNTABLES_SWAR_MASK_NEW_LINE;
            const ulong has_x = TSNTABLES_SWAR_HAS(x);
            if(has_x == 0)
             {
              m_pos = curr_end;
              curr_end += 8;

              //---
              if(curr_end > m_len)
               {
                do
                 {
                  if(m_data[m_pos] == '\n')
                    break;
                  m_pos++;
                 }
                while(m_pos < m_len);
                break; // Salimos..
               }

              //---
              continue;
             }

            //---
            // Si hay \n
            const ulong flag = TSNTABLES_SWAR_64_DEJAR_SOLO_MINUS_SIG_BYTE(has_x);
            m_pos += TSNTABLES_SWAR_64_GET_BYTE(flag); // Ahora mismo en \n
            break;
           }
         }
        else
         {
#endif // SETFILEBYLEO_DESACTIVE_SWAR
          do
           {
            if(m_data[m_pos] == '\n')
              break;
            m_pos++;
           }
          while(m_pos < m_len);
#ifndef SETFILEBYLEO_DESACTIVE_SWAR
         }
#endif // SETFILEBYLEO_DESACTIVE_SWAR

        //---
        m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(m_pos - start) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                 | long(start) << TSN_SETFILE_BIT_START_STR;


        //---
        m_pos++; // Luiego del \n

        //---
        //m_prev_start = -1;

        //---
        TSN_SETFILE_NEXT_V
        break;
       }

      //---
      case SETFILE_TOKENS_INT_TRUE:
       {
        m_prev_start = m_pos++;
        if(m_pos + 2 < m_len)
         {
          if(m_data[m_pos++] != 'r' || m_data[m_pos++] != 'u' || m_data[m_pos++] != 'e')
           {
            m_next = SETFILE_TOKENS_INT_STRING;
            break;
           }
          if(m_pos <= m_len)
           {
            m_cinta[m_cinta_pos++] = SETFILE_TOK_BOOLEAN | (long)1 << TSN_SETFILE_BIT_END_TYPE;
            m_next = SETFILE_TOKENS_INT_AFTER_TRUE_FASE_OPT;
           }
          else
           {
            m_pos++;
            m_next = SETFILE_TOKENS_INT_STRING;
           }
         }
        else
         {
          m_next = SETFILE_TOKENS_INT_STRING;
         }
        break;
       }

      //---
      case SETFILE_TOKENS_INT_FALSE:
       {
        m_prev_start = m_pos++;
        if(m_pos + 3 < m_len)
         {
          if(m_data[m_pos++] != 'a' || m_data[m_pos++] != 'l' || m_data[m_pos++] != 's' || m_data[m_pos++] != 'e')
           {
            m_next = SETFILE_TOKENS_INT_STRING;
            break;
           }
          if(m_pos <= m_len)
           {
            m_cinta[m_cinta_pos++] = SETFILE_TOK_BOOLEAN;
            m_next = SETFILE_TOKENS_INT_AFTER_TRUE_FASE_OPT; // check
           }
          else
           {
            m_pos++;
            m_next = SETFILE_TOKENS_INT_STRING;
           }
         }
        else
         {
          m_next = SETFILE_TOKENS_INT_STRING;
         }
        break;
       }

      //---
      case SETFILE_TOKENS_INT_AFTER_TRUE_FASE_OPT: // =x||false||0||true||N\Y
       {
        while(true)
         {
          const uchar f = m_data[m_pos];
          if(f < 33)
           {
            if(f == '\n') // termino.
             {
              TSN_SETFILE_NEXT_V
              break;
             }

            //---
            m_pos++;
            if(m_pos >= m_len)
             {
              m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;
              return true;
             }
            continue;
           }
          if(f == '|')
           {
            if(m_data[m_pos + 1] == '|')
             {
              m_pos += 2; // luego del |
              // Skip ws
              int state = 0;
              // 0 = false
              // 1 = 0
              // 2 = true
              do
               {
                const uchar fc = m_data[m_pos];

                // ws
                if(fc < 33)
                 {
                  m_pos++;
                  if(m_pos >= m_len)
                   {
                    m_cinta_pos = m_last_lock + 1;
                    m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(m_pos - m_prev_start) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                             | long(m_prev_start) << TSN_SETFILE_BIT_START_STR;
                    //m_pos++;
                    // m_prev_start = -1;
                    m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;
                    return true; // terminamos...
                   }
                  continue;
                 }


                // state
                switch(state)
                 {
                  case 0: // false
                   {
                    if(m_pos + 4 < m_len)
                     {
                      if(m_data[m_pos++] != 'f' || m_data[m_pos++] != 'a' || m_data[m_pos++] != 'l' || m_data[m_pos++] != 's' || m_data[m_pos++] != 'e')
                       {
                        state = -1;
                        break;
                       }
                      if(m_pos < m_len) // || YAMLPARSER_IS_PARAR_CHARCTX(m_data[m_pos])
                       {
                        m_cinta[m_cinta_pos++] = SETFILE_TOK_BOOLEAN; // bool\bool
                        state++;
                       }
                      else
                       {
                        m_pos++;
                        state = -1;
                       }
                     }
                    else //
                     {
                      state = -1;
                     }
                    break;
                   }
                  case 1: // 0
                   {
                    if(m_data[m_pos] != '0')
                     {
                      state = -1;
                     }
                    else
                     {
                      m_cinta[m_cinta_pos++] = SETFILE_TOK_BOOLEAN; // false=
                      state++;
                     }
                    break;
                   }
                  case 2: // true
                   {
                    if(m_pos + 3 < m_len)
                     {
                      if(m_data[m_pos++] != 't' || m_data[m_pos++] != 'r' || m_data[m_pos++] != 'u' || m_data[m_pos++] != 'e')
                       {
                        state = -1;
                        break;
                       }
                      if(m_pos < m_len) // || YAMLPARSER_IS_PARAR_CHARCTX(m_data[m_pos])
                       {
                        m_cinta[m_cinta_pos++] = SETFILE_TOK_BOOLEAN | long(1) << TSN_SETFILE_BIT_END_TYPE;  // bool\bool
                        state++;
                       }
                      else
                       {
                        m_pos++;
                        state = -1;
                       }
                     }
                    else //
                     {
                      state = -1;
                     }
                    break;
                   }
                  case 3: // Y\N
                   {
                    state = -2; // salida limpia
                    m_cinta[m_last_lock] |= long((fc == 'Y') ? SETFILE_TOK_OP_TYPE_Y : SETFILE_TOK_OP_TYPE_N) << TSN_SETFILE_BIT_END_TYPE;
                    m_pos++; // luego
                    //m_prev_start = -1; // Salida limpia
                    TSN_SETFILE_NEXT_V
                    break;
                   }
                 }

                // Buscar ||
                while(state >= 0)
                 {
                  if(m_data[m_pos] == '|')
                   {
                    if(m_data[m_pos + 1] == '|')
                     {
                      m_pos += 2;
                      break; // Bien continuamos
                     }
                    else
                     {
                      state = -1; // Fallo
                      break;
                     }
                   }

                  //---
                  m_pos++;
                  if(m_pos >= m_len)
                   {
                    // salimos de una vez..
                    m_cinta_pos = m_last_lock + 1;
                    m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(m_pos - m_prev_start) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                             | long(m_prev_start) << TSN_SETFILE_BIT_START_STR;
                    //m_pos++;
                    //m_prev_start = -1;
                    m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;
                    return true;
                   }
                 }
               }
              while(state >= 0);

              //---
              if(state == -1)
               {
                //Print("gf");
                m_next = SETFILE_TOKENS_INT_STRING;
                m_cinta_pos = m_last_lock + 1;
               }
             }
            else
             {
              m_cinta_pos = m_last_lock + 1; // para que sobreescibra
              m_next = SETFILE_TOKENS_INT_STRING;
             }
           }
          else
           {
            // Ahotra mismo en f que es invalido no es |
            m_pos++; // luego del |
            m_cinta_pos = m_last_lock + 1; // para que sobreescibra
            m_next = SETFILE_TOKENS_INT_STRING;
           }
          break;
         }

        //---
        break;
       }

      //---
      case SETFILE_TOKENS_INT_KEY:
       {
        const int start = m_pos++; // Luego del a=
        //Print(CharToString(m_data[m_pos]));

        //---
        int curr_end = m_pos + 8;
        if(curr_end <= m_len)
         {
          while(true)
           {
            //Print(CharArrayToString(m_data, m_pos, 7));
            const ulong chunk = (ulong)m_data[m_pos]
                                | (ulong)m_data[m_pos + 1] << 8
                                | (ulong)m_data[m_pos + 2] << 16
                                | (ulong)m_data[m_pos + 3] << 24
                                | (ulong)m_data[m_pos + 4] << 32
                                | (ulong)m_data[m_pos + 5] << 40
                                | (ulong)m_data[m_pos + 6] << 48
                                | (ulong)m_data[m_pos + 7] << 56;

            //---
            const ulong x = chunk ^ TSNTABLES_SWAR_MASK_IGUAL;
            const ulong has_x = TSNTABLES_SWAR_HAS(x);
            if(has_x == 0)
             {
              m_pos = curr_end;
              curr_end += 8;

              //---
              if(curr_end > m_len)
               {
                do
                 {
                  if(m_data[m_pos] == '=')
                    break;
                  m_pos++;
                 }
                while(m_pos < m_len);
                break;
               }
              continue;
             }

            //---
            // Si hay =
            const ulong flag = TSNTABLES_SWAR_64_DEJAR_SOLO_MINUS_SIG_BYTE(has_x);
            m_pos += TSNTABLES_SWAR_64_GET_BYTE(flag);
            break;
           }
         }
        else
         {
          do
           {
            if(m_data[m_pos] == '=')
              break;
            m_pos++;
           }
          while(m_pos < m_len);
         }

        // Justo m_pos='='
        //-- Agregamos
        m_cinta[m_cinta_pos++] = long(SETFILE_TOK_KEY) | long(m_pos - start) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                 | long(start) << TSN_SETFILE_BIT_START_STR;

        //---
        m_root_c++; // Nuevo elemento
        m_prev_start = ++m_pos; // luego del =
        m_last_lock = m_cinta_pos++; // lock aqui
        m_cinta[m_last_lock] = SETFILE_TOK_META; // meta type
        m_pares = 0;

        //---
        while(true)
         {
          const uchar f = m_data[m_pos];
          if(f < 33)
           {
            if(f == '\n')
             {
              m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(0) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                       | long(start) << TSN_SETFILE_BIT_START_STR;
              m_cinta[m_last_lock] |= long(SETFILE_TOK_OP_TYPE_VAL) << TSN_SETFILE_BIT_END_TYPE;

              //---
              TSN_SETFILE_NEXT_V
              break;
             }


            m_pos++;
            if(m_pos >= m_len)
             {
              m_cinta[m_cinta_pos++] = long(SETFILE_TOK_STRING) | long(0) << TSN_SETFILE_BIT_START_LEN_PLAIN
                                       | long(start) << TSN_SETFILE_BIT_START_STR;
              m_cinta[m_last_lock] |= long(SETFILE_TOK_OP_TYPE_VAL) << TSN_SETFILE_BIT_END_TYPE;
              m_cinta[0] |= long(m_root_c) << TSN_SETFILE_BIT_END_TYPE;
              return true;
             }
            continue;
           }
          m_next = g_tsn_setfile_tokens[f];
          break;
         }
        break;
       }

      //---
      case SETFILE_TOKENS_INT_COMENTARIO:
       {
        m_pos++; // Siguiente..

        //---
        int curr_end = m_pos + 8;
        if(curr_end <= m_len)
         {
          while(true)
           {
            //Print(CharArrayToString(m_data, m_pos, 7));
            const ulong chunk = (ulong)m_data[m_pos]
                                | (ulong)m_data[m_pos + 1] << 8
                                | (ulong)m_data[m_pos + 2] << 16
                                | (ulong)m_data[m_pos + 3] << 24
                                | (ulong)m_data[m_pos + 4] << 32
                                | (ulong)m_data[m_pos + 5] << 40
                                | (ulong)m_data[m_pos + 6] << 48
                                | (ulong)m_data[m_pos + 7] << 56;

            //---
            const ulong x = chunk ^ TSNTABLES_SWAR_MASK_NEW_LINE;
            const ulong has_x = TSNTABLES_SWAR_HAS(x);
            if(has_x == 0)
             {
              m_pos = curr_end;
              curr_end += 8;

              //--- Check
              if(curr_end > m_len)
               {
                do
                 {
                  if(m_data[m_pos] == '\n')
                    break;
                  m_pos++;
                 }
                while(m_pos < m_len);
                break;
               }
              continue;
             }

            //---
            // Si hay \n
            const ulong flag = TSNTABLES_SWAR_64_DEJAR_SOLO_MINUS_SIG_BYTE(has_x);
            m_pos += TSNTABLES_SWAR_64_GET_BYTE(flag); // Ahora mismo en \n+1 \n[aqui]
            break;
           }
         }
        else
         {
          do
           {
            if(m_data[m_pos] == '\n')
              break;
            m_pos++;
           }
          while(m_pos < m_len);
         }

        //---
        m_pos++;
        // Si o si sigue key
        TSN_SETFILE_NEXT_V
        break;
       }

      //---
      default:
        m_last_err =  SETFILE_PARSER_ERROR_INVALID_TOKEN;
        return false;
     }
   }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CSetFileParser::PerfectHashComputeHash(const string& word) const
 {
//--- Hash 64
  ulong key_hash = FNV_OFFSET_BASIS;
  const int len = StringLen(word);
  for(int i = 0; i < len; i++)
   {
    key_hash ^= (uchar)word[i];
    key_hash *= FNV_PRIME;
   }

//----
  const int seed_index = int(key_hash % m_table_bucket_size);

//--- Alg
  ulong h = key_hash + m_table_seeds[seed_index] * 0x9e3779b97f4a7c15;
  h = (h ^ (h >> 30)) * 0xbf58476d1ce4e5b9;
  h = (h ^ (h >> 27)) * 0x94d049bb133111eb;
  h ^= (h >> 31);

//--- Pos (devuelve la posicion del KEY en la cinta)
  const int fi = int(h % m_table_final_table_size);
  return m_table_key_hash[fi] == key_hash ? m_table_pos_arr[m_table_pos_index[fi]] : -1;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CSetFileParser::PerfectHashComputeHashIndex(const string& word) const
 {
//--- Hash 64
  ulong key_hash = FNV_OFFSET_BASIS;
  const int len = StringLen(word);
  for(int i = 0; i < len; i++)
   {
    key_hash ^= (uchar)word[i];
    key_hash *= FNV_PRIME;
   }

//----
  const int seed_index = int(key_hash % m_table_bucket_size);

//--- Alg
  ulong h = key_hash + m_table_seeds[seed_index] * 0x9e3779b97f4a7c15;
  h = (h ^ (h >> 30)) * 0xbf58476d1ce4e5b9;
  h = (h ^ (h >> 27)) * 0x94d049bb133111eb;
  h ^= (h >> 31);

//--- Pos
  const int fi = int(h % m_table_final_table_size);
  return m_table_key_hash[fi] == key_hash ? m_table_pos_index[fi] : -1;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::NodeHashing(int curr, const int obj_count, const int bend)
 {
//---
  m_pfh.MaxValSeed(m_max_att++);
  m_pfh.InitAlg(obj_count, 0.75, fmin(obj_count, 4));

//---
  m_table_size = obj_count;
  ArrayResize(m_table_pos_arr, obj_count);

//---
  ulong keys_h[];
  int posiciones[];
  ArrayResize(keys_h, obj_count);
  ArrayResize(posiciones, obj_count);
  int pos_c = 0;
  curr++;

//---
  while(curr < bend)
   {
    //---
    m_table_pos_arr[pos_c] = curr; // Assing (posicion del KEY)

    //---
    ulong key_hash = FNV_OFFSET_BASIS;
    int i = int(m_cinta[curr] >> TSN_SETFILE_BIT_START_STR); // start
    const int len = i + int((m_cinta[curr] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF); // start+len

    //--- Hash 64
    for(; i < len; i++)
     {
      key_hash ^= m_data[i];
      key_hash *= FNV_PRIME;
     }

    //---
    keys_h[pos_c] = key_hash;
    posiciones[pos_c] = pos_c;
    pos_c++;

    //---
    const int meta = curr + 1;
    const int val_pos = meta + 1;
    const int val_step = GetStep(val_pos);
    const bool optimizable = int(m_cinta[meta] >> TSN_SETFILE_BIT_END_TYPE) != SETFILE_TOK_OP_TYPE_VAL;

    //---
    curr += 2 + val_step * (optimizable ? 4 : 1);
   }

//---
  m_table_bucket_size = m_pfh.m_buckets_size;
  const int fn = m_pfh.m_final_table_size;
  m_table_final_table_size = fn;

//--- Iniciamos..
  ArrayResize(m_table_pos_index, fn);
  ArrayResize(m_table_key_hash, fn);
  ArrayInitialize(m_table_pos_index, -1);
  ArrayInitialize(m_table_key_hash, 0ULL);

//---
  return (m_table_ready = m_pfh.RunWValue(keys_h, posiciones, m_table_seeds, m_table_pos_index, m_table_key_hash));
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CSetFileParser::AssingFile(const string& file_name, const bool comon_flag)
 {
//---
  const int reg_type = FileGetExtension(file_name) == "settc" ? TSNSETF_RI_SETTC_FILE : TSNSETF_RI_SET_FILE;

//---
  if(reg_type == TSNSETF_RI_SETTC_FILE) // settc tape cache
   {
    ::ResetLastError();
    const int fh = FileOpen(file_name, FILE_BIN | FILE_ANSI | (comon_flag ?  FILE_COMMON : 0) | FILE_READ);
    if(fh == INVALID_HANDLE)
     {
      return -1;
     }
    // Metadata (cinta)
    m_cinta_pos = FileReadInteger(fh); // Nota que cuando se guarda se hace con el tamaño exacto
    m_cinta_reserve = m_cinta_pos; // Mismo tamaño
    const int type = FileReadInteger(fh); // Tipo ()

    //---
    if(type == TSN_SETFPARSER_SETTC_TYPE_EMBEBED)
     {
      m_len = FileReadInteger(fh);
      m_offset = FileReadInteger(fh);
      FileReadArray(fh, m_cinta, 0, m_cinta_pos); // Total
      FileReadArray(fh, m_data, 0, m_len);
      LoadTables(fh); // Perfect hash tables
     }
    else // Dir
     {
      const bool common = bool(FileReadInteger(fh, CHAR_VALUE));
      const int file_name_len = FileReadInteger(fh);
      AssingFile(FileReadString(fh, file_name_len), common); // Carga
      FileReadArray(fh, m_cinta, 0, m_cinta_pos);
      LoadTables(fh); // Perfect hash tables
     }

    //---
    FileClose(fh);
    return TSNSETF_RI_SETTC_FILE;
   }
  else // set normal
   {
    m_len = (int)FileLoad(file_name, m_data, (comon_flag ? FILE_COMMON : 0));
    m_offset = 0;

    //---
    if(m_len >= 2)
     {
      // UTF16 BE
      if(m_data[0] == 0xFF && m_data[1] == 0xFE)
       {
        const int n = (m_len - 2) >> 1;
        for(int i = 0; i < n; i++)
          m_data[i] = m_data[2 + (i * 2)]; // byte bajo (el char real)
        m_len = ArrayResize(m_data, n);
       }
      // UTF16 LE
      else
        if(m_data[0] == 0xFE && m_data[1] == 0xFF)
         {
          const int n = (m_len - 2) >> 1;
          for(int i = 0; i < n; i++)
            m_data[i] = m_data[3 + (i * 2)]; // byte alto (invertido)
          m_len = ArrayResize(m_data, n);
         }
     }
    else
      if(m_len >= 3 && m_data[0] == 0xEF && m_data[1] == 0xBB && m_data[2] == 0xBF)
       {
        // UTF-8 BOM
        m_offset = 3; // simplemente saltear
       }

    //---
    return reg_type;
   }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CSetFileParser::AutoParseByFileExt(const int file_ext_type)
 {
  switch(file_ext_type)
   {
    case TSNSETF_RI_SET_FILE:
     {
      return Parse();
     }
    case TSNSETF_RI_SETTC_FILE:
     {
      return m_cinta_pos > 0 && m_len > 0;
     }
    default:
     {
      m_last_err = SETFILE_PARSER_ERROR_INVALID_EXT_FILE;
      m_pos = 0;
      return false;
     }
   }
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
//--- Ambos
// 4 bytes: m_cinta_pos
// 4 bytes: type

//---
bool CSetFileParser::SaveCompiled(const string & file_name_out, const bool out_common_flag) const
 {
  ::ResetLastError();
  const int fh = FileOpen(file_name_out, FILE_BIN | (out_common_flag ?  FILE_COMMON : 0) | FILE_WRITE);
  if(fh == INVALID_HANDLE)
   {
    return false;
   }

//--- Comun
  FileWriteInteger(fh, m_cinta_pos);
  FileWriteInteger(fh, TSN_SETFPARSER_SETTC_TYPE_EMBEBED);

//---
// 4 bytes: len
// 4 bytes: offset
// array: cinta
// array: data

//---
  FileWriteInteger(fh, m_len);
  FileWriteInteger(fh, m_offset);
  FileWriteArray(fh, m_cinta, 0, m_cinta_pos);
  FileWriteArray(fh, m_data, 0, m_len);

//--- Perfect hash tables (m_tables)
  SaveTables(fh);

//---
  FileClose(fh);

//---
  return true;
 }

//+------------------------------------------------------------------+
bool CSetFileParser::SaveCompiled(const string & file_name_out, const bool out_common_flag, const string & file_name_set, const bool json_common_flag) const
 {
//---
  ::ResetLastError();
  const int fh = FileOpen(file_name_out, FILE_BIN | (out_common_flag ?  FILE_COMMON : 0) | FILE_WRITE);
  if(fh == INVALID_HANDLE)
   {
    return false;
   }

//--- Comun
  FileWriteInteger(fh, m_cinta_pos);
  FileWriteInteger(fh, TSN_SETFPARSER_SETTC_TYPE_FILE);

//---
// 1 bytes: comon
// 4 bytes: len
// string: name_file
// array: cinta

//---
  FileWriteInteger(fh, json_common_flag, CHAR_VALUE);
  const int len = StringLen(file_name_set);
  FileWriteInteger(fh, len);
  FileWriteString(fh, file_name_set, len);
  FileWriteArray(fh, m_cinta, 0, m_cinta_pos);

//--- Perfect hash tables (m_tables)
  SaveTables(fh);

//---
  FileClose(fh);

//---
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CSetFileParser::SaveTables(const int fh) const
 {
//---
  FileWriteInteger(fh, m_table_ready ? 1 : 0);
  if(!m_table_ready)
    return;

//---
  const int bucket_size = (int)m_table_bucket_size;
  const int final_size = (int)m_table_final_table_size;

  FileWriteInteger(fh, m_table_size);
  FileWriteInteger(fh, bucket_size);
  FileWriteInteger(fh, final_size);

//---
  FileWriteArray(fh, m_table_seeds, 0, bucket_size);
  FileWriteArray(fh, m_table_pos_index, 0, final_size);
  FileWriteArray(fh, m_table_key_hash, 0, final_size);
  FileWriteArray(fh, m_table_pos_arr, 0, m_table_size);
 }

//+------------------------------------------------------------------+
void CSetFileParser::LoadTables(const int fh)
 {
//---
  m_table_ready = FileReadInteger(fh) != 0;
  if(!m_table_ready)
   {
    m_table_size = 0;
    return;
   }

//---
  m_table_size = FileReadInteger(fh);
  const int bucket_size = FileReadInteger(fh);
  m_table_bucket_size = ulong(bucket_size);
  const int final_size = FileReadInteger(fh);
  m_table_final_table_size = ulong(final_size);

//---
  FileReadArray(fh, m_table_seeds, 0, bucket_size);
  FileReadArray(fh, m_table_pos_index, 0, final_size);
  FileReadArray(fh, m_table_key_hash, 0, final_size);
  FileReadArray(fh, m_table_pos_arr, 0, m_table_size);
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CSetFileParser::PrintCintaTypes(const int start, const int num) const
 {
//---
  int p = fmax(start, 0);
  const int end = (num == WHOLE_ARRAY || (p + num) > m_cinta_pos) ? m_cinta_pos : (p + num);

//---
  while(p < end)
   {
    const ENUM_SETFILE_TOKENS tipo = ENUM_SETFILE_TOKENS(m_cinta[p] & 0xF);
    string text = "[" + (string)p + "] " + EnumToString(tipo) + " | ";

    //---
    switch(tipo)
     {
      case SETFILE_TOK_ROOT:
       {
        const int count = int((m_cinta[p] >> TSN_SETFILE_BIT_END_TYPE) & TSN_SETFILE_MASK_COUNT);
        const int jit_try = int(m_cinta[p] >> TSN_SETFILE_BIT_START_JIT_TRY);
        text += "count=" + (string)count + " jit_try=" + (string)jit_try;
        break;
       }
      case SETFILE_TOK_INTEGER:
        text += "val=" + (string)m_cinta[p + 1];
        break;
      case SETFILE_TOK_REAL:
       {
        static BitInterpreter un;
        un.long_value = m_cinta[p + 1];
        text += "val=" + (string)un.double_value;
        break;
       }
      case SETFILE_TOK_BOOLEAN:
        text += "val=" + (string)(bool)((m_cinta[p] >> TSN_SETFILE_BIT_END_TYPE) & 1);
        break;
      case SETFILE_TOK_STRING:
       {
        const int s = int(m_cinta[p] >> TSN_SETFILE_BIT_START_STR);
        const int slen = int((m_cinta[p] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF);
        text += "start=" + (string)s + " len=" + (string)slen + " val=\"" + CharArrayToString(m_data, s, slen) + "\"";
        break;
       }
      case SETFILE_TOK_KEY:
       {
        const int s = int(m_cinta[p] >> TSN_SETFILE_BIT_START_STR);
        const int slen = int((m_cinta[p] >> TSN_SETFILE_BIT_START_LEN_PLAIN) & 0xFFFFFFF);
        text += "start=" + (string)s + " len=" + (string)slen + " key=\"" + CharArrayToString(m_data, s, slen) + "\"";
        break;
       }
      case SETFILE_TOK_META:
       {
        const int op = int(m_cinta[p] >> TSN_SETFILE_BIT_END_TYPE);
        text += "op_type=" + EnumToString(ENUM_SETFILE_TOK_OP_TYPE(op));
        break;
       }
     }

    Print(text);
    p += GetStep(p);
   }
 }

//---
}
//+------------------------------------------------------------------+
#endif // SETFILEBYLEO_SRC_PARSER_MQH