ExpressEvalByLeo/Src/MathEval/Tokenizer.mqh

//+------------------------------------------------------------------+
//|                                                    Tokenizer.mqh |
//|                                  Copyright 2026, Niquel Mendoza. |
//|                          https://www.mql5.com/es/users/nique_372 |
//+------------------------------------------------------------------+
#property copyright "Copyright 2026, Niquel Mendoza."
#property link      "https://www.mql5.com/es/users/nique_372"
#property strict

#ifndef EXPRESSEVALBYLEO_SRC_MATHEVAL_TOKENIZER_MQH
#define EXPRESSEVALBYLEO_SRC_MATHEVAL_TOKENIZER_MQH

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "Defines.mqh"

// @op(10 + 20  +  )
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
struct CMathExpTokenizer : public CTokenizerBase
 {
private:
  ENUM_BINARY_AST_MATH_ERR m_last_err;

public:
                     CMathExpTokenizer(void) : m_last_err(WRONG_VALUE) {}
                   // ~CMathExpTokenizer(void) {}

  //---
  bool               Tokenize(const string& text, TokenMathExp& tokns[]);

  //---
  __forceinline ENUM_BINARY_AST_MATH_ERR LastError() const { return m_last_err; }
  string             FormatLastError(const string& text) const;

  //---
  static void        PrintValues(const TokenMathExp& tokens[]);
 };

//+------------------------------------------------------------------+
bool CMathExpTokenizer::Tokenize(const string &text, TokenMathExp &tokns[])
 {
//---
  m_len = StringLen(text);
  m_pos = 0;
 // m_err_pos = 0;
  m_last_err = WRONG_VALUE;

//---
  int reserve_size = 16;
  ArrayResize(tokns, reserve_size, reserve_size);
  int current_size = 0;

//---
  while(m_pos < m_len)
   {
    //--- Saltamos espacios
    if(text[m_pos] < 33)
     {
      m_pos++;
      continue;
     }

    //---
    switch(text[m_pos])
     {
      //---
      case '#':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_FUNCTION_CUSTOM_CALL;
        if(!CGenericParser::ParseFunction(text, m_len, m_pos, (int&)m_last_err, m_err_pos, tokns[current_size]))
         {
          return false;
         }
        AST_OPS_CHECK_RESIZE

        //---
        break;
       }

      //---
      case '$':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_VARIABLE;
        CGenericParser::ParseVariable(text, m_len, m_pos, (int&)m_last_err, m_err_pos, tokns[current_size]);
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '%':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_MODULO) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }


      //---
      case '&':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_BIT_AND) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        //---
        break;
       }


      //--- Inicio de paraenteisis
      case '(':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_PARA_INI;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //--- Final de parentenisis
      case ')':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_PARA_END;
        AST_OPS_CHECK_RESIZE
        break;
       }


      //---
      case '*':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_MUL) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '+':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_SUMA) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '-':
       {
        //---
        if(m_pos + 1 >= m_len)
         {
          m_last_err = BINARY_AST_TOKENIZER_ERR_SIGNO_MENOS_EN_FINAL_LEN;
          m_err_pos = m_pos - 1;
          return false;
         }

        //---
        if(IsDigit(text[m_pos + 1])) // -X (X es un numero)
         {
          tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_NUMBER;
          if(!CGenericParser::ParseNumberNoHex(text, m_len, m_pos, (int&)m_last_err, m_err_pos, tokns[current_size], m_pos++))
            return false;
         }
        else
         {
          // Ahora mismo estamos en - (siguien char el que le sigue)
          tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_RESTA) << AST_LOGIC_EXTRA_TYPE_START_BIT;
         }

        //---
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '/':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_DIVICION) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '<':
       {
        m_pos++;
        if(m_pos >= m_len)
         {
          m_last_err = BINARY_AST_TOKENIZER_ERR_DEZ_IZQ_INVALIDO;
          m_err_pos = m_pos - 1;
          return false;
         }
        else
          if(text[m_pos] != '<')
           {
            m_last_err = BINARY_AST_TOKENIZER_ERR_DEZ_IZQ_SE_ESPERABA_EL_SIG_FALTANTE;
            m_err_pos = m_pos - 1;
            return false;
           }

        //---
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_BIT_DEZPLAZAMIENTO_IZQ) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '>':
       {
        m_pos++;
        if(m_pos >= m_len)
         {
          m_last_err = BINARY_AST_TOKENIZER_ERR_DEZ_DRC_INVALIDO;
          m_err_pos = m_pos - 1;
          return false;
         }
        else
          if(text[m_pos] != '>')
           {
            m_last_err = BINARY_AST_TOKENIZER_ERR_DEZ_DRC_SE_ESPERABA_EL_SIG_FALTANTE;
            m_err_pos = m_pos - 1;
            return false;
           }

        //---
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_BIT_DEZPLAZAMIENTO_DRC) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '@':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_FUNCTION_BUIL_IN_CALL;
        if(!CGenericParser::ParseFunction(text, m_len, m_pos, (int&)m_last_err, m_err_pos, tokns[current_size]))
         {
          return false;
         }
        AST_OPS_CHECK_RESIZE

        break;
       }

      //---
      case '^':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_BIT_XOR) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      case '|':
       {
        tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_OPERDOR_MATEMATICO | int(BINARY_NODE_OP_BIT_OR) << AST_LOGIC_EXTRA_TYPE_START_BIT;
        AST_OPS_CHECK_RESIZE
        break;
       }

      //---
      //---
      default:
       {
        if(IsDigit(text[m_pos]))
         {
          tokns[current_size].type = AST_TOKEN_MATH_T_VALUE_NUMBER;
          if(!CGenericParser::ParseNumber(text, m_len, m_pos, (int&)m_last_err, m_err_pos, tokns[current_size]))
            return false;

          //---
          AST_OPS_CHECK_RESIZE
          break;
         }
        else
         {
          m_last_err = BINARY_AST_TOKENIZER_ERR_INVALID_CHAR;
          m_err_pos = m_pos;
          return false;
         }
       }
     }

    //--- Siguiente
    m_pos++;
   }

//---
  ArrayResize(tokns, current_size);

//---
  return true;
 }

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
string CMathExpTokenizer::FormatLastError(const string& text) const
 {
//---
  if(m_last_err == WRONG_VALUE)
   {
    return "Not errors found";
   }

//---
  const uint r_err = m_last_err & 0xFFFFFF;
  const int type = m_last_err >> AST_LOGIC_ERR_TYPE_START_BIT;

//---
  int line = 0, col = 0;
  GetCharPosLocation(m_err_pos, text, col, line);

//---
  string error_msg = "";

//---
  if(type == AST_LOGIC_ERR_TYPE_EXTERNAL)
   {
    CGenericParser::FormatErr(r_err, error_msg);
   }
  else
   {
    switch(r_err)
     {
      case BINARY_AST_TOKENIZER_ERR_DEZ_IZQ_INVALIDO:
        error_msg = "Invalid left shift operator '<', expected '<<'";
        break;

      case BINARY_AST_TOKENIZER_ERR_DEZ_IZQ_SE_ESPERABA_EL_SIG_FALTANTE:
        error_msg = "Left shift operator missing second '<', expected '<<'";
        break;

      case BINARY_AST_TOKENIZER_ERR_DEZ_DRC_INVALIDO:
        error_msg = "Invalid right shift operator '>', expected '>>'";
        break;

      case BINARY_AST_TOKENIZER_ERR_DEZ_DRC_SE_ESPERABA_EL_SIG_FALTANTE:
        error_msg = "Right shift operator missing second '>', expected '>>'";
        break;

      case BINARY_AST_TOKENIZER_ERR_SIGNO_MENOS_EN_FINAL_LEN:
        error_msg = "Minus sign '-' found at end of expression";
        break;

      case BINARY_AST_TOKENIZER_ERR_INVALID_CHAR:
        error_msg = "Invalid character in expression";
        break;

      default:
        error_msg = "Unknown error";
        break;
     }
   }

//---
  return error_msg + StringFormat(" [Line %d, Col %d]", line, col);
 }


//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
static void CMathExpTokenizer::PrintValues(const TokenMathExp &tokens[])
 {
  const int t = ArraySize(tokens);
  Print(" TYPE | VALUE");
  for(int i = 0; i < t; i++)
   {
    const int v = tokens[i].type & 0xFFFF;
    PrintFormat("%s | %s", EnumToString(ENUM_AST_TOKEN_MATH_TYPE(v)), tokens[i].vs);
   }
 }
//+------------------------------------------------------------------+
#endif // EXPRESSEVALBYLEO_SRC_MATHEVAL_TOKENIZER_MQH