//+------------------------------------------------------------------+ //| Parser.cpp | //| Copyright 2026, Niquel Mendoza. | //| https://www.mql5.com/ | //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "Parser.h" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ // array ya allocado tambein raw solo hace falta rellenar using namespace TSN; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ // Solo lo resizea.. y maneja el tema de reserva.. nada mas.. (en m_index) // m_raw ya se maneja por mql // len tambien // m_last err tamiben se inicia // m_cintapos tambien se incia por qml // pos no asi que lo hacemos aqui // offsert se nos pasa // cinta tambien se nos pasa pero lo contrala mql // index se nos pasa nosotrs oslo modifcamos extern "C" __declspec(dllexport) bool __stdcall JsonParserResolve(const uint8_t* m_raw, const int len, int* m_last_err, int* m_cinta_pos, int* m_pos, const int m_offset, int64_t* m_cinta, int* m_index) { //--- Stacks uint32_t m_stak_num[JSONPARSERBYLEO_MAX_STACK]; int32_t m_stak_pos[JSONPARSERBYLEO_MAX_STACK]; uint16_t m_stak_state[JSONPARSERBYLEO_MAX_STACK]; int32_t m_stak_curr = -1; uint16_t m_stak_curr_state = 0; uint8_t m_next = TSN_JSON_TOK_INVALID; //--- Inicio de miembros (*m_pos) = m_offset; //--- Temp ulong prev_in_string = 0ULL; ulong prev_scape = 0ULL; int index_s = 0; //--- Intrisicos masks const __m256i mask_20 = _mm256_set1_epi8(0x20); const __m256i bs_mask = _mm256_set1_epi8('\\'); const __m256i qt_mask = _mm256_set1_epi8('"'); const __m128i all_ones = _mm_set1_epi8(0xFF); const __m256i op_table = _mm256_broadcastsi128_si256(_mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ':', '{', ',', '}', 0, 0 )); //--- Fase inicial do { // carga inicial __m256i chunk_lo = _mm256_loadu_si256((const __m256i*)(m_raw + (*m_pos))); // bytes 0-31 __m256i chunk_hi = _mm256_loadu_si256((const __m256i*)(m_raw + (*m_pos) + 32)); // bytes 32-63 //--- Detectamos bs const uint32_t bs_bits_lo = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_lo, bs_mask)); const uint32_t bs_bits_hi = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_hi, bs_mask)); //--- Detectamos qt const uint32_t qt_bits_lo = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_lo, qt_mask)); const uint32_t qt_bits_hi = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_hi, qt_mask)); //--- // compactamos ambos const ulong compacted_bs = ((ulong)bs_bits_hi << 32) | bs_bits_lo; const ulong compacted_qt = ((ulong)qt_bits_hi << 32) | qt_bits_lo; //--- const ulong compacted_bs_masked = compacted_bs & ~prev_scape; const ulong prev = (((compacted_bs_masked << 1) | TSNTABLES_MASK_IMPARES_F) - compacted_bs_masked) ^ TSNTABLES_MASK_IMPARES_F; const ulong escaped = prev ^ (compacted_bs | prev_scape); prev_scape = (prev & compacted_bs_masked) >> 63; //--- ulong qt_final = compacted_qt & ~escaped; const ulong quoted_pos = qt_final; //--- prefix xor __m128i clmul = _mm_clmulepi64_si128(_mm_set_epi64x(0, qt_final), all_ones, 0); qt_final = (uint64_t)_mm_cvtsi128_si64(clmul); const ulong in_string = qt_final ^ prev_in_string; //--- Ahora obtenemos los strucutrurales // ahora lo qeu haremos sera un or entre la masra de 0x20 // La idea econ esto es que (en las siugientes) lineas hay un problema // noramlte con shuffle si tenemos lo sigueinte {} // se genera // {} (tal cual bien) // pero si tenemos // [] se genra {} (esto es por que se genera eso en base a los bajos) // entonces esto luego con el cmp que se hace estair ocmprand [ con { lo qeu daria 0 asi qeu se perderia // los caracteres estrucutalres, por eso le debemos de aumenar 0x20, esto hace que [ pase a { // con eso soluicionas dicho problema.. __m256i curl_lo = _mm256_or_si256(chunk_lo, mask_20); __m256i curl_hi = _mm256_or_si256(chunk_hi, mask_20); // ahora usarmeos shuffle para hacer lo sigeuinte // : = 1010 // ] y } = 1011 // , = 1100 // Como para generar strucuturales rapdios, la idea es quedarse solo con los estrucuturales // Aparte metemos un eq para confirmar.. pro qeu ahroa mismo la mascara previa // peude dar falsso posiivots (dado qeu eso de 1010... puede repetirse) __m256i op_lo = _mm256_cmpeq_epi8(_mm256_shuffle_epi8(op_table, chunk_lo), curl_lo); __m256i op_hi = _mm256_cmpeq_epi8(_mm256_shuffle_epi8(op_table, chunk_hi), curl_hi); // ahora lo basmoas a 1\0 no bytes.. para trabajar.. uint32_t bits_lo = (uint32_t)_mm256_movemask_epi8(op_lo); uint32_t bits_hi = (uint32_t)_mm256_movemask_epi8(op_hi); uint64_t result = ((uint64_t)bits_hi << 32) | bits_lo; //--- overflow prev_in_string = uint64_t(static_cast(in_string) >> 63); ulong structural = (result & ~in_string) | quoted_pos; //--- Numero de unos aqui.. const int cnt = _mm_popcnt_u64(structural); int written = 0; //--- while(written + 4 <= cnt) { m_index[index_s++] = (*m_pos) + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = (*m_pos) + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = (*m_pos) + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = (*m_pos) + _tzcnt_u64(structural); structural &= structural - 1; written += 4; } // Resto (0-3 elementos sobrantes), bucle normal while(written < cnt) { m_index[index_s++] = (*m_pos) + _tzcnt_u64(structural); structural &= structural - 1; written++; } (*m_pos) += 64; // avanzamos 64 caracteres.. // Seguimos con check } while(((*m_pos) + 64) < len); //--- if(index_s < 2) { // errro *m_last_err = TSN_JSON_ERR_NOT_STRUCUTURAL_CHARS_FOUND; return false; } //--- Fase 2 int sp = 0; (*m_pos) = m_index[sp]; // Posicion m_next = g_table_json_tokens[m_raw[(*m_pos)]]; // primer //--- Iteracion while(true) { switch(m_next) { case TSN_JSON_TOK_NUMBER_NEG: { (*m_pos)++; JSONPARSERBYLEO_PARSE_NUMBER(-1) //--- JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_NUMBER: { //--- JSONPARSERBYLEO_PARSE_NUMBER(0) //--- JSONPARSERBYLEO_FULL_NEXT break; } //--- case TSN_JSON_TOK_STRING: { //--- //TSN_JSON_RESERVAR(2) //--- const int start = m_index[sp++] + 1; // luego del " (*m_pos) = m_index[sp++]; // justo en " //--- m_cinta[(*m_cinta_pos)++] = JSON_VTYPE_STRING | int64_t((*m_pos) - start) << TSN_SBL_BIT_STR_LEN | int64_t(start) << TSN_SBL_BIT_STR_START; //--- JSONPARSERBYLEO_FULL_NEXT //--- break; } case TSN_JSON_TOK_BOOL_TRUE: { //TSN_JSON_RESERVAR(1) // t r u e W // [-4] [-3] [-2] [-1] [0] (*m_pos) += 4; // 5 | 1 << TSN_SBL_BIT_START_BOOL(4) = 21 m_cinta[(*m_cinta_pos)++] = 21; //--- JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_BOOL_FALSE: { //TSN_JSON_RESERVAR(1) // f a l s e W // [] [-4] [-3] [-2 ] [-1] [0] (*m_pos) += 5; m_cinta[(*m_cinta_pos)++] = JSON_VTYPE_BOOLEAN; //--- JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_NULL: { //TSN_JSON_RESERVAR(1) // n u l l W // [-4][-3][-2][-1][] //--- (*m_pos) += 4; m_cinta[(*m_cinta_pos)++] = JSON_VTYPE_NULL; //--- JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_COMMA: { //--- m_stak_num[m_stak_curr]++; //--- (*m_pos)++; //--- if(m_stak_curr_state == TSN_JSON_CTX_IN_OBJ) { //--- const int start = m_index[sp++] + 1; // luego del " (*m_pos) = m_index[sp++]; // justo en " //Print("start: " , start, " len: " , ((*m_pos) - start)); //--- m_cinta[(*m_cinta_pos)++] = (int64_t)JSON_VTYPE_KEY | int64_t((*m_pos) - start) << TSN_SBL_BIT_STR_LEN | int64_t(start) << TSN_SBL_BIT_STR_START; // Print(EnumToString(ENUM_JSON_VTYPE(m_cinta[(*m_cinta_pos) - 1] & 0xf))); //--- (*m_pos) = m_index[sp++]; if(m_raw[(*m_pos)++] != ':') { *m_last_err = TSN_JSON_ERR_MALFODERD_KEY_EXPECTED_DOS_PUNTOS; return false; } // Ahora si buscamos valor } TSN_JSON_NEXT_VALUE_COMM_KEY break; } case TSN_JSON_TOK_LLAVE_INI: { //--- sp++; //TSN_JSON_RESERVAR(1) m_cinta[(*m_cinta_pos)] = JSON_VTYPE_OBJ; //--- m_stak_curr++; if(m_stak_curr >= JSONPARSERBYLEO_MAX_STACK) { *m_last_err = TSN_JSON_ERR_OVERFLOW_IN_STACK; return false; // overflow } //--- m_stak_curr_state = TSN_JSON_CTX_IN_OBJ; m_stak_state[m_stak_curr] = TSN_JSON_CTX_IN_OBJ; m_stak_num[m_stak_curr] = 0; m_stak_pos[m_stak_curr] = (*m_cinta_pos)++; //--- m_cinta[(*m_cinta_pos)++] = (*m_pos)++; //--- (*m_pos) = m_index[sp++]; if(m_raw[(*m_pos)] == '}') { m_next = TSN_JSON_TOK_LLAVE_END; } else { const int start = ++(*m_pos); (*m_pos) = m_index[sp++]; // justo en " //--- Capturamos m_cinta[(*m_cinta_pos)++] = JSON_VTYPE_KEY | int64_t((*m_pos) - start) << TSN_SBL_BIT_STR_LEN | int64_t(start) << TSN_SBL_BIT_STR_START; //--- (*m_pos) = m_index[sp++]; if(m_raw[(*m_pos)++] != ':') { *m_last_err = TSN_JSON_ERR_MALFODERD_KEY_EXPECTED_DOS_PUNTOS; return false; } // Ahora a buscar valor TSN_JSON_NEXT_VALUE_COMM_KEY } //--- break; } case TSN_JSON_TOK_COR_INI: { //--- sp++; //TSN_JSON_RESERVAR(1) m_cinta[(*m_cinta_pos)] = JSON_VTYPE_ARR; //--- m_stak_curr++; if(m_stak_curr >= JSONPARSERBYLEO_MAX_STACK) { *m_last_err = TSN_JSON_ERR_OVERFLOW_IN_STACK; return false; // overflow } //--- m_stak_curr_state = TSN_JSON_CTX_IN_ARR; m_stak_state[m_stak_curr] = TSN_JSON_CTX_IN_ARR; m_stak_num[m_stak_curr] = 0; m_stak_pos[m_stak_curr] = (*m_cinta_pos)++; //--- m_cinta[(*m_cinta_pos)++] = (*m_pos)++; //Print(CharToString(m_raw[(*m_pos)])); //--- const int n = m_index[sp]; if(m_raw[n] != ']') { TSN_JSON_NEXT_COR_INI } else { sp++; (*m_pos) = n; m_next = TSN_JSON_TOK_LLAVE_END; } //--- break; } // tema de ends ya lo scaa en next el sp case TSN_JSON_TOK_LLAVE_END: case TSN_JSON_TOK_COR_END: { //--- //sp++; //--- if((*m_cinta_pos) > m_stak_pos[m_stak_curr] + 2) // [a][a][v][c] { m_stak_num[m_stak_curr]++; } //--- const int lp = m_stak_pos[m_stak_curr]; m_cinta[lp] |= int64_t(m_stak_num[m_stak_curr]) << TSN_SBL_BIT_START_NUM_EL | int64_t((*m_cinta_pos) - lp) << TSN_SBL_BIT_START_NUM_C; m_cinta[lp + 1] |= int64_t((*m_pos)) << TSN_JSON_BIT_END_T; //--- //m_stak_curr--; //--- if(--m_stak_curr < 0) return true; //--- m_stak_curr_state = m_stak_state[m_stak_curr]; //--- (*m_pos)++; //--- JSONPARSERBYLEO_FULL_NEXT //--- break; } //--- default: { *m_last_err = TSN_JSON_ERR_INVALID_CHAR; return false; } } } //--- *m_last_err = TSN_JSON_ERR_JSON_NOT_FINISH; return false; } //+------------------------------------------------------------------+