//+------------------------------------------------------------------+ //| Parser.cpp | //| Copyright 2026, Niquel Mendoza. | //| https://www.mql5.com/ | //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "Parser.h" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ // array ya allocado tambein raw solo hace falta rellenar using namespace TSN; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ extern "C" __declspec(dllexport) // array ya allocado tambein raw solo hace falta rellenar bool __stdcall JsonParserResolve( const uint8_t* m_raw, // datos de entrada int64_t* m_cinta, // cinta de datos int* m_index, // array de indices structurales const int len, // tamaño de raw const int m_offset, // offset de inicio para pos //--- salida int* m_last_err, // ultimo err int* m_cinta_pos, // posicion actul en la cinta int* m_pos // ultima posicion en la lecutra de raw ) // retorna true en caso de exito, false de lo contrario.. { //--- Stacks uint32_t m_stak_num[JSONPARSERBYLEO_MAX_STACK]; int32_t m_stak_pos[JSONPARSERBYLEO_MAX_STACK]; uint16_t m_stak_state[JSONPARSERBYLEO_MAX_STACK]; int32_t m_stak_curr = -1; uint16_t m_stak_curr_state = 0; //--- Inicio de miembros int pos = m_offset; // posicion de lecutra int cinta_pos = 0; // pocision en la cinta //--- Para string y structurales ulong prev_in_string = 0ULL; ulong prev_scape = 0ULL; int index_s = 0; //--- Intrisicos masks const __m256i mask_20 = _mm256_set1_epi8(0x20); const __m256i bs_mask = _mm256_set1_epi8('\\'); const __m256i qt_mask = _mm256_set1_epi8('"'); const __m128i all_ones = _mm_set1_epi8(0xFF); const __m256i op_table = _mm256_broadcastsi128_si256(_mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ':', '{', ',', '}', 0, 0 )); //--- Fase inicial do { // carga inicial __m256i chunk_lo = _mm256_loadu_si256((const __m256i*)(m_raw + pos)); // bytes 0-31 __m256i chunk_hi = _mm256_loadu_si256((const __m256i*)(m_raw + pos + 32)); // bytes 32-63 //--- Detectamos bs const uint32_t bs_bits_lo = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_lo, bs_mask)); const uint32_t bs_bits_hi = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_hi, bs_mask)); //--- Detectamos qt const uint32_t qt_bits_lo = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_lo, qt_mask)); const uint32_t qt_bits_hi = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk_hi, qt_mask)); //--- // compactamos ambos const ulong compacted_bs = ((ulong)bs_bits_hi << 32) | bs_bits_lo; const ulong compacted_qt = ((ulong)qt_bits_hi << 32) | qt_bits_lo; //--- const ulong compacted_bs_masked = compacted_bs & ~prev_scape; const ulong prev = (((compacted_bs_masked << 1) | TSNTABLES_MASK_IMPARES_F) - compacted_bs_masked) ^ TSNTABLES_MASK_IMPARES_F; const ulong escaped = prev ^ (compacted_bs | prev_scape); prev_scape = (prev & compacted_bs_masked) >> 63; //--- ulong qt_final = compacted_qt & ~escaped; const ulong quoted_pos = qt_final; //--- prefix xor __m128i clmul = _mm_clmulepi64_si128(_mm_set_epi64x(0, qt_final), all_ones, 0); qt_final = (uint64_t)_mm_cvtsi128_si64(clmul); const ulong in_string = qt_final ^ prev_in_string; //--- Ahora obtenemos los strucutrurales // ahora lo qeu haremos sera un or entre la masra de 0x20 // La idea econ esto es que (en las siugientes) lineas hay un problema // noramlte con shuffle si tenemos lo sigueinte {} // se genera // {} (tal cual bien) // pero si tenemos // [] se genra {} (esto es por que se genera eso en base a los bajos) // entonces esto luego con el cmp que se hace estair ocmprand [ con { lo qeu daria 0 asi qeu se perderia // los caracteres estrucutalres, por eso le debemos de aumenar 0x20, esto hace que [ pase a { // con eso soluicionas dicho problema.. __m256i curl_lo = _mm256_or_si256(chunk_lo, mask_20); __m256i curl_hi = _mm256_or_si256(chunk_hi, mask_20); // ahora usarmeos shuffle para hacer lo sigeuinte // : = 1010 // ] y } = 1011 // , = 1100 // Como para generar strucuturales rapdios, la idea es quedarse solo con los estrucuturales // Aparte metemos un eq para confirmar.. pro qeu ahroa mismo la mascara previa // peude dar falsso posiivots (dado qeu eso de 1010... puede repetirse) __m256i op_lo = _mm256_cmpeq_epi8(_mm256_shuffle_epi8(op_table, chunk_lo), curl_lo); __m256i op_hi = _mm256_cmpeq_epi8(_mm256_shuffle_epi8(op_table, chunk_hi), curl_hi); // ahora lo basmoas a 1\0 no bytes.. para trabajar.. uint32_t bits_lo = (uint32_t)_mm256_movemask_epi8(op_lo); uint32_t bits_hi = (uint32_t)_mm256_movemask_epi8(op_hi); uint64_t result = ((uint64_t)bits_hi << 32) | bits_lo; //--- overflow prev_in_string = uint64_t(static_cast(in_string) >> 63); ulong structural = (result & ~in_string) | quoted_pos; //--- Numero de unos aqui.. const int cnt = _mm_popcnt_u64(structural); int written = 0; //--- while(written + 4 <= cnt) { m_index[index_s++] = pos + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = pos + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = pos + _tzcnt_u64(structural); structural &= structural - 1; m_index[index_s++] = pos + _tzcnt_u64(structural); structural &= structural - 1; written += 4; } // Resto (0-3 elementos sobrantes), bucle normal while(written < cnt) { m_index[index_s++] = pos + _tzcnt_u64(structural); structural &= structural - 1; written++; } pos += 64; // avanzamos 64 caracteres.. // Seguimos con check } while(pos < len); //--- if(index_s < 2) { *m_last_err = TSN_JSON_ERR_NOT_STRUCUTURAL_CHARS_FOUND; TSN_JSON_ON_FINISH return false; } //--- Fase 2 int sp = 0; // posicion acutal en los indices pos = m_index[sp]; // Posicion uint8_t m_next = g_table_json_tokens[m_raw[pos]]; // siguiente token //--- Iteracion while(true) { switch(m_next) { case TSN_JSON_TOK_NUMBER_NEG: { pos++; JSONPARSERBYLEO_PARSE_NUMBER(-) //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_NUMBER: { //--- JSONPARSERBYLEO_PARSE_NUMBER(+) //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_STRING: { //--- const int start = m_index[sp++] + 1; // luego del " pos = m_index[sp++]; // justo en " //--- m_cinta[cinta_pos++] = JSON_VTYPE_STRING | int64_t(pos - start) << TSN_SBL_BIT_STR_LEN | int64_t(start) << TSN_SBL_BIT_STR_START; //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_BOOL_TRUE: { // t r u e W // [-4] [-3] [-2] [-1] [0] pos += 4; // 5 | 1 << TSN_SBL_BIT_START_BOOL(4) = 21 m_cinta[cinta_pos++] = 21; //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_BOOL_FALSE: { // f a l s e W // [] [-4] [-3] [-2 ] [-1] [0] pos += 5; m_cinta[cinta_pos++] = JSON_VTYPE_BOOLEAN; //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_NULL: { // n u l l W // [-4][-3][-2][-1][] //--- pos += 4; m_cinta[cinta_pos++] = JSON_VTYPE_NULL; //--- TSN_JSON_AUMENT JSONPARSERBYLEO_FULL_NEXT break; } case TSN_JSON_TOK_COMMA: { //--- pos++; //--- if(m_stak_curr_state == TSN_JSON_CTX_IN_OBJ) { const int start = m_index[sp++] + 1; // ahora mismo sp apunta a " TSN_JSON_PARSE_KEY } TSN_JSON_NEXT_VALUE_COMM_KEY break; } case TSN_JSON_TOK_LLAVE_INI: { //--- sp++; // saltamos caracter estrucutra act //--- if(++m_stak_curr >= JSONPARSERBYLEO_MAX_STACK) { *m_last_err = TSN_JSON_ERR_OVERFLOW_IN_STACK; TSN_JSON_ON_FINISH return false; } //--- m_cinta[cinta_pos] = JSON_VTYPE_OBJ; m_stak_curr_state = TSN_JSON_CTX_IN_OBJ; m_stak_state[m_stak_curr] = TSN_JSON_CTX_IN_OBJ; m_stak_num[m_stak_curr] = 0; m_stak_pos[m_stak_curr] = cinta_pos++; //--- m_cinta[cinta_pos++] = pos++; //--- pos = m_index[sp++]; if(m_raw[pos] == '}') { //m_next = TSN_JSON_TOK_LLAVE_END; goto JSON_END_CTX; } else { const int start = pos; // luego igual pos se modifica solo lo ponemos aqui TSN_JSON_PARSE_KEY // Ahora a buscar valor TSN_JSON_NEXT_VALUE_COMM_KEY } //--- break; } case TSN_JSON_TOK_COR_INI: { //--- sp++; //--- if(++m_stak_curr >= JSONPARSERBYLEO_MAX_STACK) { *m_last_err = TSN_JSON_ERR_OVERFLOW_IN_STACK; TSN_JSON_ON_FINISH return false; } //--- m_cinta[cinta_pos] = JSON_VTYPE_ARR; m_stak_curr_state = TSN_JSON_CTX_IN_ARR; m_stak_state[m_stak_curr] = TSN_JSON_CTX_IN_ARR; m_stak_num[m_stak_curr] = 0; m_stak_pos[m_stak_curr] = cinta_pos++; //--- m_cinta[cinta_pos++] = pos++; //Print(CharToString(m_raw[pos])); //--- const int n = m_index[sp]; if(m_raw[n] != ']') { TSN_JSON_NEXT_COR_INI } else { sp++; pos = n; //m_next = TSN_JSON_TOK_COR_END; goto JSON_END_CTX; } //--- break; } // tema de ends ya lo scaa en next el sp case TSN_JSON_TOK_LLAVE_END: case TSN_JSON_TOK_COR_END: JSON_END_CTX: { //--- const int lp = m_stak_pos[m_stak_curr]; m_cinta[lp] |= int64_t(m_stak_num[m_stak_curr]) << TSN_SBL_BIT_START_NUM_EL | int64_t(cinta_pos - lp) << TSN_SBL_BIT_START_NUM_C; m_cinta[lp + 1] |= int64_t(pos) << TSN_JSON_BIT_END_T; //--- if(--m_stak_curr < 0) { TSN_JSON_ON_FINISH return true; } //--- TSN_JSON_AUMENT //--- m_stak_curr_state = m_stak_state[m_stak_curr]; //--- pos++; //--- JSONPARSERBYLEO_FULL_NEXT //--- break; } //--- default: { *m_last_err = TSN_JSON_ERR_INVALID_CHAR; TSN_JSON_ON_FINISH return false; } } } } //+------------------------------------------------------------------+