//+------------------------------------------------------------------+ //| ReportBuilder.mqh | //| Copyright 2026, MetaQuotes Ltd. | //| www.mql5.com | //+------------------------------------------------------------------+ #ifndef REQUEST_LATENCY_LAB_REPORT_BUILDER_MQH #define REQUEST_LATENCY_LAB_REPORT_BUILDER_MQH #include "..\..\Include\RequestLatencyLab\Models.mqh" #include "..\..\Include\RequestLatencyLab\CsvStorage.mqh" #include "..\..\Include\RequestLatencyLab\Statistics.mqh" struct RebuiltSample { ulong sequence; string condition_id; string experiment_id; string series_id; string outcome; uint present_mask; ulong t0, t1, t2, t3, t4, t5, t6; bool recovered; bool conflict; bool interrupted; bool late; bool deadline_exceeded; // R6-B3/B2: sticky-флаг превышения срока bool completed; bool confirmation_invalidated; // R6-B2: полное восстановление флагов metadata bool is_warmup; // R6-B2: WARMUP исключается как в онлайн //--- R4-B8: роль/операция/режим из samples.csv (исключение CLEANUP, //--- семантика applicability — как онлайн EvaluateMetric) string role; string operation; string mode; bool is_cleanup; string session_id; string symbol; // R5-B7: совместимость серий string configuration_id; // R7-B6: сверка строк серии с manifest string market_window_id; // R8-S2: связь sample->окно E3 ulong market_window_seq; // R8-S2 ulong observation_deadline_us; // R5-B3: срок метрики (колонка 49) bool callback_coverage_complete; // R5-B4: полнота локальных callback }; class CReportBuilder { public: //--- чтение samples.csv и восстановление записей static int LoadSamples(const string rel_path, RebuiltSample &samples[], const int capacity, LabError &err); //--- R6-S1: числовое поле: пустое = отсутствие метки (допустимо), //--- любой нецифровой токен = повреждённый файл (отказ). static bool ParseCsvUlong(const string txt, ulong &out); static int CsvNum(const string txt, ulong &out, LabError &err, const string what, const int row); //--- R7-B4: деление контента на ЛОГИЧЕСКИЕ записи CSV. Перевод строки //--- ВНУТРИ кавычек — данные, а не разделитель; хвостовой \r терминатора //--- записи снимается один раз (внутренний \r данных сохраняется). //--- Возвращает число сохранённых записей, total_count — полное число //--- (позволяет отвергнуть переполнение ёмкости до молчаливого обрыва). static int SplitCsvRecords(const string content, string &recs[], const int capacity, int &total_count); //--- R7-B4: консистентность present_mask и числовых ячеек: бит наличия //--- должен соответствовать непустой ячейке (иначе данные несогласованы, //--- читатель не заменяет их нулём и не пропускает молча). static bool MaskFieldConsistent(const uint pm, const uint mask, const string txt); //--- R6-B2: ЕДИНАЯ функция определения роли/применимости/валидности/срока //--- для всех путей (online, rebuild, histogram, comparisons, E4): запись //--- восстанавливается в RequestMetadata и оценивается тем же static //--- CRequestTracker::EvaluateMetric, что и в online-экспорте. static bool EvaluateRebuilt(const RebuiltSample &r, const ENUM_LAB_METRIC metric, MetricObservation &out, bool &evaluated); //--- R4-B8: семантика применимости, зеркало онлайн CRequestTracker::EvaluateMetric. //--- Возвращает true, если метрика применима и значение может считаться. static bool RebuiltApplicable(const RebuiltSample &r, const ENUM_LAB_METRIC metric); //--- расчёт интервалов из восстановленных записей static bool Interval(const RebuiltSample &r, const ENUM_LAB_METRIC metric, long &value); //--- R5-B3: конец метрики для оценки своевременности по сроку static ulong MetricEndUs(const RebuiltSample &r, const ENUM_LAB_METRIC metric); //--- summary по одной группе (R4-B8: outcome_group; R6-B2: та же //--- семантика роли/валидности/срока и те же deadline-счётчики, что //--- онлайн BuildSummary — единая EvaluateMetric без второй реализации) static bool SummarizeGroup(const RebuiltSample &samples[], const int count, const ENUM_LAB_METRIC metric, const string condition, const string outcome_group, StatSummaryRow &row); //--- R4-B8: знаковые категории T2-T4 для offset_signs_rebuilt.csv static bool OffsetSigns(const RebuiltSample &samples[], const int count, const string condition, long &neg, long &zero, long &pos); //--- полный пересчёт из каталога набора данных static bool RebuildReport(const string dataset_relative_path, const string output_relative_path, LabError &err); //--- R5-B7: чтение значения ключа в manifest.csv исходного набора static string ReadManifestValue(const string dir, const string key); //--- R4-B6: E4 — report-only workflow из ровно пяти завершённых E1 ASYNC //--- datasets. Исходные samples НЕ копируются: строки ссылаются на сессии //--- оригинала; статистика считается по объединённым первичным записям //--- (500 ASYNC наблюдений), а не усреднением пяти медиан/p99. static bool RebuildE4(const string &dirs[], const int ndirs, const string dataset_id, const string output_relative_path, LabError &err, const int expected_main_per_series = 100, const string allowed_origin = "DEMO"); }; //+------------------------------------------------------------------+ //| Внешние определения методов. | //+------------------------------------------------------------------+ //--- чтение samples.csv и восстановление записей int CReportBuilder::LoadSamples(const string rel_path, RebuiltSample &samples[], const int capacity, LabError &err) { err.Reset(); err.component = LAB_COMP_REPORT; string content; if(!CCsv::LoadUtf8(rel_path, content, err)) return(-1); //--- R7-B4: деление на ЛОГИЧЕСКИЕ записи CSV (кавычки/встроенные //--- переводы строк/CRLF): перевод строки внутри кавычек — данные, //--- а не разделитель записи; терминатор записи снимается один раз. string recs[]; if(ArrayResize(recs, capacity + 16) != (capacity + 16)) { err.code = 44; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv record buffer allocation failed"; return(-1); } int total_records = 0; //--- R8-S1: capacity — число НАБЛЮДЕНИЙ; заголовок НЕ расходует её //--- (допустимо header + capacity строк данных = capacity+1 записей). const int n = SplitCsvRecords(content, recs, capacity + 1, total_records); if(total_records > capacity + 1) { err.code = 31; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv exceeds capacity " + IntegerToString(capacity); return(-1); } if(n < 2) { err.code = 32; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv has no data rows"; return(-1); } //--- R5-B5/R6-S1: полный контроль заголовка по CSV_SCHEMA (66 имён); //--- перестановка колонок между индексами обнаруживается. string hline = recs[0]; string hcols[]; int hcount = 0; CCsv::ParseRow(hline, hcols, hcount); bool hdr_ok = (hcount >= 68); const string hdr_expect[68] = { "request_id", "sequence", "send_start_us", "send_return_us", "request_event_us", "order_event_us", "first_deal_us", "final_event_us", "retcode", "completed", "campaign_id", "experiment_id", "series_id", "session_id", "configuration_id", "condition_id", "slot_id", "parent_sequence", "cleanup_attempt", "role", "is_warmup", "operation", "mode", "logging_mode", "symbol", "side", "correlation_status", "correlation_method", "order_ticket", "position_ticket", "position_identifier", "last_deal_us", "deal_count", "executed_volume", "callback_volume", "history_volume", "deal_coverage_complete", "present_mask", "recovered", "conflict", "measurement_interrupted", "scenario_deviation", "outcome", "outcome_at_final", "final_source", "trading_state_known", "confirmation_invalidated", "confirmation_record_id", "current_confirmation_record_id", "observation_deadline_us", "collection_deadline_us", "collection_close_us", "collection_closed", "deadline_exceeded", "market_regime", "calibration_id", "bid_at_send", "ask_at_send", "spread_at_send", "tick_time_msc", "actual_type", "actual_filling", "deviation_points", "comment", "request_price", "callback_coverage_complete", "market_window_id", "market_window_seq" }; //--- R9-S1: проверяются ВСЕ объявленные имена (68), включая две //--- новые колонки связи sample->окно (market_window_id/seq). for(int hi = 0; hi < 68 && hdr_ok; hi++) if(hcount <= hi || hcols[hi] != hdr_expect[hi]) hdr_ok = false; if(!hdr_ok) { err.code = 32; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv header mismatch (cols=" + IntegerToString(hcount) + ")"; return(-1); } int count = 0; for(int i = 1; i < n; i++) // запись 0 - заголовок { string s = recs[i]; if(StringLen(s) == 0) continue; string cols[]; int ccount = 0; if(!CCsv::ParseRow(s, cols, ccount)) { err.code = 33; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv row " + IntegerToString(i) + " parse failed"; return(-1); } //--- R5-B5: повреждённые строки не исключаются молча if(ccount < 68) { err.code = 30; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv row " + IntegerToString(i) + " short: " + IntegerToString(ccount) + " cols (expected 68)"; return(-1); } if(count >= capacity) { err.code = 31; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv exceeds capacity " + IntegerToString(capacity); return(-1); } RebuiltSample r; ulong u = 0; if(CsvNum(cols[1], u, err, "sequence", i) < 0) return(-1); r.sequence = u; r.condition_id = cols[15]; r.experiment_id = cols[11]; r.series_id = cols[12]; r.outcome = cols[42]; if(CsvNum(cols[37], u, err, "present_mask", i) < 0) return(-1); r.present_mask = (uint)u; if(CsvNum(cols[2], u, err, "t0", i) < 0) return(-1); r.t0 = u; if(CsvNum(cols[3], u, err, "t1", i) < 0) return(-1); r.t1 = u; if(CsvNum(cols[4], u, err, "t2", i) < 0) return(-1); r.t2 = u; if(CsvNum(cols[5], u, err, "t3", i) < 0) return(-1); r.t3 = u; if(CsvNum(cols[6], u, err, "t4", i) < 0) return(-1); r.t4 = u; if(CsvNum(cols[31], u, err, "t5", i) < 0) return(-1); r.t5 = u; if(CsvNum(cols[7], u, err, "t6", i) < 0) return(-1); r.t6 = u; if(CsvNum(cols[49], u, err, "observation_deadline_us", i) < 0) return(-1); r.observation_deadline_us = u; r.recovered = cols[38] == "1"; r.conflict = cols[39] == "1"; r.interrupted = cols[40] == "1"; r.late = cols[53] == "1"; r.deadline_exceeded = cols[53] == "1"; // R6-B3 r.completed = cols[9] == "1"; r.confirmation_invalidated = cols[46] == "1"; // R6-B2 r.is_warmup = (ccount > 20 && cols[20] == "1"); // R6-B2 //--- R4-B8: роль, операция, режим (колонки 19/21/22) r.role = (ccount > 19 ? cols[19] : ""); r.operation = (ccount > 21 ? cols[21] : ""); r.mode = (ccount > 22 ? cols[22] : ""); r.is_cleanup = (r.role == "CLEANUP"); r.session_id = (ccount > 13 ? cols[13] : ""); r.symbol = (ccount > 24 ? cols[24] : ""); // R5-B7 r.configuration_id = (ccount > 14 ? cols[14] : ""); // R7-B6 r.market_window_id = (ccount > 66 ? cols[66] : ""); // R8-S2 if(ccount > 67) { if(CsvNum(cols[67], u, err, "market_window_seq", i) < 0) return(-1); r.market_window_seq = u; } r.callback_coverage_complete = (ccount > 65 && cols[65] == "1"); // R5-B4 //--- R7-B4: пустые числовые поля сверяются с present_mask: бит //--- наличия ↔ непустая ячейка. Изменение непустой T1 на пустую при //--- сохранённом бите (или наоборот) — несогласованный файл, отказ. if(!MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T0, cols[2]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T1, cols[3]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T2, cols[4]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T3, cols[5]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T4, cols[6]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T5, cols[31]) || !MaskFieldConsistent(r.present_mask, (uint)LAB_MASK_T6, cols[7])) { err.code = 40; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv row " + IntegerToString(i) + " timestamp/mask mismatch"; return(-1); } //--- R6-S1: ключи (сессия/sequence) уникальны — повтор = повреждение for(int dj = 0; dj < count; dj++) if(samples[dj].sequence == r.sequence) { err.code = 35; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv duplicate sequence " + IntegerToString(r.sequence); return(-1); } samples[count] = r; count++; } return(count); } bool CReportBuilder::ParseCsvUlong(const string txt, ulong &out) { out = 0; if(StringLen(txt) == 0) return(true); const int len = StringLen(txt); //--- R8-S1: диапазон ulong — не более 20 десятичных знаков if(len > 20) return(false); for(int k = 0; k < len; k++) { const ushort ch = StringGetCharacter(txt, k); if(ch < '0' || ch > '9') return(false); } if(len == 20) { //--- строго <= ULONG_MAX (18446744073709551615) const string max_ulong = "18446744073709551615"; for(int k = 0; k < 20; k++) { const ushort a = StringGetCharacter(txt, k); const ushort b = StringGetCharacter(max_ulong, k); if(a > b) return(false); if(a < b) break; } } out = (ulong)StringToInteger(txt); return(true); } int CReportBuilder::CsvNum(const string txt, ulong &out, LabError &err, const string what, const int row) { if(ParseCsvUlong(txt, out)) return(0); err.Reset(); err.component = LAB_COMP_REPORT; err.code = 34; err.severity = LAB_SEV_BLOCKER; err.message = "samples.csv " + what + " row " + IntegerToString(row) + " not numeric"; return(-1); } int CReportBuilder::SplitCsvRecords(const string content, string &recs[], const int capacity, int &total_count) { total_count = 0; int count = 0; int len = StringLen(content); bool in_quotes = false; string cur = ""; for(int i = 0; i < len; i++) { const ushort ch = StringGetCharacter(content, i); if(ch == '"') { if(in_quotes && i + 1 < len && StringGetCharacter(content, i + 1) == '"') { cur += "\"\""; i++; } else { cur += "\""; in_quotes = !in_quotes; } continue; } if(!in_quotes && ch == '\n') { total_count++; if(StringLen(cur) > 0 && StringGetCharacter(cur, StringLen(cur) - 1) == '\r') cur = StringSubstr(cur, 0, StringLen(cur) - 1); if(count < capacity) { recs[count] = cur; count++; } cur = ""; } else cur += ShortToString(ch); } if(StringLen(cur) > 0 || total_count == 0) { total_count++; if(StringLen(cur) > 0 && StringGetCharacter(cur, StringLen(cur) - 1) == '\r') cur = StringSubstr(cur, 0, StringLen(cur) - 1); if(count < capacity) { recs[count] = cur; count++; } } return(count); } bool CReportBuilder::MaskFieldConsistent(const uint pm, const uint mask, const string txt) { const bool bit = ((pm & mask) != 0); const bool empty = (StringLen(txt) == 0); return(bit != empty); } bool CReportBuilder::EvaluateRebuilt(const RebuiltSample &r, const ENUM_LAB_METRIC metric, MetricObservation &out, bool &evaluated) { evaluated = false; out.Zero(); //--- роль/операция/исход строкой -> enum (схема BuildSamples round-trip) ENUM_LAB_OPERATION op = LAB_OP_NONE; if(r.operation == "MARKET_OPEN") op = LAB_OP_MARKET_OPEN; else if(r.operation == "PENDING_CREATE") op = LAB_OP_PENDING_CREATE; else if(r.operation == "PENDING_DELETE") op = LAB_OP_PENDING_DELETE; else if(r.operation == "POSITION_CLOSE") op = LAB_OP_POSITION_CLOSE; ENUM_LAB_OUTCOME oc = LAB_OUT_UNKNOWN; if(r.outcome == "FILLED") oc = LAB_OUT_FILLED; else if(r.outcome == "PARTIAL_CANCELED") oc = LAB_OUT_PARTIAL_CANCELED; else if(r.outcome == "PARTIAL_EXPIRED") oc = LAB_OUT_PARTIAL_EXPIRED; else if(r.outcome == "CANCELED_NO_FILL") oc = LAB_OUT_CANCELED_NO_FILL; else if(r.outcome == "EXPIRED_NO_FILL") oc = LAB_OUT_EXPIRED_NO_FILL; else if(r.outcome == "REJECTED") oc = LAB_OUT_REJECTED; else if(r.outcome == "PENDING_PLACED") oc = LAB_OUT_PENDING_PLACED; else if(r.outcome == "PENDING_DELETED") oc = LAB_OUT_PENDING_DELETED; else if(r.outcome == "POSITION_CLOSED") oc = LAB_OUT_POSITION_CLOSED; else if(r.outcome == "UNEXPECTED_ACTIVATION") oc = LAB_OUT_UNEXPECTED_ACTIVATION; RequestPlan plan; plan.Zero(); plan.operation = op; plan.symbol = r.symbol; plan.role = (r.role == "WARMUP" ? LAB_ROLE_WARMUP : (r.role == "CLEANUP" ? LAB_ROLE_CLEANUP : (r.role == "PILOT" ? LAB_ROLE_PILOT : LAB_ROLE_MAIN))); plan.sequence = r.sequence; RequestMetadata md; md.Reset(plan); md.present_mask = r.present_mask; md.outcome = oc; md.sample.send_start_us = r.t0; md.sample.send_return_us = r.t1; md.sample.request_event_us = r.t2; md.sample.order_event_us = r.t3; md.sample.first_deal_us = r.t4; md.last_deal_us = r.t5; md.sample.final_event_us = r.t6; md.callback_coverage_complete = r.callback_coverage_complete; md.conflict = r.conflict; md.measurement_interrupted = r.interrupted; md.confirmation_invalidated = r.confirmation_invalidated; md.observation_deadline_us = r.observation_deadline_us; return(CRequestTracker::EvaluateMetric(md, metric, out, evaluated)); } bool CReportBuilder::RebuiltApplicable(const RebuiltSample &r, const ENUM_LAB_METRIC metric) { const bool deal_expected = (r.operation == "MARKET_OPEN" || r.operation == "POSITION_CLOSE"); const bool rejected = (r.outcome == "REJECTED"); const bool unknown_outcome = (r.outcome == "UNKNOWN"); switch(metric) { case LAB_METRIC_CALL_DURATION: case LAB_METRIC_REQUEST_DELAY: case LAB_METRIC_FINAL_STATE_DELAY: case LAB_METRIC_REMAINING_AFTER_RETURN: return(true); case LAB_METRIC_ORDER_DELAY: if(r.operation == "PENDING_DELETE" || rejected || unknown_outcome) return(false); return(true); case LAB_METRIC_FIRST_DEAL_DELAY: case LAB_METRIC_LAST_DEAL_DELAY: if(r.operation == "PENDING_CREATE" && r.outcome != "UNEXPECTED_ACTIVATION") return(false); if(rejected) return(false); if(!deal_expected && r.outcome != "UNEXPECTED_ACTIVATION") return(false); return(true); case LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET: if(!deal_expected && r.outcome != "UNEXPECTED_ACTIVATION") return(false); if(rejected) return(false); return(true); } return(false); } bool CReportBuilder::Interval(const RebuiltSample &r, const ENUM_LAB_METRIC metric, long &value) { const bool has_t0 = (r.present_mask & (uint)LAB_MASK_T0) != 0; const bool has_t1 = (r.present_mask & (uint)LAB_MASK_T1) != 0; const bool has_t2 = (r.present_mask & (uint)LAB_MASK_T2) != 0; const bool has_t3 = (r.present_mask & (uint)LAB_MASK_T3) != 0; const bool has_t4 = (r.present_mask & (uint)LAB_MASK_T4) != 0; const bool has_t5 = (r.present_mask & (uint)LAB_MASK_T5) != 0; const bool has_t6 = (r.present_mask & (uint)LAB_MASK_T6) != 0; ulong start = 0, end = 0; switch(metric) { case LAB_METRIC_CALL_DURATION: start = r.t0; end = r.t1; break; case LAB_METRIC_REQUEST_DELAY: start = r.t0; end = r.t2; break; case LAB_METRIC_ORDER_DELAY: start = r.t0; end = r.t3; break; case LAB_METRIC_FIRST_DEAL_DELAY: start = r.t0; end = r.t4; break; case LAB_METRIC_LAST_DEAL_DELAY: start = r.t0; end = r.t5; break; case LAB_METRIC_FINAL_STATE_DELAY: start = r.t0; end = r.t6; break; case LAB_METRIC_REMAINING_AFTER_RETURN: start = r.t1; end = r.t6; break; case LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET: start = r.t2; end = r.t4; break; } const bool sp = ((metric == LAB_METRIC_CALL_DURATION || metric == LAB_METRIC_REQUEST_DELAY || metric == LAB_METRIC_ORDER_DELAY || metric == LAB_METRIC_FIRST_DEAL_DELAY || metric == LAB_METRIC_LAST_DEAL_DELAY || metric == LAB_METRIC_FINAL_STATE_DELAY) && has_t0) || (metric == LAB_METRIC_REMAINING_AFTER_RETURN && has_t1) || (metric == LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET && has_t2); const bool ep = ((metric == LAB_METRIC_CALL_DURATION && has_t1) || (metric == LAB_METRIC_REQUEST_DELAY && has_t2) || (metric == LAB_METRIC_ORDER_DELAY && has_t3) || (metric == LAB_METRIC_FIRST_DEAL_DELAY && has_t4) || (metric == LAB_METRIC_LAST_DEAL_DELAY && has_t5) || ((metric == LAB_METRIC_FINAL_STATE_DELAY || metric == LAB_METRIC_REMAINING_AFTER_RETURN) && has_t6) || (metric == LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET && has_t4)); if(!sp || !ep) return(false); if(metric == LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET) value = SignedDelta(start, end); else value = (long)(end - start); return(true); } ulong CReportBuilder::MetricEndUs(const RebuiltSample &r, const ENUM_LAB_METRIC metric) { switch(metric) { case LAB_METRIC_CALL_DURATION: return(r.t1); case LAB_METRIC_REQUEST_DELAY: return(r.t2); case LAB_METRIC_ORDER_DELAY: return(r.t3); case LAB_METRIC_FIRST_DEAL_DELAY: return(r.t4); case LAB_METRIC_LAST_DEAL_DELAY: return(r.t5); case LAB_METRIC_FINAL_STATE_DELAY: return(r.t6); case LAB_METRIC_REMAINING_AFTER_RETURN: return(r.t6); case LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET: return(r.t4); } return(0); } bool CReportBuilder::SummarizeGroup(const RebuiltSample &samples[], const int count, const ENUM_LAB_METRIC metric, const string condition, const string outcome_group, StatSummaryRow &row) { row.Zero(); row.condition_id = condition; row.metric_id = LabMetricName(metric); row.outcome_group = outcome_group; double values[]; if(ArrayResize(values, count) != count) return(false); int used = 0; ulong n_deadline = 0, n_deadline_denom = 0, n_t6missing = 0; for(int i = 0; i < count; i++) { const RebuiltSample r = samples[i]; //--- R6-B2: исключение прогрева точно как онлайн (is_warmup + CLEANUP) if(r.is_cleanup || r.is_warmup) continue; if(StringLen(condition) > 0 && r.condition_id != condition) continue; if(outcome_group == "REJECTED" && r.outcome != "REJECTED") continue; if(outcome_group == "SUCCESS" && (r.outcome == "REJECTED" || r.outcome == "UNKNOWN")) continue; row.n_attempted++; //--- R6-B2: deadline-счётчики идентичны онлайн-семантике if(r.observation_deadline_us > 0) n_deadline_denom++; if(r.deadline_exceeded) { n_deadline++; if((r.present_mask & (uint)LAB_MASK_T6) == 0) n_t6missing++; } MetricObservation mo; bool ev = false; if(!EvaluateRebuilt(r, metric, mo, ev)) { row.n_applicability_unknown++; continue; } if(mo.applicability == LAB_APP_NOT_APPLICABLE) { row.n_not_applicable++; continue; } if(mo.applicability == LAB_APP_UNKNOWN) { row.n_applicability_unknown++; continue; } row.n_applicable++; if(!mo.has_value) { row.n_missing++; continue; } const ENUM_LAB_FITNESS cat = mo.category; if(cat == LAB_FIT_VALID) { if(used >= ArraySize(values)) continue; values[used] = (double)mo.value_us; used++; row.n_valid++; } else if(cat == LAB_FIT_LATE) row.n_late++; else if(cat == LAB_FIT_MISSING) row.n_missing++; else if(cat == LAB_FIT_CONFLICT) row.n_conflict++; else if(cat == LAB_FIT_INTERRUPTED) row.n_interrupted++; else row.n_not_applicable++; } if(used >= 1) // R5-S5: n=1 допустим CStatistics::Compute(values, used, row); row.n_deadline_exceeded = n_deadline; row.deadline_denominator = n_deadline_denom; row.deadline_rate = (n_deadline_denom > 0 ? (double)n_deadline / n_deadline_denom : 0.0); row.n_timestamp_missing_at_deadline = n_t6missing; row.missing_timestamp_rate = (n_deadline_denom > 0 ? (double)n_t6missing / n_deadline_denom : 0.0); return(true); } bool CReportBuilder::OffsetSigns(const RebuiltSample &samples[], const int count, const string condition, long &neg, long &zero, long &pos) { neg = 0; zero = 0; pos = 0; for(int i = 0; i < count; i++) { const RebuiltSample r = samples[i]; if(r.is_cleanup) continue; if(StringLen(condition) > 0 && r.condition_id != condition) continue; if(r.is_warmup) continue; MetricObservation mo; bool ev = false; if(!EvaluateRebuilt(r, LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET, mo, ev)) continue; if(!ev || mo.applicability != LAB_APP_APPLICABLE || !mo.has_value || mo.category != LAB_FIT_VALID) continue; const long v = mo.value_us; if(v < 0) neg++; else if(v == 0) zero++; else pos++; } return(true); } bool CReportBuilder::RebuildReport(const string dataset_relative_path, const string output_relative_path, LabError &err) { err.Reset(); err.component = LAB_COMP_REPORT; RebuiltSample samples[]; if(ArrayResize(samples, 4096) != 4096) return(false); const int n = LoadSamples(dataset_relative_path, samples, 4096, err); if(n < 0) return(false); //--- пересчёт summary (R5-B5): ЕДИНАЯ схема колонок с онлайн-export //--- (41 колонка, CSV_SCHEMA.md) и одна функция построения популяции. string out = ""; const string header[] = { "experiment_id", "condition_id", "series_id", "metric_id", "outcome_group", "population", "observation_unit", "segment_key", "source_set_id", "unit", "n_attempted", "n_applicable", "n_not_applicable", "n_applicability_unknown", "n_valid", "n_missing", "n_late", "n_conflict", "n_interrupted", "n_used", "minimum", "mean", "median", "p90", "p95", "p99", "p999", "maximum", "stddev", "percentile_method", "method_fingerprint", "p95_tail_expected_n", "p99_tail_expected_n", "p999_tail_expected_n", "tail_warning_codes", "n_deadline_exceeded", "deadline_denominator", "deadline_rate", "n_timestamp_missing_at_deadline", "missing_timestamp_rate", "conditional_distribution" }; out += CCsv::Header(header); string cols[41]; ulong rows = 0; // фактическое число строк данных string exp_id = ""; string ser_id = ""; string src_session = ""; // R6-B2: источник (session) string conds[16]; int cond_count = 0; for(int i = 0; i < n; i++) { const RebuiltSample r = samples[i]; if(StringLen(exp_id) == 0 && StringLen(r.experiment_id) > 0) exp_id = r.experiment_id; if(StringLen(ser_id) == 0 && StringLen(r.series_id) > 0) ser_id = r.series_id; if(StringLen(src_session) == 0 && StringLen(r.session_id) > 0) src_session = r.session_id; if(r.is_cleanup || r.is_warmup) // R6-B2: прогрев не сегмент continue; bool known = false; for(int c = 0; c < cond_count; c++) if(conds[c] == r.condition_id) { known = true; break; } if(!known && cond_count < 16) { conds[cond_count] = r.condition_id; cond_count++; } } //--- R5-S5: три группы исходов (ALL, SUCCESS, REJECTED); последний //--- проход (ccond==cond_count) — combined A+B; ключи уникальны. const string groups[3] = {"ALL", "SUCCESS", "REJECTED"}; for(int g = 0; g < 3; g++) { for(int ccond = 0; ccond < cond_count + 1; ccond++) { const string cond = (ccond < cond_count ? conds[ccond] : ""); for(int m = 0; m < 8; m++) { StatSummaryRow row; if(!SummarizeGroup(samples, n, (ENUM_LAB_METRIC)m, cond, groups[g], row)) continue; cols[0] = (StringLen(row.experiment_id) > 0 ? row.experiment_id : exp_id); cols[1] = row.condition_id; cols[2] = (StringLen(row.series_id) > 0 ? row.series_id : ser_id); cols[3] = row.metric_id; cols[4] = row.outcome_group; cols[5] = row.population; cols[6] = row.observation_unit; cols[7] = "{}"; cols[8] = (StringLen(row.source_set_id) > 0 ? row.source_set_id : src_session); cols[9] = "us"; cols[10] = IntegerToString(row.n_attempted); cols[11] = IntegerToString(row.n_applicable); cols[12] = IntegerToString(row.n_not_applicable); cols[13] = IntegerToString(row.n_applicability_unknown); cols[14] = IntegerToString(row.n_valid); cols[15] = IntegerToString(row.n_missing); cols[16] = IntegerToString(row.n_late); cols[17] = IntegerToString(row.n_conflict); cols[18] = IntegerToString(row.n_interrupted); cols[19] = IntegerToString(row.n_used); cols[20] = (row.has_stats ? DoubleToString(row.minimum, 10) : ""); cols[21] = (row.has_stats ? DoubleToString(row.mean, 10) : ""); cols[22] = (row.has_stats ? DoubleToString(row.median, 10) : ""); cols[23] = (row.has_stats ? DoubleToString(row.p90, 10) : ""); cols[24] = (row.has_stats ? DoubleToString(row.p95, 10) : ""); cols[25] = (row.has_stats ? DoubleToString(row.p99, 10) : ""); cols[26] = (row.has_stats ? DoubleToString(row.p999, 10) : ""); cols[27] = (row.has_stats ? DoubleToString(row.maximum, 10) : ""); cols[28] = (row.has_stats && row.n_used >= 2 ? DoubleToString(row.stddev, 10) : ""); cols[29] = row.percentile_method; cols[30] = row.percentile_method + "_V1"; cols[31] = DoubleToString(row.p95_tail_expected_n, 10); cols[32] = DoubleToString(row.p99_tail_expected_n, 10); cols[33] = DoubleToString(row.p999_tail_expected_n, 10); cols[34] = row.tail_warning_codes; cols[35] = IntegerToString(row.n_deadline_exceeded); cols[36] = IntegerToString(row.deadline_denominator); cols[37] = DoubleToString(row.deadline_rate, 12); cols[38] = IntegerToString(row.n_timestamp_missing_at_deadline); cols[39] = DoubleToString(row.missing_timestamp_rate, 12); cols[40] = (row.conditional_distribution ? "1" : "0"); out += CCsv::BuildRow(cols, 41); rows++; } } } const string summary_path = output_relative_path + "summary_rebuilt.csv"; if(!CCsv::SaveUtf8(summary_path, out, err)) return(false); //--- R5-B5: контрольное чтение — от РЕАЛЬНО созданной таблицы if(!CCsv::ReadBackVerifyFile(summary_path, (int)rows + 1, err)) return(false); //--- histogram_rebuilt.csv (без знаковой метрики T2-T4) string hist = ""; const string hh[] = {"experiment_id", "condition_id", "series_id", "metric_id", "outcome_group", "population", "segment_key", "bin_index", "lower_us", "upper_us", "count", "denominator", "share" }; hist += CCsv::Header(hh); string hcol[13]; const ulong lo[10] = {0, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000}; const ulong up[10] = {1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000, 0}; string hist_conds[16]; int hist_cond_count = 0; for(int i = 0; i < n; i++) { if(samples[i].is_cleanup || samples[i].is_warmup) continue; // R6-B2 bool known = false; for(int c2 = 0; c2 < hist_cond_count; c2++) if(hist_conds[c2] == samples[i].condition_id) { known = true; break; } if(!known && hist_cond_count < 16) { hist_conds[hist_cond_count] = samples[i].condition_id; hist_cond_count++; } } for(int m = 0; m < 8; m++) { if(m == (int)LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET) continue; // знаковая — отдельно //--- R7-B4: группы условий ИЗ ДАННЫХ + combined (как online BuildHistogram); //--- одинаковые исходные данные дают одинаковую схему таблиц. for(int c = 0; c < hist_cond_count + 1; c++) { const string cond = (c < hist_cond_count ? hist_conds[c] : ""); long vals[]; if(ArrayResize(vals, n) != n) continue; int vc = 0; for(int i = 0; i < n; i++) { if(samples[i].is_cleanup || samples[i].is_warmup) continue; // R6-B2 if(StringLen(cond) > 0 && samples[i].condition_id != cond) continue; MetricObservation mo; bool ev = false; EvaluateRebuilt(samples[i], (ENUM_LAB_METRIC)m, mo, ev); if(!ev || mo.applicability != LAB_APP_APPLICABLE || !mo.has_value || mo.category != LAB_FIT_VALID) continue; vals[vc] = (long)(mo.value_us); vc++; } ulong bins[10]; ArrayInitialize(bins, 0); if(vc > 0) HistogramCount(vals, vc, bins); for(int b = 0; b < 10; b++) { hcol[0] = exp_id; hcol[1] = cond; hcol[2] = ser_id; hcol[3] = LabMetricName((ENUM_LAB_METRIC)m); hcol[4] = "ALL"; hcol[5] = "PRIMARY"; hcol[6] = "{}"; hcol[7] = IntegerToString(b); hcol[8] = IntegerToString(lo[b]); hcol[9] = (up[b] == 0 ? "" : IntegerToString(up[b])); hcol[10] = IntegerToString(bins[b]); hcol[11] = IntegerToString(vc); hcol[12] = (vc > 0 ? DoubleToString((double)bins[b] / vc, 12) : "0"); hist += CCsv::BuildRow(hcol, 13); } } } const string hist_path = output_relative_path + "histogram_rebuilt.csv"; if(!CCsv::SaveUtf8(hist_path, hist, err)) return(false); //--- offset_signs_rebuilt.csv (знаковая метрика). R7-B4: схема РОВНО //--- как онлайн BuildOffsetSigns (10 колонок с low_us/high_us), группы //--- условий ИЗ ДАННЫХ + combined — одинаковые данные, одинаковая схема. string os = ""; const string oh[] = {"experiment_id", "condition_id", "series_id", "metric_id", "sign", "low_us", "high_us", "count", "denominator", "share" }; os += CCsv::Header(oh); string ocol[10]; const string signs[3] = {"NEGATIVE", "ZERO", "POSITIVE"}; string os_conds[16]; int os_cond_count = 0; for(int i = 0; i < n; i++) { if(samples[i].is_cleanup || samples[i].is_warmup) continue; // R6-B2 bool known = false; for(int c2 = 0; c2 < os_cond_count; c2++) if(os_conds[c2] == samples[i].condition_id) { known = true; break; } if(!known && os_cond_count < 16) { os_conds[os_cond_count] = samples[i].condition_id; os_cond_count++; } } for(int c = 0; c < os_cond_count + 1; c++) { const string cond = (c < os_cond_count ? os_conds[c] : ""); long neg = 0, zero = 0, pos = 0; OffsetSigns(samples, n, cond, neg, zero, pos); const long totals[3] = {neg, zero, pos}; const long total = neg + zero + pos; for(int k = 0; k < 3; k++) { ocol[0] = exp_id; ocol[1] = cond; ocol[2] = ser_id; ocol[3] = LabMetricName(LAB_METRIC_REQUEST_FIRST_DEAL_OFFSET); ocol[4] = signs[k]; ocol[5] = (k == 0 ? "" : (k == 1 ? "0" : "1")); ocol[6] = (k == 0 ? "-1" : (k == 1 ? "0" : "")); ocol[7] = IntegerToString(totals[k]); ocol[8] = IntegerToString(total); ocol[9] = (total > 0 ? DoubleToString((double)totals[k] / total, 12) : "0"); os += CCsv::BuildRow(ocol, 10); } } const string os_path = output_relative_path + "offset_signs_rebuilt.csv"; if(!CCsv::SaveUtf8(os_path, os, err)) return(false); //--- comparisons_rebuilt.csv (A vs B). R7-S2: ТЗ §7 — delta = B−A, //--- mean/median/p95 и относительное изменение (B−A)/A; схема с онлайн. string cmp = ""; const string ch[] = {"experiment_id", "condition_pair", "metric_id", "unit", "outcome_group", "a_n", "a_mean", "a_median", "a_p95", "b_n", "b_mean", "b_median", "b_p95", "delta_mean", "delta_median", "delta_p95", "rel_mean", "rel_median", "rel_p95" }; cmp += CCsv::Header(ch); string ccol[19]; for(int m = 0; m < 8; m++) { double va[], vb[]; int na = 0, nb = 0; if(ArrayResize(va, n) != n || ArrayResize(vb, n) != n) continue; for(int i = 0; i < n; i++) { if(samples[i].is_cleanup || samples[i].is_warmup) continue; // R6-B2 const string rc = samples[i].condition_id; MetricObservation mo; bool ev = false; EvaluateRebuilt(samples[i], (ENUM_LAB_METRIC)m, mo, ev); if(!ev || mo.applicability != LAB_APP_APPLICABLE || !mo.has_value || mo.category != LAB_FIT_VALID) continue; if(rc == "A") { va[na] = (double)(mo.value_us); na++; } else if(rc == "B") { vb[nb] = (double)(mo.value_us); nb++; } } double ma = 0, pa = 0, mb = 0, pb = 0, ea = 0, eb = 0; const bool ha = (na >= 1 && LabAvgValue(va, na, ea) && LabPercentile(va, na, 50.0, ma) && LabPercentile(va, na, 95.0, pa)); const bool hb = (nb >= 1 && LabAvgValue(vb, nb, eb) && LabPercentile(vb, nb, 50.0, mb) && LabPercentile(vb, nb, 95.0, pb)); ccol[0] = exp_id; ccol[1] = "A_vs_B"; ccol[2] = LabMetricName((ENUM_LAB_METRIC)m); ccol[3] = "us"; ccol[4] = "ALL"; ccol[5] = IntegerToString(na); ccol[6] = (ha ? DoubleToString(ea, 10) : ""); ccol[7] = (ha ? DoubleToString(ma, 10) : ""); ccol[8] = (ha ? DoubleToString(pa, 10) : ""); ccol[9] = IntegerToString(nb); ccol[10] = (hb ? DoubleToString(eb, 10) : ""); ccol[11] = (hb ? DoubleToString(mb, 10) : ""); ccol[12] = (hb ? DoubleToString(pb, 10) : ""); //--- delta = B−A; rel = (B−A)/A (ТЗ §7, стр. 419) ccol[13] = (ha && hb ? DoubleToString(eb - ea, 10) : ""); ccol[14] = (ha && hb ? DoubleToString(mb - ma, 10) : ""); ccol[15] = (ha && hb ? DoubleToString(pb - pa, 10) : ""); ccol[16] = (ha && hb && MathAbs(ea) > 1e-12 ? DoubleToString((eb - ea) / ea, 10) : ""); ccol[17] = (ha && hb && MathAbs(ma) > 1e-12 ? DoubleToString((mb - ma) / ma, 10) : ""); ccol[18] = (ha && hb && MathAbs(pa) > 1e-12 ? DoubleToString((pb - pa) / pa, 10) : ""); cmp += CCsv::BuildRow(ccol, 19); } const string cmp_path = output_relative_path + "comparisons_rebuilt.csv"; if(!CCsv::SaveUtf8(cmp_path, cmp, err)) return(false); return(true); } string CReportBuilder::ReadManifestValue(const string dir, const string key) { LabError e; string content; if(!CCsv::LoadUtf8(dir + "\\manifest.csv", content, e)) return(""); string lines[]; const int n = StringSplit(content, '\n', lines); for(int i = 0; i < n; i++) { string s = lines[i]; StringReplace(s, "\r", ""); if(StringLen(s) == 0) continue; string cols[]; int c = 0; if(!CCsv::ParseRow(s, cols, c) || c < 2) continue; if(cols[0] == key) return(cols[1]); } return(""); } bool CReportBuilder::RebuildE4(const string &dirs[], const int ndirs, const string dataset_id, const string output_relative_path, LabError &err, const int expected_main_per_series, const string allowed_origin) { err.Reset(); err.component = LAB_COMP_REPORT; if(ndirs != 5) { err.code = 20; err.message = "E4 requires exactly 5 E1 ASYNC datasets (got " + IntegerToString(ndirs) + ")"; return(false); } //--- R5-B7: пять РАЗНЫХ каталогов (повтор одного набора под разными //--- именами не даёт 500 независимых наблюдений) for(int a = 0; a < ndirs; a++) for(int b = a + 1; b < ndirs; b++) { if(dirs[a] == dirs[b]) { err.code = 23; err.message = "E4 duplicate dataset path: " + dirs[a]; return(false); } } RebuiltSample merged[]; if(ArrayResize(merged, 4096) != 4096) return(false); int total = 0; int per_series[5]; string series_session[5]; for(int d = 0; d < 5; d++) per_series[d] = 0; //--- R6-B7: уникальные sequence внутри каждого файла серии ulong seen_seq[5][4096]; int seen_count[5]; for(int d = 0; d < 5; d++) seen_count[d] = 0; // заполняется до чтения string series_symbol[5]; string series_cfg[5]; for(int d = 0; d < ndirs; d++) { //--- R6-B7: manifest обязателен (отсутствие = недостающее //--- доказательство завершения), допускается только FINISHED. const string rs = ReadManifestValue(dirs[d], "run_status"); if(rs == "") { err.code = 36; err.message = "E4 dataset " + IntegerToString(d + 1) + " manifest missing (no evidence of finish)"; return(false); } if(rs != "FINISHED") { err.code = 26; err.message = "E4 dataset " + IntegerToString(d + 1) + " run_status=" + rs + " (expected FINISHED)"; return(false); } const string nm = ReadManifestValue(dirs[d], "n_main_dispatched"); if((int)StringToInteger(nm) < expected_main_per_series) { err.code = 37; err.message = "E4 dataset " + IntegerToString(d + 1) + " n_main_dispatched=" + nm + " < expected " + IntegerToString(expected_main_per_series); return(false); } series_cfg[d] = ReadManifestValue(dirs[d], "configuration_id"); if(series_cfg[d] == "") { err.code = 31; err.message = "E4 dataset " + IntegerToString(d + 1) + " configuration_id missing"; return(false); } //--- R7-B6: обычный E4 принимает ТОЛЬКО допустимое экспериментальное //--- происхождение (DEMO); LOCAL_ONLY формирует SYNTHETIC MAIN/WARMUP, //--- и включение таких наборов в материал статьи невозможно без //--- явного перехода в отдельный синтетический режим проверки. const string org = ReadManifestValue(dirs[d], "data_origin"); if(org != allowed_origin) { err.code = 41; err.message = "E4 dataset " + IntegerToString(d + 1) + " data_origin=" + (StringLen(org) == 0 ? "" : org) + " (requires " + allowed_origin + ")"; return(false); } RebuiltSample buf[4096]; const int nb = LoadSamples(dirs[d] + "\\samples.csv", buf, 4096, err); if(nb < 0) { err.message = "E4 dataset " + IntegerToString(d + 1) + " unreadable: " + err.message; return(false); } for(int i = 0; i < nb; i++) { const RebuiltSample r = buf[i]; //--- совместимость: только E1, только ASYNC (ветвь B), без CLEANUP if(r.is_cleanup) continue; if(r.experiment_id != "E1") { err.code = 21; err.message = "E4 dataset " + IntegerToString(d + 1) + " is not E1"; return(false); } //--- R5-B7: прогрев не входит в pooled; PILOT/SYNTHETIC отвергаются if(r.role == "WARMUP") continue; if(r.role != "MAIN") { err.code = 24; err.message = "E4 dataset " + IntegerToString(d + 1) + " contains role " + r.role; return(false); } if(r.condition_id != "B" || r.mode != "ASYNC") continue; if(StringLen(r.series_id) == 0) continue; //--- R5-B7: символ серий должен совпадать if(StringLen(r.symbol) > 0) { if(StringLen(series_symbol[d]) == 0) series_symbol[d] = r.symbol; else if(r.symbol != series_symbol[d]) { err.code = 28; err.message = "E4 dataset " + IntegerToString(d + 1) + " symbol mismatch"; return(false); } } //--- R6-B7: все строки файла принадлежат ОДНОЙ сессии серии if(StringLen(series_session[d]) == 0) series_session[d] = r.session_id; else if(r.session_id != series_session[d]) { err.code = 38; err.message = "E4 dataset " + IntegerToString(d + 1) + " mixed sessions"; return(false); } //--- R7-B6: конфигурация СТРОК сверяется с manifest серии //--- (не только равенство двух деклараций configuration_id). if(StringLen(r.configuration_id) > 0 && r.configuration_id != series_cfg[d]) { err.code = 42; err.message = "E4 dataset " + IntegerToString(d + 1) + " row configuration_id mismatch (manifest vs samples)"; return(false); } //--- R6-B7: ключ измерения уникален внутри серии (дубликат — фальсификация) for(int sk = 0; sk < seen_count[d]; sk++) if(seen_seq[d][sk] == r.sequence) { err.code = 39; err.message = "E4 dataset " + IntegerToString(d + 1) + " duplicate sequence " + IntegerToString(r.sequence); return(false); } seen_seq[d][seen_count[d]++] = r.sequence; if(total >= 4096) break; merged[total] = r; per_series[d]++; total++; } } //--- R5-B7: точные квоты MAIN на каждую серию for(int d = 0; d < 5; d++) { if(per_series[d] != expected_main_per_series) { err.code = 25; err.message = "E4 dataset " + IntegerToString(d + 1) + " MAIN B-ASYNC count " + IntegerToString(per_series[d]) + " != expected " + IntegerToString(expected_main_per_series); return(false); } } //--- R5-B7: пять РАЗНЫХ исходных сессий bool sessions_unique = true; for(int a = 0; a < 5 && sessions_unique; a++) for(int b = a + 1; b < 5; b++) { if(StringLen(series_session[a]) == 0 || StringLen(series_session[b]) == 0 || series_session[a] == series_session[b]) { sessions_unique = false; break; } } if(!sessions_unique) { err.code = 27; err.message = "E4 datasets are not 5 distinct finished series"; return(false); } //--- R6-B7: совместимость источников — символ и нормализованная //--- конфигурация равны МЕЖДУ всеми пятью сериями (не только внутри). for(int a = 0; a < 5; a++) for(int b = a + 1; b < 5; b++) { if(StringLen(series_symbol[a]) == 0 || StringLen(series_symbol[b]) == 0 || series_symbol[a] != series_symbol[b]) { err.code = 34; err.message = "E4 symbol mismatch across series: " + series_symbol[a] + " vs " + series_symbol[b]; return(false); } if(series_cfg[a] != series_cfg[b]) { err.code = 32; err.message = "E4 configuration mismatch: " + series_cfg[a] + " vs " + series_cfg[b]; return(false); } } //--- pooled-порог: ровно 5*expected MAIN B-ASYNC наблюдений if(total < expected_main_per_series * 5) { err.code = 22; err.message = "E4 pooled ASYNC count insufficient: " + IntegerToString(total); return(false); } //--- summary_rebuilt_e4.csv: per-series + pooled (только объединённые //--- первичные записи; усреднение серийных статистик не производится) string out = ""; //--- R7-B6: происхождение переносится во ВСЕ производные файлы; //--- добавлены показатели полноты по серии и pooled (n_attempted, //--- отказы/пропуски/deadline — числа нельзя выводить из одного n). const string header[] = { "experiment_id", "series_id", "source_session", "data_origin", "metric_id", "n", "n_attempted", "n_applicable", "n_not_applicable", "n_applicability_unknown", "n_missing", "n_late", "n_conflict", "n_interrupted", "n_deadline_exceeded", "deadline_denominator", "deadline_rate", "median", "p90", "p95", "p99", "minimum", "maximum", "mean", "stddev" }; out += CCsv::Header(header); string cols[25]; const int metrics[7] = {LAB_METRIC_CALL_DURATION, LAB_METRIC_REQUEST_DELAY, LAB_METRIC_ORDER_DELAY, LAB_METRIC_FIRST_DEAL_DELAY, LAB_METRIC_LAST_DEAL_DELAY, LAB_METRIC_FINAL_STATE_DELAY, LAB_METRIC_REMAINING_AFTER_RETURN }; //--- pooled из 500 первичных записей for(int mi = 0; mi < 7; mi++) { const ENUM_LAB_METRIC metric = (ENUM_LAB_METRIC)metrics[mi]; double vals[]; int vc = 0; if(ArrayResize(vals, 512) != 512) continue; ulong na = 0, napp = 0, nnapp = 0, nunk = 0, nmis = 0, nlate = 0, nconf = 0, nint = 0, nddead = 0, ndden = 0, nt6m = 0; for(int i = 0; i < total; i++) { const RebuiltSample rr = merged[i]; MetricObservation mo; bool ev = false; na++; if(rr.observation_deadline_us > 0) ndden++; if(rr.deadline_exceeded) { nddead++; if((rr.present_mask & (uint)LAB_MASK_T6) == 0) nt6m++; } if(!EvaluateRebuilt(rr, metric, mo, ev) || !ev) { nunk++; continue; } if(mo.applicability == LAB_APP_NOT_APPLICABLE) { nnapp++; continue; } if(mo.applicability == LAB_APP_UNKNOWN) { nunk++; continue; } napp++; //--- R7-S1: CONFLICT/INTERRUPTED учитываются ДО проверки has_value if(mo.category == LAB_FIT_CONFLICT) { nconf++; continue; } if(mo.category == LAB_FIT_INTERRUPTED) { nint++; continue; } if(!mo.has_value) { nmis++; continue; } if(mo.category == LAB_FIT_LATE) { nlate++; continue; } if(mo.category != LAB_FIT_VALID) { nmis++; continue; } if(vc >= ArraySize(vals) && ArrayResize(vals, vc + 256) != vc + 256) break; vals[vc] = (double)(mo.value_us); vc++; } StatSummaryRow row; row.Zero(); if(vc >= 2) CStatistics::Compute(vals, vc, row); cols[0] = "E4"; cols[1] = "POOLED"; cols[2] = "(5 series)"; cols[3] = allowed_origin; cols[4] = LabMetricName(metric); cols[5] = IntegerToString(vc); cols[6] = IntegerToString(na); cols[7] = IntegerToString(napp); cols[8] = IntegerToString(nnapp); cols[9] = IntegerToString(nunk); cols[10] = IntegerToString(nmis); cols[11] = IntegerToString(nlate); cols[12] = IntegerToString(nconf); cols[13] = IntegerToString(nint); cols[14] = IntegerToString(nddead); cols[15] = IntegerToString(ndden); cols[16] = (ndden > 0 ? DoubleToString((double)nddead / ndden, 12) : "0"); cols[17] = (row.has_stats ? DoubleToString(row.median, 10) : ""); cols[18] = (row.has_stats ? DoubleToString(row.p90, 10) : ""); cols[19] = (row.has_stats ? DoubleToString(row.p95, 10) : ""); cols[20] = (row.has_stats ? DoubleToString(row.p99, 10) : ""); cols[21] = (row.has_stats ? DoubleToString(row.minimum, 10) : ""); cols[22] = (row.has_stats ? DoubleToString(row.maximum, 10) : ""); cols[23] = (row.has_stats ? DoubleToString(row.mean, 10) : ""); cols[24] = (row.has_stats ? DoubleToString(row.stddev, 10) : ""); out += CCsv::BuildRow(cols, 25); } //--- per-series variability (реальные серии E1, ссылки на исходные сессии) //--- (5 серий берутся из первых 5 исходных файлов по порядку) int offset = 0; for(int d = 0; d < 5; d++) { //--- per-series блок: пересчитываем статистику по merged срезом int from = offset; int span = per_series[d]; for(int mi = 0; mi < 7; mi++) { const ENUM_LAB_METRIC metric = (ENUM_LAB_METRIC)metrics[mi]; double vals[]; int vc = 0; if(ArrayResize(vals, 256) != 256) continue; ulong na = 0, napp = 0, nnapp = 0, nunk = 0, nmis = 0, nlate = 0, nconf = 0, nint = 0, nddead = 0, ndden = 0, nt6m = 0; for(int i = from; i < from + span && i < total; i++) { const RebuiltSample rr = merged[i]; MetricObservation mo; bool ev = false; na++; if(rr.observation_deadline_us > 0) ndden++; if(rr.deadline_exceeded) { nddead++; if((rr.present_mask & (uint)LAB_MASK_T6) == 0) nt6m++; } if(!EvaluateRebuilt(rr, metric, mo, ev) || !ev) { nunk++; continue; } if(mo.applicability == LAB_APP_NOT_APPLICABLE) { nnapp++; continue; } if(mo.applicability == LAB_APP_UNKNOWN) { nunk++; continue; } napp++; if(mo.category == LAB_FIT_CONFLICT) { nconf++; continue; } if(mo.category == LAB_FIT_INTERRUPTED) { nint++; continue; } if(!mo.has_value) { nmis++; continue; } if(mo.category == LAB_FIT_LATE) { nlate++; continue; } if(mo.category != LAB_FIT_VALID) { nmis++; continue; } if(vc >= ArraySize(vals) && ArrayResize(vals, vc + 256) != vc + 256) break; vals[vc] = (double)(mo.value_us); vc++; } StatSummaryRow row; row.Zero(); if(vc >= 2) CStatistics::Compute(vals, vc, row); cols[0] = "E4"; cols[1] = IntegerToString(d + 1); cols[2] = series_session[d]; cols[3] = allowed_origin; cols[4] = LabMetricName(metric); cols[5] = IntegerToString(vc); cols[6] = IntegerToString(na); cols[7] = IntegerToString(napp); cols[8] = IntegerToString(nnapp); cols[9] = IntegerToString(nunk); cols[10] = IntegerToString(nmis); cols[11] = IntegerToString(nlate); cols[12] = IntegerToString(nconf); cols[13] = IntegerToString(nint); cols[14] = IntegerToString(nddead); cols[15] = IntegerToString(ndden); cols[16] = (ndden > 0 ? DoubleToString((double)nddead / ndden, 12) : "0"); cols[17] = (row.has_stats ? DoubleToString(row.median, 10) : ""); cols[18] = (row.has_stats ? DoubleToString(row.p90, 10) : ""); cols[19] = (row.has_stats ? DoubleToString(row.p95, 10) : ""); cols[20] = (row.has_stats ? DoubleToString(row.p99, 10) : ""); cols[21] = (row.has_stats ? DoubleToString(row.minimum, 10) : ""); cols[22] = (row.has_stats ? DoubleToString(row.maximum, 10) : ""); cols[23] = (row.has_stats ? DoubleToString(row.mean, 10) : ""); cols[24] = (row.has_stats ? DoubleToString(row.stddev, 10) : ""); out += CCsv::BuildRow(cols, 25); } offset += span; } const string path = output_relative_path + "summary_rebuilt_e4.csv"; if(!CCsv::SaveUtf8(path, out, err)) return(false); err.message = ""; return(true); } #endif // REQUEST_LATENCY_LAB_REPORT_BUILDER_MQH //+------------------------------------------------------------------+