//+------------------------------------------------------------------+ //| Trajectory.mqh | //| Copyright 2026, DNG | //| https://www.mql5.com/ru/users/dng | //+------------------------------------------------------------------+ #ifndef DPCC_TRAJECTORY_MQH #define DPCC_TRAJECTORY_MQH #define MaxSL 1000 #define MaxTP 1000 #define DPCC_ENCODER_INPUT_WINDOW 5 #define DPCC_ENCODER_STATE_STACK 504 #define DPCC_ENCODER_WARMUP_HISTORY (DPCC_ENCODER_STATE_STACK+DPCC_ENCODER_INPUT_WINDOW-1) input group "---- RSI: период и цена расчёта ----" //--- Период индикатора RSI. input int RSIPeriod = 14; //--- Цена, используемая RSI. input ENUM_APPLIED_PRICE RSIPrice = PRICE_CLOSE; input group "---- CCI: период и цена расчёта ----" //--- Период индикатора CCI. input int CCIPeriod = 14; //--- Цена, используемая CCI. input ENUM_APPLIED_PRICE CCIPrice = PRICE_WEIGHTED; input group "---- ATR: период волатильности ----" //--- Период индикатора ATR. input int ATRPeriod = 14; input group "---- MACD: периоды и цена расчёта ----" //--- Быстрый период MACD. input int FastPeriod = 12; //--- Медленный период MACD. input int SlowPeriod = 26; //--- Период сигнальной линии MACD. input int SignalPeriod = 9; //--- Цена, используемая MACD. input ENUM_APPLIED_PRICE MACDPrice = PRICE_CLOSE; const ulong DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES = 536870912; //+------------------------------------------------------------------+ //| Global missing-H1 interval and worst-case midnight event | //| capacity | //+------------------------------------------------------------------+ #define DPCC_HISTORY_CALENDAR_MAX_DAYS 14 #define DPCC_HISTORY_CALENDAR_EVENTS_PER_DAY 2 #define DPCC_HISTORY_CALENDAR_MAX_SPAN_SECONDS (DPCC_HISTORY_CALENDAR_MAX_DAYS*86400) #define DPCC_HISTORY_CALENDAR_MIN_EVENTS (DPCC_HISTORY_CALENDAR_MAX_DAYS*DPCC_HISTORY_CALENDAR_EVENTS_PER_DAY) #include "..\\NeuroNet_DNG\\NeuroNet.mqh" //+------------------------------------------------------------------+ //| Resolve a generated Action6 plan consistently across DPCC EAs | //+------------------------------------------------------------------+ bool DPCCResolveAction6Output(CNeuronDPCCOCL *actor, const bool forwarded, float &action[], string &reason) { ArrayFree(action); reason = ""; if(CheckPointer(actor) == POINTER_INVALID) { reason = "action6.actor_invalid"; return(false); } const ENUM_DPCC_STATUS status = actor.LastStatus(); if(forwarded && status == DPCC_NO_FEASIBLE_CANDIDATE && !actor.HasValidDecision()) { if(ArrayResize(action, 6) != 6) { reason = "action6.no_plan_output_allocation"; return(false); } ArrayInitialize(action, 0.0f); return(true); } if(!forwarded || status != DPCC_OK || !actor.HasValidDecision() || actor.getOutputVal(action) != 6 || ArraySize(action) != 6) { reason = "action6.forward_or_output"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(action[i])) { ArrayFree(action); reason = StringFormat("action6.nonfinite.%d", i); return(false); } return(true); } //+------------------------------------------------------------------+ //| Shared chart-local stop button used by long DPCC training stages | //+------------------------------------------------------------------+ class CDPCCStopButton { private: string m_name, m_prefix; public: CDPCCStopButton(void) { m_name = ""; m_prefix = ""; } string Name(void) const { return(m_name); } bool Create(const string stage, const string caption) { m_prefix = "DPCC_STOP_" + stage + "_" + (string)ChartID() + "_"; ObjectsDeleteAll(0, m_prefix, 0, OBJ_BUTTON); m_name = m_prefix + (string)GetMicrosecondCount(); if(!ObjectCreate(0, m_name, OBJ_BUTTON, 0, 0, 0)) { m_name = ""; return(false); } const int button_width = 160; const int right_margin = 20; bool configured = true; configured = ObjectSetInteger(0, m_name, OBJPROP_CORNER, CORNER_RIGHT_UPPER) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_XDISTANCE, button_width + right_margin) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_YDISTANCE, 5) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_XSIZE, button_width) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_YSIZE, 28) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_COLOR, clrWhite) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_BGCOLOR, clrFireBrick) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_BORDER_COLOR, clrMaroon) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_SELECTABLE, true) && configured; configured = ObjectSetInteger(0, m_name, OBJPROP_HIDDEN, false) && configured; configured = ObjectSetString(0, m_name, OBJPROP_TEXT, caption) && configured; if(!configured) { ObjectDelete(0, m_name); m_name = ""; return(false); } ChartRedraw(); return(true); } bool Clicked(const int id, const string object_name) const { return(id == CHARTEVENT_OBJECT_CLICK && m_name != "" && object_name == m_name); } bool SetText(const string caption) { if(m_name == "" || !ObjectSetString(0, m_name, OBJPROP_TEXT, caption)) return(false); ChartRedraw(); return(true); } void RemoveAdvisor(void) { EventKillTimer(); Comment(""); Delete(); ExpertRemove(); } void Delete(void) { if(m_prefix != "") ObjectsDeleteAll(0, m_prefix, 0, OBJ_BUTTON); else if(m_name != "") ObjectDelete(0, m_name); m_name = ""; m_prefix = ""; ChartRedraw(); } }; #ifndef DPCC_CHECKPOINT_ARTIFACTS_MQH #define DPCC_CHECKPOINT_ARTIFACTS_MQH //+------------------------------------------------------------------+ //| ArtifactResolver.mqh: bounded DPCC artifact byte resolver | //+------------------------------------------------------------------+ #ifndef DPCC_ARTIFACT_RESOLVER_MQH #define DPCC_ARTIFACT_RESOLVER_MQH //+------------------------------------------------------------------+ //| Readiness.mqh: typed DPCC resource-readiness core | //+------------------------------------------------------------------+ #ifndef DPCC_READINESS_MQH #define DPCC_READINESS_MQH #define DPCC_RESOURCE_MAX_MATRIX_ROWS 1000000 enum ENUM_DPCC_GATE_VERDICT { DPCC_GATE_NOT_RUN = 0, DPCC_GATE_PASS = 1, DPCC_GATE_FAIL = 2, DPCC_GATE_BLOCKED = 3 }; struct SDPCCResourceShape { long K, V, H, D, M, J, E, A, B, account_features; }; struct SDPCCResourceProfile { string readiness_profile_id, calibration_state, profile_id, device, driver, opencl_version, terminal_build; long frozen_at_utc; SDPCCResourceShape shapes; long projector_max_evaluations, projector_max_iterations; double p95_limit_ms, p99_limit_ms, max_compute_ms, max_decision_age_ms; long tracked_device_limit_bytes, total_device_limit_bytes, reserve_device_bytes; long warmup_decisions, measured_decisions, repeats; string censored_policy, algorithmic_limit, benchmark_cases[], stress_policy, execution_program; }; struct SDPCCResourceReportClaims { long evaluated_at_utc; ENUM_DPCC_GATE_VERDICT pretrain_resources, full_resources; }; struct SDPCCResourceReference { string path, sha256, id; }; struct SDPCCResourceMeasurement { string measurement_id, stage, profile_id, benchmark_case, device, driver, opencl_version, terminal_build, projection_mode, execution_program; long repeat_index, warmup_completed, projector_max_evaluations, projector_max_iterations, measured_started_at_utc; SDPCCResourceShape shapes; long attempts, completed, censored, timeouts, stale_decisions; double p50_ms, p95_ms, p99_ms, max_ms, max_decision_age_ms; long tracked_peak_bytes, total_peak_bytes, available_device_bytes; bool total_memory_telemetry_available; }; struct SDPCCResourceReportRow { SDPCCResourceMeasurement measurement; SDPCCResourceReference evidence[]; }; struct SDPCCResourceReport { SDPCCResourceReportClaims claims; string profile_ref_id, profile_ref_sha256; SDPCCResourceReportRow rows[]; }; struct SDPCCResourceSample { double duration_ms, decision_age_ms; bool censored, timeout, stale; long tracked_bytes, total_bytes, available_bytes; }; struct SDPCCResourceEvidence { string evidence_id, profile_ref_id, profile_ref_sha256; SDPCCResourceMeasurement measurement; SDPCCResourceSample samples[]; }; //+------------------------------------------------------------------+ //| Function DPCCResourceFiniteNonnegative | //+------------------------------------------------------------------+ bool DPCCResourceFiniteNonnegative(const double value) { return(MathIsValidNumber(value) && value >= 0.0); } //+------------------------------------------------------------------+ //| Function DPCCResourceClose | //+------------------------------------------------------------------+ bool DPCCResourceClose(const double left, const double right) { if(!MathIsValidNumber(left) || !MathIsValidNumber(right)) return(false); return(MathAbs(left - right) <= MathMax(1.0e-8, 1.0e-8 * MathMax(MathAbs(left), MathAbs(right)))); } //+------------------------------------------------------------------+ //| Function DPCCResourceShapeEqual | //+------------------------------------------------------------------+ bool DPCCResourceShapeEqual(const SDPCCResourceShape &left, const SDPCCResourceShape &right) { return(left.K == right.K && left.V == right.V && left.H == right.H && left.D == right.D && left.M == right.M && left.J == right.J && left.E == right.E && left.A == right.A && left.B == right.B && left.account_features == right.account_features); } //+------------------------------------------------------------------+ //| Function DPCCResourceMeasurementEqual | //+------------------------------------------------------------------+ bool DPCCResourceMeasurementEqual(const SDPCCResourceMeasurement &a, const SDPCCResourceMeasurement &b) { return(a.measurement_id == b.measurement_id && a.stage == b.stage && a.profile_id == b.profile_id && a.benchmark_case == b.benchmark_case && a.device == b.device && a.driver == b.driver && a.opencl_version == b.opencl_version && a.terminal_build == b.terminal_build && a.projection_mode == b.projection_mode && a.execution_program == b.execution_program && a.repeat_index == b.repeat_index && a.warmup_completed == b.warmup_completed && a.projector_max_evaluations == b.projector_max_evaluations && a.projector_max_iterations == b.projector_max_iterations && a.measured_started_at_utc == b.measured_started_at_utc && DPCCResourceShapeEqual(a.shapes, b.shapes) && a.attempts == b.attempts && a.completed == b.completed && a.censored == b.censored && a.timeouts == b.timeouts && a.stale_decisions == b.stale_decisions && DPCCResourceClose(a.p50_ms, b.p50_ms) && DPCCResourceClose(a.p95_ms, b.p95_ms) && DPCCResourceClose(a.p99_ms, b.p99_ms) && DPCCResourceClose(a.max_ms, b.max_ms) && DPCCResourceClose(a.max_decision_age_ms, b.max_decision_age_ms) && a.tracked_peak_bytes == b.tracked_peak_bytes && a.total_peak_bytes == b.total_peak_bytes && a.available_device_bytes == b.available_device_bytes && a.total_memory_telemetry_available == b.total_memory_telemetry_available); } //+------------------------------------------------------------------+ //| Function DPCCBindResourceReportRow | //+------------------------------------------------------------------+ bool DPCCBindResourceReportRow(const SDPCCResourceProfile &profile, const SDPCCResourceReportRow &row, const SDPCCResourceEvidence &evidence, const string caller_observed_evidence_sha256, const string caller_observed_profile_sha256, string &reason) { if(evidence.profile_ref_id != profile.readiness_profile_id) { reason = "readiness_profile.id"; return(false); } if(evidence.profile_ref_sha256 != caller_observed_profile_sha256) { reason = "readiness_profile.sha256"; return(false); } if(row.measurement.profile_id != evidence.measurement.profile_id || !DPCCResourceMeasurementEqual(row.measurement, evidence.measurement)) { reason = "resource_measurements.measurement"; return(false); } for(int i = 0; i < ArraySize(row.evidence); i++) if(row.evidence[i].id == evidence.evidence_id && row.evidence[i].sha256 == caller_observed_evidence_sha256) { reason = ""; return(true); } reason = "resource_measurements.evidence"; return(false); } //+------------------------------------------------------------------+ //| Function DPCCBindResourceReportProfile | //+------------------------------------------------------------------+ bool DPCCBindResourceReportProfile(const SDPCCResourceProfile &profile, const SDPCCResourceReport &report, const string caller_observed_sha256, string &reason) { if(report.profile_ref_id != profile.readiness_profile_id) { reason = "readiness_profile.id"; return(false); } if(report.profile_ref_sha256 != caller_observed_sha256) { reason = "readiness_profile.sha256"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCResourceCaseIndex | //+------------------------------------------------------------------+ int DPCCResourceCaseIndex(const SDPCCResourceProfile &profile, const string value) { for(int i = 0; i < ArraySize(profile.benchmark_cases); i++) if(profile.benchmark_cases[i] == value) return(i); return(-1); } //+------------------------------------------------------------------+ //| Function DPCCResourceModeIndex | //+------------------------------------------------------------------+ int DPCCResourceModeIndex(const string value) { if(value == "P0") return(0); if(value == "P1") return(1); if(value == "P2") return(2); return(-1); } //+------------------------------------------------------------------+ //| Function DPCCRecomputeResourceEvidence | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCRecomputeResourceEvidence(const SDPCCResourceProfile &profile, const SDPCCResourceEvidence &evidence, SDPCCResourceMeasurement &derived, string &reason) { derived = evidence.measurement; reason = ""; if(evidence.profile_ref_id != profile.readiness_profile_id) { reason = "readiness_profile.id"; return(DPCC_GATE_BLOCKED); } const int count = ArraySize(evidence.samples); if(count != (int)evidence.measurement.attempts) { reason = "measurement.attempts.samples"; return(DPCC_GATE_FAIL); } if(count <= 0) { reason = "samples.empty"; return(DPCC_GATE_FAIL); } double completed_values[]; ArrayResize(completed_values, count); int completed_count = 0; long censored_count = 0, timeout_count = 0, stale_count = 0; double max_duration = 0.0, max_age = 0.0; long tracked_peak = 0, total_peak = 0, available_min = LONG_MAX; for(int i = 0; i < count; i++) { const SDPCCResourceSample sample = evidence.samples[i]; if(!DPCCResourceFiniteNonnegative(sample.duration_ms) || !DPCCResourceFiniteNonnegative(sample.decision_age_ms) || sample.tracked_bytes < 0 || sample.total_bytes < 0 || sample.available_bytes < 0 || sample.tracked_bytes > sample.total_bytes) { reason = StringFormat("samples[%d].numeric", i); return(DPCC_GATE_FAIL); } const bool timeout = (sample.duration_ms > profile.max_compute_ms); const bool stale = (sample.decision_age_ms > profile.max_decision_age_ms); if(sample.timeout != timeout) { reason = StringFormat("samples[%d].timeout", i); return(DPCC_GATE_FAIL); } if(sample.stale != stale) { reason = StringFormat("samples[%d].stale", i); return(DPCC_GATE_FAIL); } if(sample.censored) censored_count++; else completed_values[completed_count++] = sample.duration_ms; if(timeout) timeout_count++; if(stale) stale_count++; max_duration = MathMax(max_duration, sample.duration_ms); max_age = MathMax(max_age, sample.decision_age_ms); tracked_peak = MathMax(tracked_peak, sample.tracked_bytes); total_peak = MathMax(total_peak, sample.total_bytes); available_min = MathMin(available_min, sample.available_bytes); } ArrayResize(completed_values, completed_count); ArraySort(completed_values); derived.attempts = count; derived.completed = completed_count; derived.censored = censored_count; derived.timeouts = timeout_count; derived.stale_decisions = stale_count; derived.p50_ms = 0.0; derived.p95_ms = 0.0; derived.p99_ms = 0.0; if(completed_count > 0) { derived.p50_ms = ((completed_count % 2) == 0 ? (completed_values[completed_count / 2 - 1] + completed_values[completed_count / 2]) / 2.0 : completed_values[completed_count / 2]); derived.p95_ms = completed_values[(int)MathCeil(0.95 * completed_count) - 1]; derived.p99_ms = completed_values[(int)MathCeil(0.99 * completed_count) - 1]; } derived.max_ms = max_duration; derived.max_decision_age_ms = max_age; derived.tracked_peak_bytes = tracked_peak; derived.total_peak_bytes = total_peak; derived.available_device_bytes = available_min; if(evidence.measurement.completed != derived.completed || evidence.measurement.censored != derived.censored || evidence.measurement.timeouts != derived.timeouts || evidence.measurement.stale_decisions != derived.stale_decisions || !DPCCResourceClose(evidence.measurement.p50_ms, derived.p50_ms) || !DPCCResourceClose(evidence.measurement.p95_ms, derived.p95_ms) || !DPCCResourceClose(evidence.measurement.p99_ms, derived.p99_ms) || !DPCCResourceClose(evidence.measurement.max_ms, derived.max_ms) || !DPCCResourceClose(evidence.measurement.max_decision_age_ms, derived.max_decision_age_ms) || evidence.measurement.tracked_peak_bytes != derived.tracked_peak_bytes || evidence.measurement.total_peak_bytes != derived.total_peak_bytes || evidence.measurement.available_device_bytes != derived.available_device_bytes) { reason = "measurement.summary"; return(DPCC_GATE_FAIL); } if(censored_count > 0 || timeout_count > 0 || stale_count > 0) { reason = "measurement.sample_failure"; return(DPCC_GATE_FAIL); } return(DPCC_GATE_PASS); } //+------------------------------------------------------------------+ //| Function DPCCEvaluateResourceMeasurement | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCEvaluateResourceMeasurement(const SDPCCResourceProfile &profile, const SDPCCResourceMeasurement &measurement, const string expected_stage, const long evaluated_at_utc, string &reason) { reason = ""; if(profile.calibration_state != "FROZEN") { reason = "profile.calibration_state"; return(DPCC_GATE_BLOCKED); } if(expected_stage != "FULL_PIPELINE" && expected_stage != "PRETRAIN_ESTIMATE") { reason = "stage"; return(DPCC_GATE_BLOCKED); } if(measurement.stage != expected_stage) { reason = "measurement.stage"; return(DPCC_GATE_BLOCKED); } if(measurement.profile_id != profile.profile_id) { reason = "measurement.profile_id"; return(DPCC_GATE_BLOCKED); } if(measurement.device != profile.device) { reason = "measurement.device"; return(DPCC_GATE_BLOCKED); } if(measurement.driver != profile.driver) { reason = "measurement.driver"; return(DPCC_GATE_BLOCKED); } if(measurement.opencl_version != profile.opencl_version) { reason = "measurement.opencl_version"; return(DPCC_GATE_BLOCKED); } if(measurement.terminal_build != profile.terminal_build) { reason = "measurement.terminal_build"; return(DPCC_GATE_BLOCKED); } if(measurement.execution_program != profile.execution_program) { reason = "measurement.execution_program"; return(DPCC_GATE_BLOCKED); } if(!DPCCResourceShapeEqual(measurement.shapes, profile.shapes)) { reason = "measurement.shapes"; return(DPCC_GATE_BLOCKED); } if(measurement.projector_max_evaluations != profile.projector_max_evaluations) { reason = "measurement.projector_max_evaluations"; return(DPCC_GATE_BLOCKED); } if(measurement.projector_max_iterations != profile.projector_max_iterations) { reason = "measurement.projector_max_iterations"; return(DPCC_GATE_BLOCKED); } if(measurement.warmup_completed != profile.warmup_decisions) { reason = "measurement.warmup_completed"; return(DPCC_GATE_BLOCKED); } if(measurement.attempts != profile.measured_decisions) { reason = "measurement.attempts"; return(DPCC_GATE_BLOCKED); } if(measurement.measured_started_at_utc <= profile.frozen_at_utc) { reason = "measurement.measured_started_at_utc"; return(DPCC_GATE_BLOCKED); } if(evaluated_at_utc < measurement.measured_started_at_utc) { reason = "evaluated_at_utc"; return(DPCC_GATE_BLOCKED); } if(!measurement.total_memory_telemetry_available) { reason = "measurement.total_memory_telemetry_available"; return(DPCC_GATE_BLOCKED); } if(DPCCResourceModeIndex(measurement.projection_mode) < 0) { reason = "measurement.projection_mode"; return(DPCC_GATE_BLOCKED); } if(DPCCResourceCaseIndex(profile, measurement.benchmark_case) < 0) { reason = "measurement.benchmark_case"; return(DPCC_GATE_BLOCKED); } if(measurement.repeat_index < 0 || measurement.repeat_index >= profile.repeats) { reason = "measurement.repeat_index"; return(DPCC_GATE_BLOCKED); } if(!DPCCResourceFiniteNonnegative(measurement.p50_ms) || !DPCCResourceFiniteNonnegative(measurement.p95_ms) || !DPCCResourceFiniteNonnegative(measurement.p99_ms) || !DPCCResourceFiniteNonnegative(measurement.max_ms) || !DPCCResourceFiniteNonnegative(measurement.max_decision_age_ms)) { reason = "measurement.numeric"; return(DPCC_GATE_FAIL); } if(measurement.censored > 0) { reason = "measurement.censored"; return(DPCC_GATE_FAIL); } if(measurement.timeouts > 0) { reason = "measurement.timeouts"; return(DPCC_GATE_FAIL); } if(measurement.stale_decisions > 0) { reason = "measurement.stale_decisions"; return(DPCC_GATE_FAIL); } if(measurement.p95_ms > profile.p95_limit_ms) { reason = "measurement.p95_ms"; return(DPCC_GATE_FAIL); } if(measurement.p99_ms > profile.p99_limit_ms) { reason = "measurement.p99_ms"; return(DPCC_GATE_FAIL); } if(measurement.max_ms > profile.max_compute_ms) { reason = "measurement.max_ms"; return(DPCC_GATE_FAIL); } if(measurement.max_decision_age_ms > profile.max_decision_age_ms) { reason = "measurement.max_decision_age_ms"; return(DPCC_GATE_FAIL); } if(measurement.tracked_peak_bytes > profile.tracked_device_limit_bytes) { reason = "measurement.tracked_peak_bytes"; return(DPCC_GATE_FAIL); } if(measurement.total_peak_bytes > profile.total_device_limit_bytes) { reason = "measurement.total_peak_bytes"; return(DPCC_GATE_FAIL); } if(measurement.total_peak_bytes > measurement.available_device_bytes || profile.reserve_device_bytes > measurement.available_device_bytes - measurement.total_peak_bytes) { reason = "measurement.available_device_bytes"; return(DPCC_GATE_FAIL); } return(DPCC_GATE_PASS); } //+------------------------------------------------------------------+ //| Function DPCCEvaluateResourceGate | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCEvaluateResourceGate(const SDPCCResourceProfile &profile, const string stage, const long evaluated_at_utc, const SDPCCResourceMeasurement &measurements[], const ENUM_DPCC_GATE_VERDICT &raw_verdicts[], string &reason) { reason = ""; const int count = ArraySize(measurements); if(count == 0) { reason = "resource_measurements.empty"; return(DPCC_GATE_NOT_RUN); } bool blocked = false, failed = false; const int case_count = ArraySize(profile.benchmark_cases); if(case_count <= 0 || profile.repeats <= 0 || case_count > DPCC_RESOURCE_MAX_MATRIX_ROWS / 3 || profile.repeats > DPCC_RESOURCE_MAX_MATRIX_ROWS / (3 * case_count)) { reason = "resource_measurements.capacity"; return(DPCC_GATE_BLOCKED); } const long expected_long = 3L * case_count * profile.repeats; if(expected_long <= 0 || expected_long > DPCC_RESOURCE_MAX_MATRIX_ROWS) { reason = "resource_measurements.capacity"; return(DPCC_GATE_BLOCKED); } const int expected = (int)expected_long; bool seen[]; if(ArrayResize(seen, expected) != expected) { reason = "resource_measurements.capacity"; return(DPCC_GATE_BLOCKED); } ArrayInitialize(seen, false); for(int i = 0; i < count; i++) { string row_reason = ""; ENUM_DPCC_GATE_VERDICT verdict = (i < ArraySize(raw_verdicts) ? raw_verdicts[i] : DPCC_GATE_BLOCKED); const ENUM_DPCC_GATE_VERDICT evaluated = DPCCEvaluateResourceMeasurement(profile, measurements[i], stage, evaluated_at_utc, row_reason); if(verdict == DPCC_GATE_FAIL || evaluated == DPCC_GATE_FAIL) failed = true; if(verdict == DPCC_GATE_BLOCKED || evaluated == DPCC_GATE_BLOCKED) blocked = true; const int mode = DPCCResourceModeIndex(measurements[i].projection_mode); const int scenario = DPCCResourceCaseIndex(profile, measurements[i].benchmark_case); const long repeat = measurements[i].repeat_index; if(mode < 0 || scenario < 0 || repeat < 0 || repeat >= profile.repeats) { blocked = true; continue; } const int key = (mode * ArraySize(profile.benchmark_cases) + scenario) * (int)profile.repeats + (int)repeat; if(seen[key]) blocked = true; seen[key] = true; } if(count != expected) blocked = true; for(int i = 0; i < expected; i++) if(!seen[i]) blocked = true; if(failed) { reason = (blocked ? "resource_measurements.fail;resource_measurements.matrix" : "resource_measurements.fail"); return(DPCC_GATE_FAIL); } if(blocked) { reason = "resource_measurements.matrix"; return(DPCC_GATE_BLOCKED); } return(DPCC_GATE_PASS); } #endif #define DPCC_RESOLVER_MAX_JSON_BYTES 4194304 struct SDPCCArtifactResolveResult { string canonical_path; string observed_sha256; bool sha256_matches; uchar bytes[]; }; //+------------------------------------------------------------------+ //| Function DPCCClearArtifactResolveResult | //+------------------------------------------------------------------+ void DPCCClearArtifactResolveResult(SDPCCArtifactResolveResult &result) { result.canonical_path = ""; result.observed_sha256 = ""; result.sha256_matches = false; ArrayResize(result.bytes, 0); } //+------------------------------------------------------------------+ //| Function DPCCResolverLowerSha256 | //+------------------------------------------------------------------+ bool DPCCResolverLowerSha256(const string value) { if(StringLen(value) != 64) return(false); for(int i = 0; i < 64; i++) { const ushort code = StringGetCharacter(value, i); if(!((code >= '0' && code <= '9') || (code >= 'a' && code <= 'f'))) return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCResolverPreparePath | //+------------------------------------------------------------------+ bool DPCCResolverPreparePath(const string source, string &normalized, string &reason, const string field) { normalized = ""; const int length = StringLen(source); if(length <= 0) { reason = field + ".empty"; return(false); } for(int i = 0; i < length; i++) { const ushort code = StringGetCharacter(source, i); if(code < 32 || code == 127) { reason = field + ".control"; return(false); } if(code == ':' || code == '*' || code == '?') { reason = field + ".portable"; return(false); } } normalized = source; StringReplace(normalized, "\\", "/"); if(StringLen(normalized) <= 0 || StringGetCharacter(normalized, 0) == '/') { normalized = ""; reason = field + ".absolute"; return(false); } if(StringGetCharacter(normalized, StringLen(normalized) - 1) == '/') { normalized = ""; reason = field + ".trailing_separator"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCResolverFinalSegment | //+------------------------------------------------------------------+ bool DPCCResolverFinalSegment(const string path, string &segment) { segment = ""; const int length = StringLen(path); int start = 0; for(int i = 0; i < length; i++) if(StringGetCharacter(path, i) == '/') start = i + 1; segment = StringSubstr(path, start); return(segment != "" && segment != "." && segment != ".."); } //+------------------------------------------------------------------+ //| Function DPCCResolverAppendSegments | //+------------------------------------------------------------------+ bool DPCCResolverAppendSegments(const string path, string &segments[], const int floor, string &reason, const string field) { const int length = StringLen(path); int start = 0; for(int i = 0; i <= length; i++) { if(i < length && StringGetCharacter(path, i) != '/') continue; const string segment = StringSubstr(path, start, i - start); start = i + 1; if(segment == "" || segment == ".") continue; if(segment == "..") { const int count = ArraySize(segments); if(count <= floor) { reason = field + ".escape"; return(false); } if(ArrayResize(segments, count - 1) != count - 1) { reason = field + ".capacity"; return(false); } continue; } const int count = ArraySize(segments); if(ArrayResize(segments, count + 1) != count + 1) { reason = field + ".capacity"; return(false); } segments[count] = segment; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCResolverJoinSegments | //+------------------------------------------------------------------+ string DPCCResolverJoinSegments(const string &segments[]) { string result = ""; for(int i = 0; i < ArraySize(segments); i++) { if(i > 0) result += "/"; result += segments[i]; } return(result); } //+------------------------------------------------------------------+ //| Function DPCCNormalizePackagePath | //+------------------------------------------------------------------+ bool DPCCNormalizePackagePath(const string package_root_relative, const string containing_json_package_path, const string declared_path, string &canonical_sandbox_path, string &reason) { canonical_sandbox_path = ""; reason = ""; string root = "", containing = "", declared = "", final_segment = ""; if(!DPCCResolverPreparePath(package_root_relative, root, reason, "package_root_relative") || !DPCCResolverPreparePath(containing_json_package_path, containing, reason, "containing_json_path") || !DPCCResolverPreparePath(declared_path, declared, reason, "reference.path")) return(false); if(!DPCCResolverFinalSegment(root, final_segment) || !DPCCResolverFinalSegment(containing, final_segment) || !DPCCResolverFinalSegment(declared, final_segment)) { reason = "path.final_segment"; return(false); } string segments[]; if(!DPCCResolverAppendSegments(root, segments, 0, reason, "package_root_relative")) return(false); const int root_count = ArraySize(segments); if(root_count <= 0) { reason = "package_root_relative.empty"; return(false); } if(!DPCCResolverAppendSegments(containing, segments, root_count, reason, "containing_json_path")) return(false); if(ArraySize(segments) <= root_count) { reason = "containing_json_path.empty"; return(false); } if(ArrayResize(segments, ArraySize(segments) - 1) < root_count) { reason = "containing_json_path.capacity"; return(false); } if(!DPCCResolverAppendSegments(declared, segments, root_count, reason, "reference.path")) return(false); if(ArraySize(segments) <= root_count) { reason = "reference.path.empty"; return(false); } canonical_sandbox_path = DPCCResolverJoinSegments(segments); return(true); } //+------------------------------------------------------------------+ //| Function DPCCReadArtifactBytes | //+------------------------------------------------------------------+ bool DPCCReadArtifactBytes(const string canonical_sandbox_path, const bool common_files, const long max_bytes, uchar &bytes[], string &reason) { ArrayResize(bytes, 0); reason = ""; if(max_bytes < 0) { reason = "max_bytes"; return(false); } int flags = FILE_READ | FILE_BIN | FILE_SHARE_READ; if(common_files) flags |= FILE_COMMON; ResetLastError(); const int handle = FileOpen(canonical_sandbox_path, flags); if(handle == INVALID_HANDLE) { reason = StringFormat("file.open.%d", GetLastError()); return(false); } ResetLastError(); const ulong length = FileSize(handle); const int size_error = GetLastError(); if(size_error != 0) { FileClose(handle); reason = StringFormat("file.size.%d", size_error); return(false); } if(length > (ulong)max_bytes || length > (ulong)INT_MAX) { FileClose(handle); reason = "file.size.limit"; return(false); } const int count = (int)length; if(ArrayResize(bytes, count) != count) { FileClose(handle); ArrayResize(bytes, 0); reason = "file.bytes.capacity"; return(false); } if(count > 0) { ResetLastError(); const uint read = FileReadArray(handle, bytes, 0, count); if(read != (uint)count) { const int read_error = GetLastError(); FileClose(handle); ArrayResize(bytes, 0); reason = StringFormat("file.read.%u.%d", read, read_error); return(false); } } FileClose(handle); return(true); } //+------------------------------------------------------------------+ //| Function DPCCSHA256Lower | //+------------------------------------------------------------------+ bool DPCCSHA256Lower(const uchar &bytes[], string &digest_hex, string &reason) { digest_hex = ""; reason = ""; //--- MQL5 CryptEncode returns zero for an empty source, the same value it //--- uses for an API failure. Preserve the standard SHA-256 empty message. if(ArraySize(bytes) == 0) { digest_hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; return(true); } uchar key[], digest[]; ArrayResize(key, 0); ResetLastError(); const int result = CryptEncode(CRYPT_HASH_SHA256, bytes, key, digest); if(result != 32 || ArraySize(digest) != 32) { reason = StringFormat("sha256.%d.%d.%d", result, ArraySize(digest), GetLastError()); return(false); } for(int i = 0; i < 32; i++) digest_hex += StringFormat("%02x", (uint)digest[i]); return(true); } //+------------------------------------------------------------------+ //| Function DPCCResolveArtifact | //+------------------------------------------------------------------+ bool DPCCResolveArtifact(const string package_root_relative, const string containing_json_package_path, const SDPCCResourceReference &reference, const bool common_files, const long max_bytes, SDPCCArtifactResolveResult &result, string &reason) { DPCCClearArtifactResolveResult(result); reason = ""; if(!DPCCResolverLowerSha256(reference.sha256)) { reason = "reference.sha256"; return(false); } if(!DPCCNormalizePackagePath(package_root_relative, containing_json_package_path, reference.path, result.canonical_path, reason)) { DPCCClearArtifactResolveResult(result); return(false); } uchar bytes[]; if(!DPCCReadArtifactBytes(result.canonical_path, common_files, max_bytes, bytes, reason)) { DPCCClearArtifactResolveResult(result); return(false); } if(ArrayCopy(result.bytes, bytes) != ArraySize(bytes)) { DPCCClearArtifactResolveResult(result); reason = "result.bytes.capacity"; return(false); } if(!DPCCSHA256Lower(result.bytes, result.observed_sha256, reason)) { DPCCClearArtifactResolveResult(result); return(false); } result.sha256_matches = (result.observed_sha256 == reference.sha256); if(!result.sha256_matches) { reason = "reference.sha256.mismatch"; return(false); } return(true); } #endif //+------------------------------------------------------------------+ //| IncrementalSHA256.mqh: incremental SHA-256 | //+------------------------------------------------------------------+ #ifndef DPCC_INCREMENTAL_SHA256_MQH #define DPCC_INCREMENTAL_SHA256_MQH #define DPCC_SHA256_BLOCK_BYTES 64 #define DPCC_SHA256_DIGEST_BYTES 32 #define DPCC_SHA256_MAX_BYTES 0x1FFFFFFFFFFFFFFF //+------------------------------------------------------------------+ //| SHA-256 round constants | //+------------------------------------------------------------------+ const uint EXTDPCCSHA256K[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 }; //+------------------------------------------------------------------+ //| CDPCCIncrementalSHA256 hashes arbitrary byte chunks | //+------------------------------------------------------------------+ class CDPCCIncrementalSHA256 { private: uint m_state[8]; uchar m_block[DPCC_SHA256_BLOCK_BYTES]; uint m_used; ulong m_total_bytes; bool m_initialized; bool m_finalized; //+------------------------------------------------------------------+ //| RotateRight performs a 32-bit right rotation | //+------------------------------------------------------------------+ uint RotateRight(const uint value, const uint shift) { return((value >> shift) | (value << (32 - shift))); } //+------------------------------------------------------------------+ //| CompressBlock processes one complete 64-byte block | //+------------------------------------------------------------------+ void CompressBlock() { uint words[64]; for(int i = 0; i < 16; i++) { const int offset = i * 4; words[i] = ((uint)m_block[offset] << 24) | ((uint)m_block[offset + 1] << 16) | ((uint)m_block[offset + 2] << 8) | (uint)m_block[offset + 3]; } for(int i = 16; i < 64; i++) { const uint x = words[i - 15]; const uint y = words[i - 2]; const uint sigma0 = RotateRight(x, 7) ^ RotateRight(x, 18) ^ (x >> 3); const uint sigma1 = RotateRight(y, 17) ^ RotateRight(y, 19) ^ (y >> 10); const ulong sum = (ulong)words[i - 16] + (ulong)sigma0 + (ulong)words[i - 7] + (ulong)sigma1; words[i] = (uint)(sum & 0xFFFFFFFF); } uint a = m_state[0]; uint b = m_state[1]; uint c = m_state[2]; uint d = m_state[3]; uint e = m_state[4]; uint f = m_state[5]; uint g = m_state[6]; uint h = m_state[7]; for(int i = 0; i < 64; i++) { const uint sum1 = RotateRight(e, 6) ^ RotateRight(e, 11) ^ RotateRight(e, 25); const uint choose = (e & f) ^ (~e & g); const ulong t1 = (ulong)h + (ulong)sum1 + (ulong)choose + (ulong)EXTDPCCSHA256K[i] + (ulong)words[i]; const uint sum0 = RotateRight(a, 2) ^ RotateRight(a, 13) ^ RotateRight(a, 22); const uint majority = (a & b) ^ (a & c) ^ (b & c); const ulong t2 = (ulong)sum0 + (ulong)majority; h = g; g = f; f = e; e = d + (uint)(t1 & 0xFFFFFFFF); d = c; c = b; b = a; a = (uint)((t1 + t2) & 0xFFFFFFFF); } m_state[0] += (uint)a; m_state[1] += (uint)b; m_state[2] += (uint)c; m_state[3] += (uint)d; m_state[4] += (uint)e; m_state[5] += (uint)f; m_state[6] += (uint)g; m_state[7] += (uint)h; } public: //+------------------------------------------------------------------+ //| Init resets the hash state for a new message | //+------------------------------------------------------------------+ bool Init(string &reason) { reason = ""; m_state[0] = 0x6a09e667; m_state[1] = 0xbb67ae85; m_state[2] = 0x3c6ef372; m_state[3] = 0xa54ff53a; m_state[4] = 0x510e527f; m_state[5] = 0x9b05688c; m_state[6] = 0x1f83d9ab; m_state[7] = 0x5be0cd19; for(int i = 0; i < DPCC_SHA256_BLOCK_BYTES; i++) m_block[i] = 0; m_used = 0; m_total_bytes = 0; m_initialized = true; m_finalized = false; return(true); } //+------------------------------------------------------------------+ //| Update consumes a byte range without retaining the whole message | //+------------------------------------------------------------------+ bool Update(const uchar &data[], const int offset, const int length, string &reason) { reason = ""; if(!m_initialized) { reason = "sha256.state.uninitialized"; return(false); } if(m_finalized) { reason = "sha256.state.finalized"; return(false); } if(offset < 0 || length < 0 || offset > ArraySize(data) || length > ArraySize(data) - offset) { reason = "sha256.input.range"; return(false); } if(m_total_bytes > DPCC_SHA256_MAX_BYTES - (ulong)length) { reason = "sha256.input.length_overflow"; return(false); } m_total_bytes += (ulong)length; for(int i = 0; i < length; i++) { m_block[m_used] = data[offset + i]; m_used++; if(m_used == DPCC_SHA256_BLOCK_BYTES) { CompressBlock(); m_used = 0; } } return(true); } //+------------------------------------------------------------------+ //| Final emits digest bytes and lowercase hexadecimal | //+------------------------------------------------------------------+ bool Final(uchar &digest[], string &hex, string &reason) { ArrayResize(digest, 0); hex = ""; reason = ""; if(!m_initialized) { reason = "sha256.state.uninitialized"; return(false); } if(m_finalized) { reason = "sha256.state.finalized"; return(false); } m_finalized = true; const ulong bit_length = m_total_bytes * 8; m_block[m_used] = 0x80; m_used++; if(m_used > 56) { for(uint i = m_used; i < DPCC_SHA256_BLOCK_BYTES; i++) m_block[i] = 0; CompressBlock(); m_used = 0; } for(uint i = m_used; i < 56; i++) m_block[i] = 0; for(uint i = 0; i < 8; i++) m_block[63 - i] = (uchar)(bit_length >> (i * 8)); CompressBlock(); if(ArrayResize(digest, DPCC_SHA256_DIGEST_BYTES) != DPCC_SHA256_DIGEST_BYTES) { ArrayResize(digest, 0); reason = "sha256.output.capacity"; return(false); } for(int i = 0; i < 8; i++) { digest[i * 4] = (uchar)(m_state[i] >> 24); digest[i * 4 + 1] = (uchar)(m_state[i] >> 16); digest[i * 4 + 2] = (uchar)(m_state[i] >> 8); digest[i * 4 + 3] = (uchar)m_state[i]; } const string alphabet = "0123456789abcdef"; for(int i = 0; i < DPCC_SHA256_DIGEST_BYTES; i++) { hex += StringSubstr(alphabet, (int)(digest[i] >> 4), 1); hex += StringSubstr(alphabet, (int)(digest[i] & 0x0F), 1); } return(true); } }; #endif //+------------------------------------------------------------------+ //| DPCCAP01 bounded streaming role frames | //+------------------------------------------------------------------+ #define DPCC_AP_HEADER_BYTES 128 #define DPCC_AP_SECTION_BYTES 64 #define DPCC_AP_FOOTER_BYTES 48 #define DPCC_AP_HASH_CHUNK 65536 #define DPCC_AP_CONTRACT "ACSRM_DPCC_ACTION6_PRICE_V1" #define DPCC_M05_MARKET_REVISION "CNET_ACTION6_FACTUAL_FEATURES_STAGE_PROFILE_SHA_H1_OHLC_20261017" #define DPCC_TRADING_REVISION "CNET_ACTION6_20261014" #define DPCC_M05_TARGET_POLICY "FUTURE_FEATURE_PERIOD_NORM_REMAINING_SUFFIX_RESAMPLE_V1" #define DPCC_M05_PRICE_TARGET_POLICY "SERVER_H1_BAR_OPEN_FUTURE_H1_OHLC_V1" struct SDPCCAPSection { uint id; uint encoding; ulong offset; ulong length; string sha256; }; //--- Trusted caller contract for one native CNet section, not admission evidence. struct SDPCCAPNativeLoadContract { uint role; uint section_id; uint encoding; ulong max_native_bytes; string source_abi_sha256; //--- Full profile snapshot SHA; Market retains its producer snapshot. string profile_sha256; //--- Market-only projection SHA; empty for Trading roles. string market_profile_sha256; string data_sha256; //--- Empty only for the Market role. string market_dependency_sha256; string source_archive_sha256; string task_id; string stage; bool market_ready; bool calibration_completed; bool stage02_quality_pass; }; //+------------------------------------------------------------------+ //| Read a little-endian unsigned field from a bounded record | //+------------------------------------------------------------------+ ulong DPCCAPReadUnsigned(const uchar &bytes[], const int offset, const int width) { ulong value = 0; for(int i = 0; i < width; i++) value |= ((ulong)bytes[offset + i] << (i * 8)); return(value); } //+------------------------------------------------------------------+ //| Write a little-endian unsigned field in a bounded record | //+------------------------------------------------------------------+ void DPCCAPWriteUnsigned(uchar &bytes[], const int offset, const int width, const ulong value) { for(int i = 0; i < width; i++) bytes[offset + i] = (uchar)(value >> (i * 8)); } //+------------------------------------------------------------------+ //| Compare the exact ASCII magic without a terminating zero | //+------------------------------------------------------------------+ bool DPCCAPMagic(const uchar &bytes[], const int offset, const string magic) { for(int i = 0; i < StringLen(magic); i++) if(bytes[offset + i] != (uchar)StringGetCharacter(magic, i)) return(false); return(true); } //+------------------------------------------------------------------+ //| Copy an exact ASCII magic into a bounded record | //+------------------------------------------------------------------+ void DPCCAPWriteMagic(uchar &bytes[], const int offset, const string magic) { for(int i = 0; i < StringLen(magic); i++) bytes[offset + i] = (uchar)StringGetCharacter(magic, i); } //+------------------------------------------------------------------+ //| Decode a validated lowercase SHA256 into its raw digest | //+------------------------------------------------------------------+ bool DPCCAPWriteSHA(uchar &bytes[], const int offset, const string hex) { if(!DPCCResolverLowerSha256(hex)) return(false); const string alphabet = "0123456789abcdef"; for(int i = 0; i < 32; i++) bytes[offset + i] = (uchar)(StringFind(alphabet, StringSubstr(hex, i * 2, 1)) * 16 + StringFind(alphabet, StringSubstr(hex, i * 2 + 1, 1))); return(true); } //+------------------------------------------------------------------+ //| Encode a raw SHA256 field without retaining its payload | //+------------------------------------------------------------------+ string DPCCAPReadSHA(const uchar &bytes[], const int offset) { const string alphabet = "0123456789abcdef"; string hex = ""; for(int i = 0; i < 32; i++) { hex += StringSubstr(alphabet, (int)(bytes[offset + i] >> 4), 1); hex += StringSubstr(alphabet, (int)(bytes[offset + i] & 15), 1); } return(hex); } //+------------------------------------------------------------------+ //| Hash the exact UTF8 contract identifier | //+------------------------------------------------------------------+ bool DPCCAPContractSHA(string &hex, string &reason) { uchar bytes[], digest[]; const int count = StringToCharArray(DPCC_AP_CONTRACT, bytes, 0, WHOLE_ARRAY, CP_UTF8); CDPCCIncrementalSHA256 hash; if(count < 1) { reason = "dpccap.contract.utf8"; return(false); } if(!hash.Init(reason) || !hash.Update(bytes, 0, count - 1, reason)) return(false); return(hash.Final(digest, hex, reason)); } //+------------------------------------------------------------------+ //| Read an exact fixed record from an existing binary handle | //+------------------------------------------------------------------+ bool DPCCAPReadRecord(const int handle, const ulong offset, const int count, uchar &bytes[], string &reason) { if(offset > (ulong)LONG_MAX || count < 0 || ArrayResize(bytes, count) != count) { reason = "dpccap.record.capacity"; return(false); } ResetLastError(); if(!FileSeek(handle, (long)offset, SEEK_SET) || FileReadArray(handle, bytes, 0, count) != (uint)count) { reason = StringFormat("dpccap.record.read.%d", GetLastError()); return(false); } return(true); } //+------------------------------------------------------------------+ //| Write an exact fixed record without a second frame copy | //+------------------------------------------------------------------+ bool DPCCAPWriteRecord(const int handle, const ulong offset, const uchar &bytes[], string &reason) { ResetLastError(); if(offset > (ulong)LONG_MAX || !FileSeek(handle, (long)offset, SEEK_SET) || FileWriteArray(handle, bytes, 0, ArraySize(bytes)) != (uint)ArraySize(bytes)) { reason = StringFormat("dpccap.record.write.%d", GetLastError()); return(false); } return(true); } //+------------------------------------------------------------------+ //| Hash a bounded file range through one fixed-size scratch buffer | //+------------------------------------------------------------------+ bool DPCCAPHashRange(const int handle, const ulong offset, const ulong length, string &hex, string &reason) { hex = ""; const ulong size = FileSize(handle); if(offset > size || length > size - offset || offset > (ulong)LONG_MAX) { reason = "dpccap.hash.range"; return(false); } uchar bytes[], digest[]; if(ArrayResize(bytes, DPCC_AP_HASH_CHUNK) != DPCC_AP_HASH_CHUNK) { reason = "dpccap.hash.capacity"; return(false); } CDPCCIncrementalSHA256 hash; if(!hash.Init(reason)) return(false); if(!FileSeek(handle, (long)offset, SEEK_SET)) { reason = "dpccap.hash.seek"; return(false); } ulong remaining = length; while(remaining > 0) { const int count = (remaining > DPCC_AP_HASH_CHUNK ? DPCC_AP_HASH_CHUNK : (int)remaining); if(FileReadArray(handle, bytes, 0, count) != (uint)count) { reason = "dpccap.hash.read"; return(false); } if(!hash.Update(bytes, 0, count, reason)) return(false); remaining -= (ulong)count; } return(hash.Final(digest, hex, reason)); } //+------------------------------------------------------------------+ //| Validate identity, dense layout and digests before native Load | //+------------------------------------------------------------------+ bool DPCCAPValidateHandle(const int handle, const uint role, const string source_abi, const ulong max_bytes, const int max_sections, SDPCCAPSection §ions[], string &reason) { ArrayResize(sections, 0); reason = ""; if(handle == INVALID_HANDLE || (role != 1 && role != 2) || !DPCCResolverLowerSha256(source_abi) || max_bytes < DPCC_AP_HEADER_BYTES + DPCC_AP_FOOTER_BYTES || max_bytes > (ulong)LONG_MAX || max_sections < 1) { reason = "dpccap.arguments"; return(false); } //--- Check the whole-file bound and header before allocating the section table. const ulong total = FileSize(handle); if(total > max_bytes || total < DPCC_AP_HEADER_BYTES + DPCC_AP_FOOTER_BYTES) { reason = "dpccap.file.size"; return(false); } uchar header[]; if(!DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason)) return(false); if(!DPCCAPMagic(header, 0, "DPCCAP01")) { reason = "dpccap.header.magic"; return(false); } if(DPCCAPReadUnsigned(header, 8, 4) != 1) { reason = "dpccap.header.version"; return(false); } if(DPCCAPReadUnsigned(header, 12, 4) != role) { reason = "dpccap.header.role"; return(false); } if(DPCCAPReadUnsigned(header, 20, 4) != 1) { reason = "dpccap.header.incomplete"; return(false); } if(DPCCAPReadUnsigned(header, 24, 8) != total) { reason = "dpccap.header.total_bytes"; return(false); } if(DPCCAPReadUnsigned(header, 32, 8) != DPCC_AP_HEADER_BYTES) { reason = "dpccap.header.table_offset"; return(false); } if(DPCCAPReadSHA(header, 40) != source_abi) { reason = "dpccap.header.source_abi"; return(false); } string contract = ""; if(!DPCCAPContractSHA(contract, reason)) return(false); if(DPCCAPReadSHA(header, 72) != contract) { reason = "dpccap.header.contract"; return(false); } for(int i = 104; i < 128; i++) if(header[i] != 0) { reason = "dpccap.header.reserved"; return(false); } const ulong count = DPCCAPReadUnsigned(header, 16, 4); if(count < 1 || count > (ulong)max_sections || count > (total - DPCC_AP_HEADER_BYTES - DPCC_AP_FOOTER_BYTES) / DPCC_AP_SECTION_BYTES) { reason = "dpccap.header.section_count"; return(false); } SDPCCAPSection candidate[]; if(ArrayResize(candidate, (int)count) != (int)count) { reason = "dpccap.table.capacity"; return(false); } //--- Check all section extents before reading any payload. const ulong footer_offset = total - DPCC_AP_FOOTER_BYTES; ulong next = DPCC_AP_HEADER_BYTES + count * DPCC_AP_SECTION_BYTES; uchar record[]; for(int i = 0; i < (int)count; i++) { if(!DPCCAPReadRecord(handle, DPCC_AP_HEADER_BYTES + (ulong)i * DPCC_AP_SECTION_BYTES, DPCC_AP_SECTION_BYTES, record, reason)) return(false); candidate[i].id = (uint)DPCCAPReadUnsigned(record, 0, 4); candidate[i].encoding = (uint)DPCCAPReadUnsigned(record, 4, 4); candidate[i].offset = DPCCAPReadUnsigned(record, 8, 8); candidate[i].length = DPCCAPReadUnsigned(record, 16, 8); candidate[i].sha256 = DPCCAPReadSHA(record, 24); if(i > 0 && candidate[i].id <= candidate[i - 1].id) { reason = "dpccap.table.id"; return(false); } if(candidate[i].encoding < 1 || candidate[i].encoding > 4) { reason = "dpccap.table.encoding"; return(false); } if(candidate[i].offset != next) { reason = "dpccap.table.offset"; return(false); } if(next > footer_offset || candidate[i].length > footer_offset - next) { reason = "dpccap.table.length"; return(false); } for(int j = 56; j < 64; j++) if(record[j] != 0) { reason = "dpccap.table.reserved"; return(false); } next += candidate[i].length; } if(next != footer_offset) { reason = "dpccap.table.dense_end"; return(false); } uchar footer[]; if(!DPCCAPReadRecord(handle, footer_offset, DPCC_AP_FOOTER_BYTES, footer, reason)) return(false); if(!DPCCAPMagic(footer, 0, "APEND001")) { reason = "dpccap.footer.magic"; return(false); } if(DPCCAPReadUnsigned(footer, 8, 8) != total) { reason = "dpccap.footer.total_bytes"; return(false); } //--- Authenticate sections and the complete header/table/footer prefix. string observed = ""; for(int i = 0; i < (int)count; i++) { if(!DPCCAPHashRange(handle, candidate[i].offset, candidate[i].length, observed, reason)) return(false); if(observed != candidate[i].sha256) { reason = "dpccap.section.sha256"; return(false); } } if(!DPCCAPHashRange(handle, 0, total - 32, observed, reason)) return(false); if(observed != DPCCAPReadSHA(footer, 16)) { reason = "dpccap.footer.sha256"; return(false); } if(ArrayResize(sections, (int)count) != (int)count) { ArrayResize(sections, 0); reason = "dpccap.table.copy"; return(false); } for(int i = 0; i < (int)count; i++) sections[i] = candidate[i]; return(true); } //+------------------------------------------------------------------+ //| Keep a validated role file open while its sections are consumed | //+------------------------------------------------------------------+ class CDPCCAPRoleReader { private: int m_handle; int m_role_lock; int m_active; SDPCCAPSection m_sections[]; uint m_role; string m_source_abi; public: CDPCCAPRoleReader(void) : m_handle(INVALID_HANDLE), m_role_lock(INVALID_HANDLE), m_active(-1), m_role(0), m_source_abi("") {} ~CDPCCAPRoleReader(void) { Close(); } void Close(void) { if(m_handle != INVALID_HANDLE) FileClose(m_handle); if(m_role_lock != INVALID_HANDLE) FileClose(m_role_lock); m_handle = INVALID_HANDLE; m_role_lock = INVALID_HANDLE; m_active = -1; m_role = 0; m_source_abi = ""; ArrayResize(m_sections, 0); } bool Open(const string path, const bool common, const uint role, const string source_abi, const ulong max_bytes, const int max_sections, string &reason) { Close(); string canonical = ""; if(!DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path) { reason = "dpccap.path.canonical"; return(false); } int flags = FILE_READ | FILE_BIN; if(common) flags |= FILE_COMMON; //--- The same role lock covers current and bak throughout extraction. int last_separator = -1; for(int i = 0; i < StringLen(path); i++) if(StringGetCharacter(path, i) == '/') last_separator = i; const string lock_path = StringSubstr(path, 0, last_separator + 1) + (role == 1 ? "Market.writer.lock" : "Trading.writer.lock"); m_role_lock = FileOpen(lock_path, FILE_READ | FILE_WRITE | FILE_BIN | (common ? FILE_COMMON : 0)); if(m_role_lock == INVALID_HANDLE) { reason = "dpccap.reader.writer_active"; return(false); } m_handle = FileOpen(path, flags); if(m_handle == INVALID_HANDLE) { reason = StringFormat("dpccap.file.open.%d", GetLastError()); Close(); return(false); } if(!DPCCAPValidateHandle(m_handle, role, source_abi, max_bytes, max_sections, m_sections, reason)) { Close(); return(false); } if(role == 2) { const uint expected_ids[] = {1, 110, 111, 112, 113, 114, 115, 200}; if(ArraySize(m_sections) != ArraySize(expected_ids)) { reason = "dpccap.trading.section_count"; Close(); return(false); } for(int i = 0; i < ArraySize(expected_ids); i++) if(m_sections[i].id != expected_ids[i] || m_sections[i].length == 0 || m_sections[i].encoding != (i == 0 || i == 7 ? 1 : 2)) { reason = "dpccap.trading.section_inventory"; Close(); return(false); } } m_role = role; m_source_abi = source_abi; return(true); } //--- Publishes a fresh owned network only after bounded extraction and graph checks. bool LoadNative(const SDPCCAPNativeLoadContract &expected, CArrayObj &expected_graph, const uint &expected_abis[], COpenCLMy *shared_context, CNet *&model, string &reason); bool LoadNativeInto(const SDPCCAPNativeLoadContract &expected, CNet &destination, COpenCLMy *shared_context, string &reason); bool LoadReadyMarketInto(const SDPCCAPNativeLoadContract &expected, CNet &destination, COpenCLMy *shared_context, string &reason, const bool allow_pending_training = false); bool ReadReadyMarketContract(const string market_profile_sha256, const string task_id, const ulong max_native_bytes, SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training = false); bool ReadJSONSection(const uint section_id, const ulong maximum_bytes, string &text, string &reason); bool VerifySectionIntegrity(const uint section_id, const uint encoding, string &reason); bool LoadNativeCore(const SDPCCAPNativeLoadContract &expected, CArrayObj &expected_graph, const uint &expected_abis[], COpenCLMy *shared_context, CNet *destination, CNet *&model, const bool require_ready_market, string &reason); bool ArtifactSHA256(string &sha, string &reason) { sha = ""; if(m_handle == INVALID_HANDLE || m_active != -1) { reason = "dpccap.reader.artifact_hash_state"; return(false); } return(DPCCAPHashRange(m_handle, 0, FileSize(m_handle), sha, reason)); } int SectionCount(void) { return(ArraySize(m_sections)); } bool BeginSection(const uint id, const uint encoding, int &handle, string &reason) { handle = INVALID_HANDLE; reason = ""; if(m_handle == INVALID_HANDLE || m_active != -1) { reason = "dpccap.reader.state"; return(false); } for(int i = 0; i < ArraySize(m_sections); i++) if(m_sections[i].id == id && m_sections[i].encoding == encoding) { if(!FileSeek(m_handle, (long)m_sections[i].offset, SEEK_SET)) { reason = "dpccap.section.seek"; Close(); return(false); } m_active = i; handle = m_handle; return(true); } reason = "dpccap.section.missing"; return(false); } bool EndSection(string &reason) { reason = ""; if(m_handle == INVALID_HANDLE || m_active < 0) { reason = "dpccap.reader.state"; return(false); } if(FileTell(m_handle) != m_sections[m_active].offset + m_sections[m_active].length) { reason = "dpccap.section.cursor"; Close(); return(false); } m_active = -1; return(true); } }; //+------------------------------------------------------------------+ //| Stream native role payloads without retaining a second model | //| copy | //+------------------------------------------------------------------+ class CDPCCAPRoleWriter { private: int m_handle; int m_count; int m_written; bool m_active; bool m_failed; ulong m_next; ulong m_max_bytes; uint m_last_id; uint m_role; string m_path; string m_source_abi; bool m_common; uchar m_header[]; SDPCCAPSection m_section; bool Fail(const string failure, string &reason) { reason = failure; m_failed = true; return(false); } public: CDPCCAPRoleWriter(void) : m_handle(INVALID_HANDLE), m_count(0), m_written(0), m_active(false), m_failed(false), m_next(0), m_max_bytes(0), m_last_id(0), m_role(0), m_common(false) {} ~CDPCCAPRoleWriter(void) { Close(); } void Close(void) { if(m_handle != INVALID_HANDLE) FileClose(m_handle); m_handle = INVALID_HANDLE; m_active = false; m_failed = true; } bool Open(const string path, const bool common, const uint role, const string source_abi, const int count, const ulong max_bytes, string &reason) { Close(); reason = ""; string canonical = ""; if((role != 1 && role != 2) || !DPCCResolverLowerSha256(source_abi) || count < 1 || max_bytes > (ulong)LONG_MAX || max_bytes < DPCC_AP_HEADER_BYTES + DPCC_AP_FOOTER_BYTES || (ulong)count > (max_bytes - DPCC_AP_HEADER_BYTES - DPCC_AP_FOOTER_BYTES) / DPCC_AP_SECTION_BYTES) return(Fail("dpccap.arguments", reason)); if(!DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path) return(Fail("dpccap.path.canonical", reason)); const int common_flag = (common ? FILE_COMMON : 0); if(FileIsExist(path, common_flag)) return(Fail("dpccap.writer.destination_exists", reason)); //--- Reserve only fixed records; payloads are written directly by their owners. if(ArrayResize(m_header, DPCC_AP_HEADER_BYTES) != DPCC_AP_HEADER_BYTES) return(Fail("dpccap.header.capacity", reason)); ArrayInitialize(m_header, 0); DPCCAPWriteMagic(m_header, 0, "DPCCAP01"); DPCCAPWriteUnsigned(m_header, 8, 4, 1); DPCCAPWriteUnsigned(m_header, 12, 4, role); DPCCAPWriteUnsigned(m_header, 16, 4, (ulong)count); DPCCAPWriteUnsigned(m_header, 32, 8, DPCC_AP_HEADER_BYTES); DPCCAPWriteSHA(m_header, 40, source_abi); string contract = ""; if(!DPCCAPContractSHA(contract, reason)) return(Fail(reason, reason)); DPCCAPWriteSHA(m_header, 72, contract); m_handle = FileOpen(path, FILE_READ | FILE_WRITE | FILE_BIN | common_flag); if(m_handle == INVALID_HANDLE) return(Fail(StringFormat("dpccap.file.open.%d", GetLastError()), reason)); if(!DPCCAPWriteRecord(m_handle, 0, m_header, reason)) return(Fail(reason, reason)); uchar record[]; if(ArrayResize(record, DPCC_AP_SECTION_BYTES) != DPCC_AP_SECTION_BYTES) return(Fail("dpccap.table.capacity", reason)); ArrayInitialize(record, 0); for(int i = 0; i < count; i++) if(!DPCCAPWriteRecord(m_handle, DPCC_AP_HEADER_BYTES + (ulong)i * DPCC_AP_SECTION_BYTES, record, reason)) return(Fail(reason, reason)); m_count = count; m_written = 0; m_active = false; m_failed = false; m_next = DPCC_AP_HEADER_BYTES + (ulong)count * DPCC_AP_SECTION_BYTES; m_max_bytes = max_bytes; m_last_id = 0; m_role = role; m_path = path; m_source_abi = source_abi; m_common = common; return(true); } bool BeginSection(const uint id, const uint encoding, int &handle, string &reason) { handle = INVALID_HANDLE; reason = ""; if(m_handle == INVALID_HANDLE || m_failed || m_active || m_written >= m_count) return(Fail("dpccap.writer.state", reason)); if(m_written > 0 && id <= m_last_id) return(Fail("dpccap.table.id", reason)); if(encoding < 1 || encoding > 4) return(Fail("dpccap.table.encoding", reason)); if(!FileSeek(m_handle, (long)m_next, SEEK_SET)) return(Fail("dpccap.section.seek", reason)); m_section.id = id; m_section.encoding = encoding; m_section.offset = m_next; m_section.length = 0; m_section.sha256 = ""; m_active = true; handle = m_handle; return(true); } bool EndSection(string &reason) { reason = ""; if(m_handle == INVALID_HANDLE || m_failed || !m_active) return(Fail("dpccap.writer.state", reason)); const ulong end = FileTell(m_handle); if(end < m_section.offset || end != FileSize(m_handle) || end > m_max_bytes - DPCC_AP_FOOTER_BYTES) return(Fail("dpccap.section.write_extent", reason)); m_section.length = end - m_section.offset; if(!DPCCAPHashRange(m_handle, m_section.offset, m_section.length, m_section.sha256, reason)) return(Fail(reason, reason)); uchar record[]; if(ArrayResize(record, DPCC_AP_SECTION_BYTES) != DPCC_AP_SECTION_BYTES) return(Fail("dpccap.table.capacity", reason)); ArrayInitialize(record, 0); DPCCAPWriteUnsigned(record, 0, 4, m_section.id); DPCCAPWriteUnsigned(record, 4, 4, m_section.encoding); DPCCAPWriteUnsigned(record, 8, 8, m_section.offset); DPCCAPWriteUnsigned(record, 16, 8, m_section.length); DPCCAPWriteSHA(record, 24, m_section.sha256); if(!DPCCAPWriteRecord(m_handle, DPCC_AP_HEADER_BYTES + (ulong)m_written * DPCC_AP_SECTION_BYTES, record, reason)) return(Fail(reason, reason)); m_next = end; m_last_id = m_section.id; m_written++; m_active = false; return(true); } //+------------------------------------------------------------------+ //| Stream one Market or frozen SkillTarget native CNet section | //+------------------------------------------------------------------+ bool WriteMarketNativeSection(const uint section_id, CNet &model, string &reason) { reason = ""; if(m_role != 1 || (section_id != 100 && section_id != 101)) return(Fail("dpccap.writer.market_native_section", reason)); int handle = INVALID_HANDLE; if(!BeginSection(section_id, 2, handle, reason)) return(false); if(!model.Save(handle)) { Close(); return(Fail("dpccap.writer.market_native_save", reason)); } if(!EndSection(reason)) return(false); return(true); } //+------------------------------------------------------------------+ //| Stream one native Action6 Trading role without a model copy | //+------------------------------------------------------------------+ bool WriteTradingNativeSection(const uint section_id, CNet &model, string &reason) { reason = ""; if(m_role != 2 || section_id < 110 || section_id > 115) return(Fail("dpccap.writer.trading_native_section", reason)); int handle = INVALID_HANDLE; if(!BeginSection(section_id, 2, handle, reason)) return(false); if(!model.Save(handle)) { Close(); return(Fail("dpccap.writer.trading_native_save", reason)); } if(!EndSection(reason)) return(false); return(true); } bool Finish(string &reason) { reason = ""; if(m_handle == INVALID_HANDLE || m_failed || m_active || m_written != m_count) return(Fail("dpccap.writer.state", reason)); //--- Commit a completed header and authenticate the footer prefix before publishing. const ulong total = m_next + DPCC_AP_FOOTER_BYTES; uchar footer[]; if(ArrayResize(footer, DPCC_AP_FOOTER_BYTES) != DPCC_AP_FOOTER_BYTES) return(Fail("dpccap.footer.capacity", reason)); ArrayInitialize(footer, 0); DPCCAPWriteMagic(footer, 0, "APEND001"); DPCCAPWriteUnsigned(footer, 8, 8, total); if(!DPCCAPWriteRecord(m_handle, m_next, footer, reason)) return(Fail(reason, reason)); DPCCAPWriteUnsigned(m_header, 20, 4, 1); DPCCAPWriteUnsigned(m_header, 24, 8, total); if(!DPCCAPWriteRecord(m_handle, 0, m_header, reason)) return(Fail(reason, reason)); string digest = ""; if(!DPCCAPHashRange(m_handle, 0, total - 32, digest, reason)) return(Fail(reason, reason)); DPCCAPWriteSHA(footer, 16, digest); if(!DPCCAPWriteRecord(m_handle, m_next, footer, reason)) return(Fail(reason, reason)); FileFlush(m_handle); FileClose(m_handle); m_handle = INVALID_HANDLE; //--- Reopen the closed file and fully verify it before reporting a usable frame. //--- Caller may own the exclusive role lock, so do not recursively acquire it here. const int verify = FileOpen(m_path, FILE_READ | FILE_BIN | (m_common ? FILE_COMMON : 0)); if(verify == INVALID_HANDLE) return(Fail("dpccap.writer.readback_open", reason)); SDPCCAPSection sections[]; const bool valid = DPCCAPValidateHandle(verify, m_role, m_source_abi, m_max_bytes, m_count, sections, reason); FileClose(verify); return(valid || Fail(reason, reason)); } }; //+------------------------------------------------------------------+ //| StrictUtf8.mqh: strict bounded UTF-8 source adapter | //+------------------------------------------------------------------+ #ifndef DPCC_STRICT_UTF8_MQH #define DPCC_STRICT_UTF8_MQH //--- Reason codes: //--- range.count_negative, range.count_exceeds_array, limit.invalid, //--- limit.exceeded, limit.capacity, bytes.nul, bytes.bom, utf8.truncated, //--- utf8.lead, utf8.continuation, utf8.overlong, utf8.surrogate, //--- utf8.out_of_range, roundtrip.encode, roundtrip.length, //--- roundtrip.mismatch. //--- No normalization is performed; the selected byte range is decoded as-is. //+------------------------------------------------------------------+ //| DPCCStrictUtf8ValidateBytes rejects malformed bounded UTF-8 | //+------------------------------------------------------------------+ bool DPCCStrictUtf8ValidateBytes(const uchar &bytes[], const int exact_count, string &reason) { reason = ""; if(exact_count < 0) { reason = "range.count_negative"; return(false); } if(exact_count > ArraySize(bytes)) { reason = "range.count_exceeds_array"; return(false); } for(int i = 0; i < exact_count; i++) { if(bytes[i] == 0) { reason = "bytes.nul"; return(false); } } if(exact_count >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) { reason = "bytes.bom"; return(false); } int i = 0; while(i < exact_count) { const uchar lead = bytes[i]; if(lead <= 0x7F) { i++; continue; } int width = 0; if(lead >= 0xC0 && lead <= 0xDF) width = 2; else if(lead >= 0xE0 && lead <= 0xEF) width = 3; else if(lead >= 0xF0 && lead <= 0xF7) width = 4; else { reason = "utf8.lead"; return(false); } if(lead == 0xC0 || lead == 0xC1) { reason = "utf8.overlong"; return(false); } if(lead >= 0xF5) { reason = "utf8.out_of_range"; return(false); } if(width > exact_count - i) { reason = "utf8.truncated"; return(false); } for(int offset = 1; offset < width; offset++) { if(bytes[i + offset] < 0x80 || bytes[i + offset] > 0xBF) { reason = "utf8.continuation"; return(false); } } const uchar second = bytes[i + 1]; if(lead == 0xE0 && second < 0xA0) { reason = "utf8.overlong"; return(false); } if(lead == 0xED && second >= 0xA0) { reason = "utf8.surrogate"; return(false); } if(lead == 0xF0 && second < 0x90) { reason = "utf8.overlong"; return(false); } if(lead == 0xF4 && second > 0x8F) { reason = "utf8.out_of_range"; return(false); } i += width; } return(true); } //+------------------------------------------------------------------+ //| DPCCStrictUtf8Decode decodes and verifies an exact UTF-8 range | //+------------------------------------------------------------------+ bool DPCCStrictUtf8Decode(const uchar &bytes[], const int exact_count, const long max_bytes, string &decoded, string &reason) { decoded = ""; reason = ""; if(exact_count < 0) { reason = "range.count_negative"; return(false); } if(exact_count > ArraySize(bytes)) { reason = "range.count_exceeds_array"; return(false); } if(max_bytes < 0) { reason = "limit.invalid"; return(false); } if((long)exact_count > max_bytes) { reason = "limit.exceeded"; return(false); } if(exact_count >= INT_MAX) { reason = "limit.capacity"; return(false); } if(!DPCCStrictUtf8ValidateBytes(bytes, exact_count, reason)) return(false); if(exact_count == 0) { //--- MQL5 reports an empty StringToCharArray result as zero, indistinguishable //--- from failure; the empty string encodes to the exact empty input range. decoded = ""; return(true); } string candidate = CharArrayToString(bytes, 0, exact_count, CP_UTF8); uchar encoded[]; ResetLastError(); const int encoded_count = StringToCharArray(candidate, encoded, 0, WHOLE_ARRAY, CP_UTF8); if(encoded_count <= 0) { reason = StringFormat("roundtrip.encode.%d", GetLastError()); return(false); } if(encoded_count != exact_count + 1 || ArraySize(encoded) < encoded_count || encoded[encoded_count - 1] != 0) { reason = "roundtrip.length"; return(false); } for(int i = 0; i < exact_count; i++) { if(encoded[i] != bytes[i]) { reason = "roundtrip.mismatch"; return(false); } } decoded = candidate; return(true); } #endif #ifndef DPCC_JSON_MQH #define DPCC_JSON_MQH #define DPCC_JSON_MAX_DEPTH 128 #define DPCC_JSON_MAX_NODES 262144 #define DPCC_JSON_KEY_SLOTS 524287 #define DPCC_JSON_MAX_SOURCE_CHARS 4194304 enum ENUM_DPCC_JSON_TYPE { DPCC_JSON_INVALID = 0, DPCC_JSON_OBJECT = 1, DPCC_JSON_ARRAY = 2, DPCC_JSON_STRING = 3, DPCC_JSON_NUMBER = 4, DPCC_JSON_BOOLEAN = 5, DPCC_JSON_NULL = 6 }; struct SDPCCJsonNode { ENUM_DPCC_JSON_TYPE type; string text; string key; int first_child; int last_child; int next_sibling; int child_count; int parent; }; //+------------------------------------------------------------------+ //| Declaration CDPCCJsonDocument | //+------------------------------------------------------------------+ class CDPCCJsonDocument { private: string m_source; int m_length; int m_pos; int m_count; int m_capacity; int m_root; string m_error; SDPCCJsonNode m_nodes[]; int m_key_slots[]; int m_key_capacity; bool IsWhitespace(const ushort ch) { return(ch == 32 || ch == 9 || ch == 10 || ch == 13); } bool IsDigit(const ushort ch) { return(ch >= 48 && ch <= 57); } bool IsNonzeroDigit(const ushort ch) { return(ch >= 49 && ch <= 57); } void SkipWhitespace(void) { while(m_pos < m_length && IsWhitespace(StringGetCharacter(m_source, m_pos))) m_pos++; } bool Fail(const string reason) { if(m_error == "") m_error = reason; return(false); } void ClearDocument(void) { ArrayResize(m_nodes, 0); ArrayResize(m_key_slots, 0); m_count = 0; m_capacity = 0; m_key_capacity = 0; m_root = -1; } long KeyHash(const int parent, const string key) { long hash = (parent + 1) % 2147483629; int length = StringLen(key); for(int i = 0; i < length; i++) hash = (hash * 131 + (long)StringGetCharacter(key, i)) % 2147483629; return(hash); } bool EnsureKeyIndex(void) { if(m_key_capacity > 0) return(true); if(ArrayResize(m_key_slots, DPCC_JSON_KEY_SLOTS) < DPCC_JSON_KEY_SLOTS) return(Fail("KEY_INDEX_STORAGE")); ArrayInitialize(m_key_slots, -1); m_key_capacity = DPCC_JSON_KEY_SLOTS; return(true); } int FindObjectKey(const int parent, const string key) { if(m_key_capacity == 0) return(-1); int slot = (int)(KeyHash(parent, key) % m_key_capacity); for(int probe = 0; probe < m_key_capacity; probe++) { int node = m_key_slots[slot]; if(node < 0) return(-1); if(m_nodes[node].parent == parent && m_nodes[node].key == key) return(node); slot++; if(slot >= m_key_capacity) slot = 0; } return(-1); } bool RegisterObjectKey(const int parent, const int child) { if(!EnsureKeyIndex()) return(false); string key = m_nodes[child].key; int slot = (int)(KeyHash(parent, key) % m_key_capacity); for(int probe = 0; probe < m_key_capacity; probe++) { int existing = m_key_slots[slot]; if(existing < 0) { m_key_slots[slot] = child; return(true); } if(m_nodes[existing].parent == parent && m_nodes[existing].key == key) return(Fail("DUPLICATE_KEY")); slot++; if(slot >= m_key_capacity) slot = 0; } return(Fail("KEY_INDEX_FULL")); } int AddNode(const ENUM_DPCC_JSON_TYPE type, const string text, const string key, const int parent) { if(m_count >= DPCC_JSON_MAX_NODES) { Fail("NODE_LIMIT"); return(-1); } if(m_count >= m_capacity) { int next_capacity = (m_capacity == 0 ? 128 : m_capacity * 2); if(next_capacity > DPCC_JSON_MAX_NODES) next_capacity = DPCC_JSON_MAX_NODES; if(ArrayResize(m_nodes, next_capacity) < next_capacity) { Fail("NODE_STORAGE"); return(-1); } m_capacity = next_capacity; } int index = m_count++; m_nodes[index].type = type; m_nodes[index].text = text; m_nodes[index].key = key; m_nodes[index].first_child = -1; m_nodes[index].last_child = -1; m_nodes[index].next_sibling = -1; m_nodes[index].child_count = 0; m_nodes[index].parent = parent; if(parent < 0) m_root = index; else { if(m_nodes[parent].first_child < 0) m_nodes[parent].first_child = index; else m_nodes[m_nodes[parent].last_child].next_sibling = index; m_nodes[parent].last_child = index; m_nodes[parent].child_count++; } return(index); } bool ReadHex4(ushort &value) { if(m_pos + 4 > m_length) return(Fail("INVALID_UNICODE")); int result = 0; for(int i = 0; i < 4; i++) { ushort ch = StringGetCharacter(m_source, m_pos++); int digit = -1; if(ch >= 48 && ch <= 57) digit = (int)(ch - 48); else if(ch >= 65 && ch <= 70) digit = (int)(ch - 65 + 10); else if(ch >= 97 && ch <= 102) digit = (int)(ch - 97 + 10); if(digit < 0) return(Fail("INVALID_UNICODE")); result = (result << 4) | digit; } value = (ushort)result; return(true); } bool ParseString(string &value) { value = ""; if(m_pos >= m_length || StringGetCharacter(m_source, m_pos) != 34) return(Fail("EXPECTED_STRING")); m_pos++; while(m_pos < m_length) { ushort ch = StringGetCharacter(m_source, m_pos++); if(ch == 34) return(true); if(ch < 32) return(Fail("STRING_CONTROL")); if(ch == 92) { if(m_pos >= m_length) return(Fail("INVALID_ESCAPE")); ushort escaped = StringGetCharacter(m_source, m_pos++); if(escaped == 34 || escaped == 92 || escaped == 47) value += ShortToString(escaped); else if(escaped == 98) value += ShortToString(8); else if(escaped == 102) value += ShortToString(12); else if(escaped == 110) value += ShortToString(10); else if(escaped == 114) value += ShortToString(13); else if(escaped == 116) value += ShortToString(9); else if(escaped == 117) { ushort first = 0; if(!ReadHex4(first)) return(false); if(first >= 0xD800 && first <= 0xDBFF) { if(m_pos + 2 > m_length || StringGetCharacter(m_source, m_pos) != 92 || StringGetCharacter(m_source, m_pos + 1) != 117) return(Fail("INVALID_SURROGATE")); m_pos += 2; ushort second = 0; if(!ReadHex4(second)) return(false); if(second < 0xDC00 || second > 0xDFFF) return(Fail("INVALID_SURROGATE")); value += ShortToString(first); value += ShortToString(second); } else if(first >= 0xDC00 && first <= 0xDFFF) return(Fail("INVALID_SURROGATE")); else value += ShortToString(first); } else return(Fail("INVALID_ESCAPE")); continue; } if(ch >= 0xD800 && ch <= 0xDBFF) { if(m_pos >= m_length) return(Fail("INVALID_SURROGATE")); ushort second = StringGetCharacter(m_source, m_pos); if(second < 0xDC00 || second > 0xDFFF) return(Fail("INVALID_SURROGATE")); value += ShortToString(ch); value += ShortToString(second); m_pos++; } else if(ch >= 0xDC00 && ch <= 0xDFFF) return(Fail("INVALID_SURROGATE")); else value += ShortToString(ch); } return(Fail("UNTERMINATED_STRING")); } bool IsValueDelimiter(const int position) { if(position >= m_length) return(true); ushort ch = StringGetCharacter(m_source, position); return(IsWhitespace(ch) || ch == 44 || ch == 93 || ch == 125); } bool ParseNumberLexeme(string &lexeme) { int start = m_pos; if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 45) m_pos++; if(m_pos >= m_length) return(Fail("INVALID_NUMBER")); ushort ch = StringGetCharacter(m_source, m_pos); if(ch == 48) { m_pos++; if(m_pos < m_length && IsDigit(StringGetCharacter(m_source, m_pos))) return(Fail("INVALID_NUMBER")); } else if(IsNonzeroDigit(ch)) { do { m_pos++; } while(m_pos < m_length && IsDigit(StringGetCharacter(m_source, m_pos))); } else return(Fail("INVALID_NUMBER")); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 46) { m_pos++; if(m_pos >= m_length || !IsDigit(StringGetCharacter(m_source, m_pos))) return(Fail("INVALID_NUMBER")); while(m_pos < m_length && IsDigit(StringGetCharacter(m_source, m_pos))) m_pos++; } if(m_pos < m_length && (StringGetCharacter(m_source, m_pos) == 69 || StringGetCharacter(m_source, m_pos) == 101)) { m_pos++; if(m_pos < m_length && (StringGetCharacter(m_source, m_pos) == 43 || StringGetCharacter(m_source, m_pos) == 45)) m_pos++; if(m_pos >= m_length || !IsDigit(StringGetCharacter(m_source, m_pos))) return(Fail("INVALID_NUMBER")); while(m_pos < m_length && IsDigit(StringGetCharacter(m_source, m_pos))) m_pos++; } if(!IsValueDelimiter(m_pos)) return(Fail("INVALID_NUMBER")); lexeme = StringSubstr(m_source, start, m_pos - start); return(true); } bool ParseObject(const int node, const int depth) { m_pos++; SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 125) { m_pos++; return(true); } while(m_pos < m_length) { string name = ""; if(!ParseString(name)) return(false); if(FindObjectKey(node, name) >= 0) return(Fail("DUPLICATE_KEY")); SkipWhitespace(); if(m_pos >= m_length || StringGetCharacter(m_source, m_pos) != 58) return(Fail("EXPECTED_COLON")); m_pos++; if(!ParseValue(node, name, depth + 1)) return(false); if(!RegisterObjectKey(node, m_nodes[node].last_child)) return(false); SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 125) { m_pos++; return(true); } if(m_pos >= m_length || StringGetCharacter(m_source, m_pos) != 44) return(Fail("EXPECTED_COMMA_OR_END")); m_pos++; SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 125) return(Fail("EXPECTED_OBJECT_KEY")); } return(Fail("UNTERMINATED_OBJECT")); } bool ParseArray(const int node, const int depth) { m_pos++; SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 93) { m_pos++; return(true); } while(m_pos < m_length) { if(!ParseValue(node, "", depth + 1)) return(false); SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 93) { m_pos++; return(true); } if(m_pos >= m_length || StringGetCharacter(m_source, m_pos) != 44) return(Fail("EXPECTED_COMMA_OR_END")); m_pos++; SkipWhitespace(); if(m_pos < m_length && StringGetCharacter(m_source, m_pos) == 93) return(Fail("EXPECTED_VALUE")); } return(Fail("UNTERMINATED_ARRAY")); } bool ParseValue(const int parent, const string key, const int depth) { SkipWhitespace(); if(m_pos >= m_length) return(Fail("EXPECTED_VALUE")); ushort ch = StringGetCharacter(m_source, m_pos); if(ch == 123 || ch == 91) { if(depth >= DPCC_JSON_MAX_DEPTH) return(Fail("DEPTH_LIMIT")); ENUM_DPCC_JSON_TYPE type = (ch == 123 ? DPCC_JSON_OBJECT : DPCC_JSON_ARRAY); int node = AddNode(type, "", key, parent); if(node < 0) return(false); if(type == DPCC_JSON_OBJECT) return(ParseObject(node, depth)); return(ParseArray(node, depth)); } if(ch == 34) { string value = ""; if(!ParseString(value)) return(false); return(AddNode(DPCC_JSON_STRING, value, key, parent) >= 0); } if(ch == 116 && StringSubstr(m_source, m_pos, 4) == "true" && IsValueDelimiter(m_pos + 4)) { m_pos += 4; return(AddNode(DPCC_JSON_BOOLEAN, "true", key, parent) >= 0); } if(ch == 102 && StringSubstr(m_source, m_pos, 5) == "false" && IsValueDelimiter(m_pos + 5)) { m_pos += 5; return(AddNode(DPCC_JSON_BOOLEAN, "false", key, parent) >= 0); } if(ch == 110 && StringSubstr(m_source, m_pos, 4) == "null" && IsValueDelimiter(m_pos + 4)) { m_pos += 4; return(AddNode(DPCC_JSON_NULL, "", key, parent) >= 0); } if(ch == 45 || IsDigit(ch)) { string number = ""; if(!ParseNumberLexeme(number)) return(false); return(AddNode(DPCC_JSON_NUMBER, number, key, parent) >= 0); } return(Fail("EXPECTED_VALUE")); } public: CDPCCJsonDocument(void) { m_source = ""; m_length = 0; m_pos = 0; m_count = 0; m_capacity = 0; m_root = -1; m_error = ""; } bool Parse(const string source, string &reason) { ClearDocument(); m_error = ""; m_source = ""; m_length = 0; m_pos = 0; m_key_capacity = 0; const int source_length = StringLen(source); if(source_length > DPCC_JSON_MAX_SOURCE_CHARS) { reason = "SOURCE_LIMIT"; return(false); } m_source = source; m_length = source_length; SkipWhitespace(); if(m_pos >= m_length) Fail("EMPTY_INPUT"); else ParseValue(-1, "", 0); if(m_error == "") { SkipWhitespace(); if(m_pos != m_length) Fail("TRAILING_DATA"); } if(m_error != "") { reason = m_error; ClearDocument(); return(false); } reason = ""; return(true); } int Root(void) { return(m_root); } int NodeCount(void) { return(m_count); } ENUM_DPCC_JSON_TYPE Type(const int index) { if(index < 0 || index >= m_count) return(DPCC_JSON_INVALID); return(m_nodes[index].type); } string Text(const int index) { if(index < 0 || index >= m_count) return(""); return(m_nodes[index].text); } string Key(const int index) { if(index < 0 || index >= m_count) return(""); return(m_nodes[index].key); } int ChildCount(const int index) { if(index < 0 || index >= m_count) return(0); return(m_nodes[index].child_count); } int ChildAt(const int index, const int child_index) { if(index < 0 || index >= m_count || child_index < 0 || child_index >= m_nodes[index].child_count) return(-1); int child = m_nodes[index].first_child; for(int i = 0; i < child_index; i++) child = m_nodes[child].next_sibling; return(child); } int FirstChild(const int index) { if(index < 0 || index >= m_count) return(-1); return(m_nodes[index].first_child); } int NextSibling(const int index) { if(index < 0 || index >= m_count) return(-1); return(m_nodes[index].next_sibling); } int FindMember(const int index, const string name) { if(index < 0 || index >= m_count || m_nodes[index].type != DPCC_JSON_OBJECT) return(-1); for(int child = FirstChild(index); child >= 0; child = NextSibling(child)) if(m_nodes[child].key == name) return(child); return(-1); } bool TryNumber(const int index, double &value) { if(Type(index) != DPCC_JSON_NUMBER) return(false); double converted = StringToDouble(m_nodes[index].text); if(!MathIsValidNumber(converted)) return(false); value = converted; return(true); } }; #endif enum ENUM_DPCC_ENTRY_STAGE { DPCC_STAGE_STUDYFORECAST = 0, DPCC_STAGE_STUDY = 1, DPCC_STAGE_STUDYONLINE = 2, DPCC_STAGE_TEST = 3, DPCC_STAGE_UNASSIGNED = 4 }; enum ENUM_DPCC_RESUME_MODE { DPCC_RESUME_EXACT = 0, DPCC_RESUME_NEW_EPISODE_FROM_WEIGHTS = 1, DPCC_RESUME_NEW = 2, DPCC_RESUME_UNASSIGNED = 3 }; struct SDPCCExecutionControl { string package_root; //--- Embedded DPC metadata; no external settings file. string profile_text; string market_path; SDPCCAPNativeLoadContract market_contract; float condition_scale[109], condition_bias[109]; //--- Full assigned profile identity for Actor/Critic. string profile_sha256; //--- Stable Market-stage identity, excluding downstream sections. string market_profile_sha256; string task_id; string diagnostic_case; ENUM_TIMEFRAMES timeframe; ENUM_DPCC_ENTRY_STAGE stage; ENUM_DPCC_RESUME_MODE resume_mode; ENUM_DPCC_MODE mode; ulong stop_after_updates; }; //+------------------------------------------------------------------+ //| Configure EA controls directly; settings files are not required | //+------------------------------------------------------------------+ bool DPCCConfigureExecutionControl(const string package_root, const ENUM_TIMEFRAMES timeframe, const ENUM_DPCC_ENTRY_STAGE stage, const string diagnostic_case, const ulong stop_after_updates, const ENUM_DPCC_RESUME_MODE resume_mode, const ENUM_DPCC_MODE mode, const ENUM_DPCC_ENTRY_STAGE expected_stage, SDPCCExecutionControl &control, string &reason, const bool require_chart_timeframe = false) { ZeroMemory(control); reason = ""; const ENUM_TIMEFRAMES selected = (timeframe == PERIOD_CURRENT ? (ENUM_TIMEFRAMES)_Period : timeframe); string canonical = ""; if(!DPCCCheckpointCanonicalPath(package_root, canonical, reason) || PeriodSeconds(selected) <= 0 || stage != expected_stage || stage < DPCC_STAGE_STUDYFORECAST || stage > DPCC_STAGE_TEST || mode < DPCC_P0 || mode > DPCC_P2) { if(reason == "") reason = "execution_control.inputs"; return(false); } control.package_root = canonical; control.timeframe = selected; control.stage = stage; control.diagnostic_case = diagnostic_case; control.stop_after_updates = stop_after_updates; control.mode = mode; control.resume_mode = (resume_mode == DPCC_RESUME_UNASSIGNED ? (stage == DPCC_STAGE_STUDYONLINE ? DPCC_RESUME_NEW_EPISODE_FROM_WEIGHTS : DPCC_RESUME_NEW) : resume_mode); if(control.resume_mode < DPCC_RESUME_EXACT || control.resume_mode > DPCC_RESUME_NEW) { reason = "execution_control.resume_mode"; return(false); } //--- Direct Z/account inputs use the existing neutral affine transform. ArrayInitialize(control.condition_scale, 1.0f); ArrayInitialize(control.condition_bias, 0.0f); return(true); } //+------------------------------------------------------------------+ //| Read a Market DPC through its native reader and embedded | //| manifest | //+------------------------------------------------------------------+ bool DPCCLoadMarketExecutionProfile(SDPCCExecutionControl &control, const bool common_files, string &reason) { reason = ""; const string path = control.package_root + "/Market.current.dpc"; const int handle = FileOpen(path, FILE_READ | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("market_profile.open.%d", GetLastError()); return(false); } const ulong file_bytes = FileSize(handle); uchar header[]; const bool read_header = DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason); FileClose(handle); if(!read_header || file_bytes == 0 || file_bytes > (ulong)LONG_MAX) return(false); const string source_abi = DPCCAPReadSHA(header, 40); CDPCCAPRoleReader reader; if(!reader.Open(path, common_files, 1, source_abi, file_bytes, 64, reason)) return(false); string manifest_text = ""; const bool manifest_read = reader.ReadJSONSection(1, 1048576, manifest_text, reason); if(!manifest_read) { reader.Close(); return(false); } CDPCCJsonDocument manifest; if(!manifest.Parse(manifest_text, reason)) { reader.Close(); return(false); } const int root = manifest.Root(), profile = manifest.FindMember(root, "profile"); string task = "", market_sha = "", profile_text = ""; bool metadata_ok = (DPCCResourceReadString(manifest, root, "task_id", task, reason, "market.task_id") && DPCCResourceReadString(manifest, profile, "market_sha256", market_sha, reason, "market.profile.market_sha256") && DPCCResourceReadString(manifest, profile, "contents", profile_text, reason, "market.profile.contents")); SDPCCAPNativeLoadContract native; if(metadata_ok) metadata_ok = reader.ReadReadyMarketContract(market_sha, task, file_bytes, native, reason, true); reader.Close(); if(!metadata_ok) return(false); CDPCCJsonDocument embedded; if(!embedded.Parse(profile_text, reason)) return(false); //--- Old Market metadata may predate the downstream affine coordinates. //--- Preserve explicit coordinates when present; absent coordinates are direct inputs. float scale[109], bias[109]; ArrayInitialize(scale, 1.0f); ArrayInitialize(bias, 0.0f); const int condition = embedded.FindMember(embedded.Root(), "condition"); const int scales = embedded.FindMember(condition, "scale"), biases = embedded.FindMember(condition, "bias"); if(scales >= 0 || biases >= 0) { if(embedded.Type(scales) != DPCC_JSON_ARRAY || embedded.Type(biases) != DPCC_JSON_ARRAY || embedded.ChildCount(scales) != 109 || embedded.ChildCount(biases) != 109) { reason = "market_profile.condition_shape"; return(false); } for(int i = 0; i < 109; i++) { double s = 0, b = 0; if(!embedded.TryNumber(embedded.ChildAt(scales, i), s) || !embedded.TryNumber(embedded.ChildAt(biases, i), b) || !MathIsValidNumber(s) || s <= 0 || !MathIsValidNumber((float)s) || (float)s <= 0 || !MathIsValidNumber(b) || !MathIsValidNumber((float)b)) { reason = StringFormat("market_profile.condition_value_%d", i); return(false); } scale[i] = (float)s; bias[i] = (float)b; } } control.market_path = path; control.market_contract = native; control.profile_text = profile_text; control.profile_sha256 = native.profile_sha256; control.market_profile_sha256 = native.market_profile_sha256; control.task_id = task; if(ArrayCopy(control.condition_scale, scale) != 109 || ArrayCopy(control.condition_bias, bias) != 109) { reason = "market_profile.condition_capacity"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCExecutionProfileString | //+------------------------------------------------------------------+ bool DPCCExecutionProfileString(CDPCCJsonDocument &document, const int root, const string key, string &value, string &reason) { const int node = document.FindMember(root, key); if(node < 0) { reason = "profile.required." + key; return(false); } if(document.Type(node) != DPCC_JSON_STRING) { reason = "profile.required_string." + key; return(false); } value = document.Text(node); if(value == "") { reason = "profile.empty." + key; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCExecutionDiagnosticValid | //+------------------------------------------------------------------+ bool DPCCExecutionDiagnosticValid(const string value) { if(value == "") return(true); if(StringLen(value) != 3 || StringGetCharacter(value, 0) != 'A') return(false); return(StringGetCharacter(value, 1) >= '0' && StringGetCharacter(value, 1) <= '9' && StringGetCharacter(value, 2) >= '0' && StringGetCharacter(value, 2) <= '9'); } //+------------------------------------------------------------------+ //| Escape one JSON string for the deterministic profile projection | //+------------------------------------------------------------------+ string DPCCExecutionJSONString(const string value) { string encoded = "\""; for(int i = 0; i < StringLen(value); i++) { const ushort c = StringGetCharacter(value, i); if(c == '\"' || c == '\\') encoded += "\\" + ShortToString(c); else if(c < 32) encoded += StringFormat("\\u%04x", (uint)c); else encoded += ShortToString(c); } return(encoded + "\""); } //+------------------------------------------------------------------+ //| Serialize one parsed JSON node without source whitespace | //+------------------------------------------------------------------+ bool DPCCExecutionProfileNodeJSON(CDPCCJsonDocument &document, const int node, string &json, string &reason, const int depth) { json = ""; if(node < 0 || depth > DPCC_JSON_MAX_DEPTH) { reason = "profile.market_scope.node_or_depth"; return(false); } const ENUM_DPCC_JSON_TYPE type = document.Type(node); if(type == DPCC_JSON_STRING) { json = DPCCExecutionJSONString(document.Text(node)); return(true); } if(type == DPCC_JSON_NUMBER) { json = document.Text(node); return(json != ""); } if(type == DPCC_JSON_BOOLEAN) { json = document.Text(node); return(json == "true" || json == "false"); } if(type == DPCC_JSON_NULL) { json = "null"; return(true); } if(type == DPCC_JSON_ARRAY) { json = "["; for(int i = 0; i < document.ChildCount(node); i++) { const int child = document.ChildAt(node, i); string value = ""; if(!DPCCExecutionProfileNodeJSON(document, child, value, reason, depth + 1)) return(false); json += (i == 0 ? "" : ",") + value; } json += "]"; return(true); } if(type == DPCC_JSON_OBJECT) { int order[]; const int count = document.ChildCount(node); if(ArrayResize(order, count) != count) { reason = "profile.market_scope.object_order_storage"; return(false); } for(int i = 0; i < count; i++) order[i] = document.ChildAt(node, i); //--- Sort keys so formatting and object-member order do not change the stage identity. for(int i = 1; i < count; i++) { const int current = order[i]; int j = i - 1; while(j >= 0 && StringCompare(document.Key(order[j]), document.Key(current), true) > 0) { order[j + 1] = order[j]; j--; } order[j + 1] = current; } json = "{"; for(int i = 0; i < count; i++) { string value = ""; if(!DPCCExecutionProfileNodeJSON(document, order[i], value, reason, depth + 1)) return(false); json += (i == 0 ? "" : ",") + DPCCExecutionJSONString(document.Key(order[i])) + ":" + value; } json += "}"; return(true); } reason = "profile.market_scope.unsupported_json_type"; return(false); } //+------------------------------------------------------------------+ //| Serialize a fixed subset of one profile object | //+------------------------------------------------------------------+ bool DPCCExecutionSelectedProfileObject(CDPCCJsonDocument &document, const int parent, const string &keys[], string &json, string &reason) { json = "{"; for(int i = 0; i < ArraySize(keys); i++) { if(i > 0) json += ","; json += DPCCExecutionJSONString(keys[i]) + ":"; const int node = document.FindMember(parent, keys[i]); if(node < 0) json += "null"; else { string value = ""; if(!DPCCExecutionProfileNodeJSON(document, node, value, reason, 0)) return(false); json += value; } } json += "}"; return(true); } //+------------------------------------------------------------------+ //| Bind only settings consumed by M05 and its Market readiness gate | //+------------------------------------------------------------------+ bool DPCCExecutionMarketProfileSHA(CDPCCJsonDocument &document, const int root, string &sha, string &reason) { sha = ""; reason = ""; if(document.Type(root) != DPCC_JSON_OBJECT) { reason = "profile.market_scope.root_object_required"; return(false); } const string market_keys[] = {"contract_id", "revision", "profile_kind", "task_id", "symbol_source", "timeframe", "shape", "diffusion", "price", "projection", "numeric", "data", "persistence", "market_phases", "resource_target" }; string wire = "DPCC_EXECUTION_PROFILE_MARKET_SCOPE_V1\n"; for(int i = 0; i < ArraySize(market_keys); i++) { const int node = document.FindMember(root, market_keys[i]); string value = "null"; if(node >= 0 && !DPCCExecutionProfileNodeJSON(document, node, value, reason, 0)) return(false); wire += market_keys[i] + "=" + value + "\n"; } const int donor = document.FindMember(root, "donor"); const string donor_keys[] = {"market_preset", "ompb_calibration_preset", "market_source_identity_sha256"}; string selected = ""; if(!DPCCExecutionSelectedProfileObject(document, donor, donor_keys, selected, reason)) return(false); wire += "donor_market=" + selected + "\n"; const int limits = document.FindMember(root, "limits"); const string verify_limit_keys[] = {"max_price_error", "min_scenario_coverage", "min_distinct_scenarios"}; if(!DPCCExecutionSelectedProfileObject(document, limits, verify_limit_keys, selected, reason)) return(false); wire += "price_verify_limits=" + selected + "\n"; uchar bytes[]; const int byte_count = StringToCharArray(wire, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(byte_count <= 1 || ArrayResize(bytes, byte_count - 1) != byte_count - 1 || !DPCCSHA256Lower(bytes, sha, reason)) { if(reason == "") reason = "profile.market_scope.sha256"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Explicit binding supplied by the admitted epoch/config owner | //+------------------------------------------------------------------+ struct SDPCCCheckpointBinding { string training_config_sha256; string model_manifest_sha256; string bank_manifest_sha256; string dataset_manifest_sha256; string time_index_sha256; string normalization_section_sha256; string decoder_plan_sha256; string denoiser_plan_sha256; }; struct SDPCCCheckpointCursor { uint stage; uint completed_epoch; uint next_epoch; uint next_record; uint decoder_updates; uint denoiser_updates; }; struct SDPCCCheckpointArtifact { string manifest_path; string manifest_sha256; string decoder_path; string decoder_sha256; string denoiser_path; string denoiser_sha256; string binding_sha256; string initial_manifest_path; string initial_manifest_sha256; bool common_files; ulong decoder_state; ulong denoiser_state; string decoder_probe_sha256; string denoiser_probe_sha256; double decoder_probe_loss; double denoiser_probe_loss; SDPCCCheckpointCursor cursor; }; //+------------------------------------------------------------------+ //| Function DPCCCheckpointTextSHA | //+------------------------------------------------------------------+ bool DPCCCheckpointTextSHA(const string source, string &sha, string &reason) { uchar bytes[], digest[]; int count = StringToCharArray(source, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(count <= 0) { reason = "checkpoint.utf8"; return(false); } if(bytes[count - 1] == 0) count--; CDPCCIncrementalSHA256 hasher; if(!hasher.Init(reason) || !hasher.Update(bytes, 0, count, reason)) return(false); return(hasher.Final(digest, sha, reason)); } //+------------------------------------------------------------------+ //| Hash actual bytes incrementally without materializing a CNet | //+------------------------------------------------------------------+ bool DPCCCheckpointFileSHA(const string path, const bool common_files, const ulong limit, string &sha, string &reason) { sha = ""; reason = ""; const int flags = FILE_READ | FILE_BIN | FILE_SHARE_READ | (common_files ? FILE_COMMON : 0); const int handle = FileOpen(path, flags); if(handle == INVALID_HANDLE) { reason = "checkpoint.hash.open"; return(false); } ResetLastError(); const ulong length = FileSize(handle); if(GetLastError() != 0 || length == 0 || length > limit || limit == 0) { FileClose(handle); reason = "checkpoint.hash.length"; return(false); } CDPCCIncrementalSHA256 hasher; uchar bytes[], digest[]; if(!hasher.Init(reason) || ArrayResize(bytes, 65536) != 65536) { FileClose(handle); reason = "checkpoint.hash.capacity"; return(false); } ulong consumed = 0; while(consumed < length) { const uint wanted = (uint)(length - consumed < 65536 ? length - consumed : (ulong)65536); const uint read = FileReadArray(handle, bytes, 0, wanted); if(read != wanted || !hasher.Update(bytes, 0, (int)read, reason)) { FileClose(handle); reason = "checkpoint.hash.read"; return(false); } consumed += read; } const bool complete = (FileTell(handle) == length && FileSize(handle) == length); FileClose(handle); if(!complete) { reason = "checkpoint.hash.changed_length"; return(false); } return(hasher.Final(digest, sha, reason)); } //+------------------------------------------------------------------+ //| Copy one pinned local artifact to Common without replacing data | //+------------------------------------------------------------------+ bool DPCCStageFileToCommon(const string path, const string expected_sha, const ulong maximum_bytes, string &reason) { reason = ""; string canonical = "", actual = ""; if(!DPCCResolverLowerSha256(expected_sha) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path) { reason = "common.stage.path"; return(false); } if(FileIsExist(path, FILE_COMMON)) { if(!DPCCCheckpointFileSHA(path, true, maximum_bytes, actual, reason) || actual != expected_sha) { reason = "common.stage.destination_mismatch:" + path; return(false); } return(true); } if(!FileIsExist(path)) { reason = "common.stage.source_missing:" + path; return(false); } string folders = path; StringReplace(folders, "\\", "/"); int slash = StringFind(folders, "/"); while(slash >= 0) { const string folder = StringSubstr(folders, 0, slash); if(folder != "") FolderCreate(folder, FILE_COMMON); slash = StringFind(folders, "/", slash + 1); } ResetLastError(); if(!FileCopy(path, 0, path, FILE_COMMON)) { reason = StringFormat("common.stage.copy.%d:", GetLastError()) + path; return(false); } if(!DPCCCheckpointFileSHA(path, true, maximum_bytes, actual, reason) || actual != expected_sha) { reason = "common.stage.sha_mismatch:" + path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Write an authenticated text artifact without replacement | //+------------------------------------------------------------------+ bool DPCCCheckpointWriteText(const string path, const bool common_files, const string text, string &reason, const bool overwrite = false) { if(!overwrite && FileIsExist(path, common_files ? FILE_COMMON : 0)) { reason = "checkpoint.destination_exists"; return(false); } uchar bytes[]; int count = StringToCharArray(text, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(count <= 0) { reason = "checkpoint.manifest.utf8"; return(false); } if(bytes[count - 1] == 0) count--; ResetLastError(); const int handle = FileOpen(path, FILE_WRITE | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("checkpoint.manifest.open.%d", GetLastError()); return(false); } ResetLastError(); bool written = (FileWriteArray(handle, bytes, 0, count) == (uint)count); FileFlush(handle); written = (written && GetLastError() == 0); FileClose(handle); if(!written) reason = "checkpoint.manifest.write_or_flush"; return(written); } //--- Manifest text is generated from admitted typed values, then compared //--- byte-for-byte on restore; no generic or permissive manifest parser. string DPCCCheckpointManifest(const SDPCCCheckpointArtifact &artifact) { return("DPCC_NATIVE_FULL_STATE_V1\n" + artifact.binding_sha256 + "\n" + artifact.decoder_path + "\n" + artifact.decoder_sha256 + "\n" + artifact.denoiser_path + "\n" + artifact.denoiser_sha256 + "\n" + (string)artifact.common_files + "\n" + (string)artifact.decoder_state + "\n" + (string)artifact.denoiser_state + "\n" + (string)artifact.cursor.stage + "\n" + (string)artifact.cursor.completed_epoch + "\n" + (string)artifact.cursor.next_epoch + "\n" + (string)artifact.cursor.next_record + "\n" + (string)artifact.cursor.decoder_updates + "\n" + (string)artifact.cursor.denoiser_updates + "\n" + artifact.decoder_probe_sha256 + "\n" + artifact.denoiser_probe_sha256 + "\n" + StringFormat("%.17g\n%.17g\n", artifact.decoder_probe_loss, artifact.denoiser_probe_loss) + artifact.initial_manifest_path + "\n" + artifact.initial_manifest_sha256 + "\nEND_DPCC_NATIVE_FULL_STATE_V1\n"); } //+------------------------------------------------------------------+ //| Function DPCCCheckpointOutputSHA | //+------------------------------------------------------------------+ bool DPCCCheckpointOutputSHA(const float &values[], string &sha, string &reason) { string text = "DPCC_FLOAT_OUTPUT_V1\n"; for(int i = 0; i < ArraySize(values); i++) { if(!MathIsValidNumber(values[i])) { reason = "checkpoint.output.numeric"; return(false); } text += StringFormat("%.17g\n", (double)values[i]); } return(DPCCCheckpointTextSHA(text, sha, reason)); } //+------------------------------------------------------------------+ //| Function DPCCCheckpointCanonicalPath | //+------------------------------------------------------------------+ bool DPCCCheckpointCanonicalPath(const string path, string &canonical, string &reason) { string prepared = ""; if(!DPCCResolverPreparePath(path, prepared, reason, "checkpoint.path")) return(false); string segments[]; if(!DPCCResolverAppendSegments(prepared, segments, 0, reason, "checkpoint.path")) return(false); canonical = DPCCResolverJoinSegments(segments); if(StringLen(canonical) == 0 || canonical != prepared) { reason = "checkpoint.path.canonical"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCCheckpointReadUnsigned | //+------------------------------------------------------------------+ bool DPCCCheckpointReadUnsigned(const string lexeme, const ulong maximum, ulong &value) { value = 0; const int count = StringLen(lexeme); if(count == 0 || count > 20 || (count > 1 && StringGetCharacter(lexeme, 0) == '0')) return(false); for(int i = 0; i < count; i++) { const ushort ch = StringGetCharacter(lexeme, i); if(ch < '0' || ch > '9') return(false); const ulong digit = (ulong)(ch - '0'); if(value > (maximum - digit) / 10) return(false); value = value * 10 + digit; } return((string)value == lexeme); } //+------------------------------------------------------------------+ //| Reads only exact canonical version-one persisted typed manifests | //+------------------------------------------------------------------+ bool DPCCReadCheckpointArtifact(const string manifest_path, const bool common_files, const string expected_sha, SDPCCCheckpointArtifact &artifact, string &reason) { ZeroMemory(artifact); reason = ""; string canonical = ""; if(!DPCCResolverLowerSha256(expected_sha) || !DPCCCheckpointCanonicalPath(manifest_path, canonical, reason)) { reason = "checkpoint.reader.reference"; return(false); } uchar bytes[]; string observed = "", source = ""; if(!DPCCReadArtifactBytes(canonical, common_files, 65536, bytes, reason) || !DPCCSHA256Lower(bytes, observed, reason) || observed != expected_sha || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 65536, source, reason)) { reason = "checkpoint.reader.bytes_sha_or_utf8"; return(false); } string fields[22]; int from = 0; for(uint i = 0; i < 22; i++) { const int end = StringFind(source, "\n", from); if(end < from) { reason = "checkpoint.reader.field_count"; return(false); } fields[i] = StringSubstr(source, from, end - from); from = end + 1; } if(from != StringLen(source) || fields[0] != "DPCC_NATIVE_FULL_STATE_V1" || fields[21] != "END_DPCC_NATIVE_FULL_STATE_V1") { reason = "checkpoint.reader.version_or_extra"; return(false); } SDPCCCheckpointArtifact candidate; ZeroMemory(candidate); candidate.manifest_path = manifest_path; candidate.manifest_sha256 = expected_sha; candidate.common_files = common_files; candidate.binding_sha256 = fields[1]; candidate.decoder_path = fields[2]; candidate.decoder_sha256 = fields[3]; candidate.denoiser_path = fields[4]; candidate.denoiser_sha256 = fields[5]; if(fields[6] != (string)common_files || !DPCCResolverLowerSha256(fields[1]) || !DPCCResolverLowerSha256(fields[3]) || !DPCCResolverLowerSha256(fields[5]) || !DPCCCheckpointCanonicalPath(fields[2], canonical, reason) || !DPCCCheckpointCanonicalPath(fields[4], canonical, reason) || fields[2] == fields[4] || fields[2] == manifest_path || fields[4] == manifest_path || !DPCCCheckpointReadUnsigned(fields[7], ULONG_MAX, candidate.decoder_state) || candidate.decoder_state == 0 || !DPCCCheckpointReadUnsigned(fields[8], ULONG_MAX, candidate.denoiser_state) || candidate.denoiser_state == 0) { reason = "checkpoint.reader.bindings_paths_or_state"; return(false); } ulong cursor_values[6]; for(uint i = 0; i < 6; i++) if(!DPCCCheckpointReadUnsigned(fields[9 + i], UINT_MAX, cursor_values[i])) { reason = "checkpoint.reader.cursor_overflow_or_lexeme"; return(false); } candidate.cursor.stage = (uint)cursor_values[0]; candidate.cursor.completed_epoch = (uint)cursor_values[1]; candidate.cursor.next_epoch = (uint)cursor_values[2]; candidate.cursor.next_record = (uint)cursor_values[3]; candidate.cursor.decoder_updates = (uint)cursor_values[4]; candidate.cursor.denoiser_updates = (uint)cursor_values[5]; candidate.decoder_probe_sha256 = fields[15]; candidate.denoiser_probe_sha256 = fields[16]; CDPCCJsonDocument number; if(!DPCCResolverLowerSha256(fields[15]) || !DPCCResolverLowerSha256(fields[16]) || !number.Parse(fields[17], reason) || !number.TryNumber(number.Root(), candidate.decoder_probe_loss) || candidate.decoder_probe_loss < 0 || !number.Parse(fields[18], reason) || !number.TryNumber(number.Root(), candidate.denoiser_probe_loss) || candidate.denoiser_probe_loss < 0) { reason = "checkpoint.reader.finite_nonnegative_probe"; return(false); } candidate.initial_manifest_path = fields[19]; candidate.initial_manifest_sha256 = fields[20]; if(candidate.cursor.stage > 2 || candidate.cursor.next_record != 0 || candidate.cursor.completed_epoch == UINT_MAX || candidate.cursor.next_epoch != candidate.cursor.completed_epoch + 1 || (candidate.cursor.stage == 0 && (candidate.cursor.completed_epoch != 0 || candidate.cursor.decoder_updates != 0 || candidate.cursor.denoiser_updates != 0 || StringLen(fields[19]) != 0 || StringLen(fields[20]) != 0)) || (candidate.cursor.stage != 0 && (!DPCCCheckpointCanonicalPath(fields[19], canonical, reason) || !DPCCResolverLowerSha256(fields[20]) || fields[19] == manifest_path)) || DPCCCheckpointManifest(candidate) != source) { reason = "checkpoint.reader.epoch_initial_or_noncanonical"; return(false); } artifact = candidate; return(true); } #endif //+------------------------------------------------------------------+ //| Historical price targets retained with source-specific | //| resolution | //+------------------------------------------------------------------+ struct SDPCCHistoricalPriceTicks { bool available; long time_msc, count; double ohlc[4]; }; struct SDPCCHistoricalPriceBar { bool available; long time_msc; }; struct SDPCCStudyForecastBar { long server_open, tick_volume, spread; double ohlc[4]; float indicators[5]; uint raw_version; ENUM_SYMBOL_CHART_MODE chart_quote_mode; SDPCCHistoricalPriceTicks price_ticks; SDPCCHistoricalPriceBar price_bar; }; //+------------------------------------------------------------------+ //| Validate native indicator applied-price values | //+------------------------------------------------------------------+ double DPCCHistoricalPriceTargetValue(const SDPCCStudyForecastBar &bar, const int component) { if(component < 0 || component >= 4) return(EMPTY_VALUE); if(bar.raw_version == 3 && bar.price_bar.available) return(bar.ohlc[component]); if(bar.raw_version == 2 && bar.price_ticks.available) return(bar.price_ticks.ohlc[component]); return(EMPTY_VALUE); } //+------------------------------------------------------------------+ //| Function DPCCHistoricalPriceTargetTimeMsc | //+------------------------------------------------------------------+ long DPCCHistoricalPriceTargetTimeMsc(const SDPCCStudyForecastBar &bar) { if(bar.raw_version == 3 && bar.price_bar.available) return(bar.price_bar.time_msc); if(bar.raw_version == 2 && bar.price_ticks.available) return(bar.price_ticks.time_msc); return(0); } //+------------------------------------------------------------------+ //| Function DPCCStudyForecastAppliedPriceValid | //+------------------------------------------------------------------+ bool DPCCStudyForecastAppliedPriceValid(const long value) { return(value == PRICE_CLOSE || value == PRICE_OPEN || value == PRICE_HIGH || value == PRICE_LOW || value == PRICE_MEDIAN || value == PRICE_TYPICAL || value == PRICE_WEIGHTED); } //+------------------------------------------------------------------+ //| Encode the actual fixed-period indicator acquisition parameters | //+------------------------------------------------------------------+ string DPCCStudyForecastIndicatorParams(const ENUM_APPLIED_PRICE rsi_price, const ENUM_APPLIED_PRICE cci_price, const ENUM_APPLIED_PRICE macd_price) { return("PARAMS|" + (string)RSIPeriod + "|" + EnumToString(rsi_price) + "|" + (string)CCIPeriod + "|" + EnumToString( cci_price) + "|" + (string)ATRPeriod + "|" + (string)FastPeriod + "|" + (string)SlowPeriod + "|" + (string)SignalPeriod + "|" + EnumToString(macd_price)); } //+------------------------------------------------------------------+ //| Historical H1 price observation with explicit source resolution | //+------------------------------------------------------------------+ struct SDPCCHistoricalPriceSnapshot { ulong decision_id; string source_server, symbol, raw_sha256; ENUM_TIMEFRAMES resolved_timeframe; bool quote_basis_resolved; ENUM_SYMBOL_CHART_MODE chart_quote_mode; string quote_basis, quote_basis_source; long quote_time_server, quote_time_msc; string quote_time_resolution; bool target_time_resolved; long anchor_bar_index, anchor_bar_open_server; long completed_bar_index, completed_bar_open_server, completed_bar_successor_server; uint atr_period; double anchor, scale, tick_size; }; union UDPCCObservedFloatBits { float value; uint bits; }; //--- Bind the Spot FX reference quote to the retained historical Account24 state. bool DPCCBindHistoricalFxReference(const SDPCCShape &shape, float &context[], const float &account24[], string &reason) { reason = ""; SDPCCLayout layout, rollout; uint tokens = 0; if(!DPCCCompositeLayouts(shape, layout, rollout, tokens) || ArraySize(context) != (int)layout.offsets[10] || (ArraySize(account24) != 24 && ArraySize(account24) != 33)) { reason = "historical_fx_reference.shape"; return(false); } const float prior_ask = account24[16]; if(!MathIsValidNumber((double)prior_ask) || prior_ask <= 0) { reason = "historical_fx_reference.prior_ask"; return(false); } context[(int)layout.offsets[5] + 14] = prior_ask; return(true); } //--- Resolve one timestamp in the raw reader's strictly increasing sequence in O(log N). bool DPCCStudyForecastUniqueServerOpen(const SDPCCStudyForecastBar &bars[], const long server_open, int &index, string &reason) { index = -1; reason = ""; if(server_open <= 0 || ArraySize(bars) == 0) {reason = "study_forecast.time_join.arguments"; return(false);} int lo = 0, hi = ArraySize(bars); while(lo < hi) { const int mid = lo + (hi - lo) / 2; if(bars[mid].server_open < server_open) lo = mid + 1; else hi = mid; } if(lo >= ArraySize(bars) || bars[lo].server_open != server_open) {reason = "study_forecast.time_join.missing"; return(false);} if((lo > 0 && bars[lo - 1].server_open == server_open) || (lo + 1 < ArraySize(bars) && bars[lo + 1].server_open == server_open)) {reason = "study_forecast.time_join.ambiguous"; return(false);} index = lo; return(true); } //--- Join admitted factual labels to the same raw H1 rows used by T15, at float32 artifact precision. bool DPCCStudyForecastMatchNativeHorizon(const SDPCCStudyForecastBar &bars[], const int decision, const float &target_native_ohlc[], const long &server_opens[], string &reason) { reason = ""; const int horizon = ArraySize(server_opens); if(decision < 0 || horizon <= 0 || ArraySize(target_native_ohlc) != horizon * 4 || (ulong)decision + (ulong)horizon > (ulong)ArraySize(bars)) {reason = "study_forecast.native_join.shape"; return(false);} for(int h = 0; h < horizon; h++) { const int raw = decision + h; if(server_opens[h] <= 0 || bars[raw].server_open != server_opens[h]) {reason = "study_forecast.native_join.time"; return(false);} for(int channel = 0; channel < 4; channel++) { UDPCCObservedFloatBits source, observed; source.value = (float)bars[raw].ohlc[channel]; observed.value = target_native_ohlc[h * 4 + channel]; if(!MathIsValidNumber(bars[raw].ohlc[channel]) || !MathIsValidNumber((double)source.value) || !MathIsValidNumber((double)observed.value) || source.value <= 0 || source.bits != observed.bits) {reason = "study_forecast.native_join.ohlc_float32"; return(false);} } } return(true); } //--- Decoded wire fields; source identity and clock admission belong to the caller. struct SDPCCClosedHistoryRow { long ordinal, successor_open_server, spread, real_volume; SDPCCStudyForecastBar bar; }; //--- Set the fields common to every layer description before applying role-specific parameters. bool DPCCSetLayerBaseDefaults(CLayerDescription *description, const uint type, const uint count, const uint batch) { if(CheckPointer(description) == POINTER_INVALID) return(false); description.type = type; description.count = count; description.batch = batch; description.activation = None; description.optimization = ADAM; return(true); } //+------------------------------------------------------------------+ //| DPCCBuildNineClosedBarFeatures builds the shared 9-channel input | //+------------------------------------------------------------------+ bool DPCCBuildNineClosedBarFeatures(const double open, const double high, const double low, const double close, const long tick_volume, const float &indicators[], float &out[]) { if(tick_volume < 0 || ArraySize(indicators) != 5 || ArraySize(out) != 9 || !MathIsValidNumber(open) || !MathIsValidNumber(high) || !MathIsValidNumber(low) || !MathIsValidNumber(close)) return(false); for(int i = 0; i < 5; i++) if(!MathIsValidNumber((double)indicators[i]) || indicators[i] == EMPTY_VALUE) return(false); out[0] = (float)(close - open); out[1] = (float)(high - open); out[2] = (float)(low - open); out[3] = (float)(tick_volume / 1000.0f); for(int i = 0; i < 5; i++) out[4 + i] = indicators[i]; for(int i = 0; i < 9; i++) if(!MathIsValidNumber((double)out[i])) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStudyForecastFeature | //+------------------------------------------------------------------+ bool DPCCStudyForecastFeature(const SDPCCStudyForecastBar &bar, float &out[]) { if(ArrayResize(out, 9) != 9 || bar.tick_volume < 0) return(false); const double raw_open = bar.ohlc[0]; if(!MathIsValidNumber(raw_open) || raw_open <= 0 || !MathIsValidNumber(bar.ohlc[1]) || !MathIsValidNumber(bar.ohlc[2]) || !MathIsValidNumber(bar.ohlc[3]) || bar.ohlc[1] < bar.ohlc[2] || bar.ohlc[1] < raw_open || bar.ohlc[1] < bar.ohlc[3] || bar.ohlc[2] > raw_open || bar.ohlc[2] > bar.ohlc[3]) return(false); return(DPCCBuildNineClosedBarFeatures(bar.ohlc[0], bar.ohlc[1], bar.ohlc[2], bar.ohlc[3], bar.tick_volume, bar.indicators, out)); } //+------------------------------------------------------------------+ //| DPCCPackStudyForecastInput packs five prior bars in CNet order | //+------------------------------------------------------------------+ bool DPCCPackStudyForecastInput(const SDPCCStudyForecastBar &bars[], const int decision, float &out[]) { ArrayResize(out, 0); if(decision < 5 || decision > ArraySize(bars)) return(false); float packed[]; if(ArrayResize(packed, 45) != 45) return(false); float values[]; for(int b = 0; b < 5; b++) { if(!DPCCStudyForecastFeature(bars[decision - 1 - b], values)) return(false); for(int v = 0; v < 9; v++) packed[v * 5 + b] = values[v]; } if(ArrayCopy(out, packed) != 45) { ArrayResize(out, 0); return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCDecodeClosedHistoryRow reads one closed-history source row | //+------------------------------------------------------------------+ bool DPCCDecodeClosedHistoryRow(const string text, const long expected_ordinal, SDPCCClosedHistoryRow &row, string &reason) { //--- Validate the wire fields before converting observations. string fields[]; const int field_count = StringSplit(text, '|', fields); if(expected_ordinal < 0 || (field_count != 16 && field_count != 18 && field_count != 24) || fields[0] != "ROW") { reason = "closed_history.row.fields"; return(false); } SDPCCClosedHistoryRow candidate; ZeroMemory(candidate); candidate.bar.raw_version = 1; candidate.bar.chart_quote_mode = (ENUM_SYMBOL_CHART_MODE) - 1; ulong ordinal = 0, successor = 0, ticks = 0, spread = 0, volume = 0; if(!DPCCCheckpointReadUnsigned(fields[1], (ulong)LONG_MAX, ordinal) || ordinal != (ulong)expected_ordinal || !DPCCCheckpointReadUnsigned(fields[3], (ulong)LONG_MAX, successor) || !DPCCCheckpointReadUnsigned(fields[8], (ulong)LONG_MAX, ticks) || !DPCCCheckpointReadUnsigned(fields[9], (ulong)INT_MAX, spread) || !DPCCCheckpointReadUnsigned(fields[10], (ulong)LONG_MAX, volume)) { reason = "closed_history.row.integer"; return(false); } const datetime server_open = StringToTime(fields[2]); if(server_open <= 0 || (string)server_open != fields[2] || (long)successor <= (long)server_open) { reason = "closed_history.row.server_time"; return(false); } candidate.ordinal = (long)ordinal; candidate.successor_open_server = (long)successor; candidate.spread = (long)spread; candidate.real_volume = (long)volume; candidate.bar.server_open = (long)server_open; candidate.bar.tick_volume = (long)ticks; candidate.bar.spread = candidate.spread; //--- Parse each external number without allowing fields to change array shape. for(int c = 0; c < 9; c++) { const int field = (c < 4 ? 4 + c : 11 + c - 4); CDPCCJsonDocument document; double value = 0; if(!document.Parse(fields[field], reason) || !document.TryNumber(document.Root(), value) || !MathIsValidNumber(value) || value == EMPTY_VALUE) { reason = "closed_history.row.number"; return(false); } if(c < 4) { if(value <= 0) { reason = "closed_history.row.ohlc"; return(false); } candidate.bar.ohlc[c] = value; } else candidate.bar.indicators[c - 4] = (float)value; } //--- V2 tick observations remain hourly; V3 binds the selected timeframe OHLC by bar-open time. if(field_count == 24) { ulong available = 0, time_msc = 0, tick_count = 0; const long period_seconds = PeriodSeconds(PERIOD_H1); if(period_seconds <= 0 || fields[16] != "TICKS_H1_V1" || !DPCCCheckpointReadUnsigned(fields[17], 1, available) || !DPCCCheckpointReadUnsigned(fields[18], (ulong)LONG_MAX, time_msc) || !DPCCCheckpointReadUnsigned(fields[19], (ulong)LONG_MAX, tick_count) || candidate.bar.server_open > LONG_MAX / 1000 - period_seconds || candidate.successor_open_server > LONG_MAX / 1000) {reason = "closed_history.row.tick_fields"; return(false);} candidate.bar.raw_version = 2; candidate.bar.price_ticks.available = (available == 1); candidate.bar.price_ticks.time_msc = (long)time_msc; candidate.bar.price_ticks.count = (long)tick_count; for(int c = 0; c < 4; c++) { CDPCCJsonDocument document; double value = 0; if(!document.Parse(fields[20 + c], reason) || !document.TryNumber(document.Root(), value) || !MathIsValidNumber(value) || (available == 1 ? value <= 0 : value != 0)) {reason = "closed_history.row.tick_number"; return(false);} candidate.bar.price_ticks.ohlc[c] = value; } if(available == 1) { if(tick_count == 0 || time_msc < (ulong)candidate.bar.server_open * 1000 || time_msc >= (ulong)(candidate.bar.server_open + period_seconds) * 1000 || time_msc >= (ulong)candidate.successor_open_server * 1000 || candidate.bar.price_ticks.ohlc[1] < MathMax(candidate.bar.price_ticks.ohlc[0], candidate.bar.price_ticks.ohlc[3]) || candidate.bar.price_ticks.ohlc[2] > MathMin(candidate.bar.price_ticks.ohlc[0], candidate.bar.price_ticks.ohlc[3])) {reason = "closed_history.row.tick_geometry"; return(false);} } else if(time_msc != 0 || tick_count != 0) {reason = "closed_history.row.tick_unavailable"; return(false);} } else if(field_count == 18) { ulong time_msc = 0; if((fields[16] != "H1_OHLC_V1" && fields[16] != "BAR_OHLC_V1") || !DPCCCheckpointReadUnsigned(fields[17], (ulong)LONG_MAX, time_msc) || candidate.bar.server_open > LONG_MAX / 1000 || time_msc != (ulong)candidate.bar.server_open * 1000) {reason = "closed_history.row.bar_ohlc_anchor"; return(false);} candidate.bar.raw_version = 3; candidate.bar.price_bar.available = true; candidate.bar.price_bar.time_msc = (long)time_msc; } //--- Apply the canonical geometry and feature checks before publishing the row. float features[]; if(!DPCCStudyForecastFeature(candidate.bar, features)) { reason = "closed_history.row.feature"; return(false); } row = candidate; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Raw history limits retain the legacy tick contract | //+------------------------------------------------------------------+ #define DPCC_RAW_LEGACY_MAX_ROWS 65536 #define DPCC_RAW_LEGACY_MAX_BYTES 33554432 #define DPCC_RAW_V3_MAX_ROWS 1048576 #define DPCC_RAW_V3_MAX_BYTES 134217728 //+------------------------------------------------------------------+ //| Select a version bound before allocating the complete raw file | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastRawBytes(const string path, const bool common_files, uchar &bytes[], long &maximum_bytes, string &reason) { ArrayResize(bytes, 0); maximum_bytes = DPCC_RAW_LEGACY_MAX_BYTES; reason = ""; ResetLastError(); const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("study_forecast.raw.open:%d", GetLastError()); return(false); } const string v3_header = "DPCC_FRESH_MARKET_RAW_V3\n"; const int prefix_count = StringLen(v3_header); uchar prefix[]; ResetLastError(); const ulong length = FileSize(handle); const int size_error = GetLastError(); if(size_error != 0 || length < (ulong)prefix_count || length > (ulong)INT_MAX || ArrayResize(prefix, prefix_count) != prefix_count || FileReadArray(handle, prefix, 0, prefix_count) != (uint)prefix_count) { FileClose(handle); reason = "study_forecast.raw.header_or_length"; return(false); } if(CharArrayToString(prefix, 0, prefix_count, CP_UTF8) == v3_header) maximum_bytes = DPCC_RAW_V3_MAX_BYTES; if(length > (ulong)maximum_bytes || ArrayResize(bytes, (int)length) != (int)length) { FileClose(handle); ArrayResize(bytes, 0); reason = "study_forecast.raw.version_limit_or_capacity"; return(false); } if(!FileSeek(handle, 0, SEEK_SET) || FileReadArray(handle, bytes, 0, (int)length) != (uint)length) { FileClose(handle); ArrayResize(bytes, 0); reason = "study_forecast.raw.read"; return(false); } FileClose(handle); return(true); } //+------------------------------------------------------------------+ //| Validate an explicit native chart timeframe in a raw identity | //+------------------------------------------------------------------+ bool DPCCStudyForecastTimeframeName(const string value) { const ENUM_TIMEFRAMES periods[] = {PERIOD_M1, PERIOD_M2, PERIOD_M3, PERIOD_M4, PERIOD_M5, PERIOD_M6, PERIOD_M10, PERIOD_M12, PERIOD_M15, PERIOD_M20, PERIOD_M30, PERIOD_H1, PERIOD_H2, PERIOD_H3, PERIOD_H4, PERIOD_H6, PERIOD_H8, PERIOD_H12, PERIOD_D1, PERIOD_W1, PERIOD_MN1 }; for(int i = 0; i < ArraySize(periods); i++) if(value == EnumToString(periods[i])) return(true); return(false); } //+------------------------------------------------------------------+ //| Function DPCCReadStudyForecastRawEx | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastRawEx(const string path, const string expected, SDPCCStudyForecastBar &bars[], string &raw_sha, long &successor_open_server, string &reason, string &source_server, string &source_symbol, string &source_timeframe, string &indicator_params, const bool common_files = false, const string expected_indicator_params = "", const bool verify_expected_sha = true, const bool compute_sha = true, const bool verify_source_identity = true) { successor_open_server = 0; source_server = ""; source_symbol = ""; source_timeframe = ""; indicator_params = ""; uchar bytes[]; long maximum_bytes = 0; string text = "", actual = ""; if(!DPCCReadStudyForecastRawBytes(path, common_files, bytes, maximum_bytes, reason) || (compute_sha && !DPCCSHA256Lower(bytes, actual, reason)) || (!compute_sha && verify_expected_sha) || (verify_expected_sha && actual != expected) || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), maximum_bytes, text, reason)) return(false); string lines[]; const int line_count = StringSplit(text, '\n', lines); if(line_count >= 5) indicator_params = lines[4]; const bool version2 = (line_count >= 9 && lines[0] == "DPCC_FRESH_MARKET_RAW_V2"); const bool version3 = (line_count >= 9 && lines[0] == "DPCC_FRESH_MARKET_RAW_V3"); const bool quote_basis_version = (version2 || version3); const int max_rows = (version3 ? DPCC_RAW_V3_MAX_ROWS : DPCC_RAW_LEGACY_MAX_ROWS); if(StringFind(text, "\r") >= 0 || line_count < 9 || (!version2 && !version3 && lines[0] != "DPCC_FRESH_MARKET_RAW_V1") || lines[line_count - 1] != "") return(false); if(expected_indicator_params != "" && lines[4] != expected_indicator_params) { reason = "study_forecast.raw.donor_indicator_params_mismatch"; return(false); } string identity[]; if(StringSplit(lines[2], '|', identity) != (quote_basis_version ? 5 : 4) || identity[0] != "IDENTITY" || StringLen(identity[1]) == 0 || (verify_source_identity && (StringLen(identity[2]) == 0 || StringLen(identity[3]) == 0)) || (quote_basis_version && ((verify_source_identity && !DPCCStudyForecastTimeframeName(identity[3])) || (verify_source_identity && version2 && identity[3] != "PERIOD_H1") || (identity[4] != "BID" && identity[4] != "LAST")))) { reason = "study_forecast.raw.identity"; return(false); } if(quote_basis_version && verify_source_identity) { long current_basis = -1; if(!SymbolInfoInteger(identity[2], SYMBOL_CHART_MODE, current_basis) || current_basis != (long)(identity[4] == "BID" ? SYMBOL_CHART_MODE_BID : SYMBOL_CHART_MODE_LAST)) {reason = "study_forecast.raw.quote_basis_changed"; return(false);} } const int row_count = line_count - 8; if(row_count <= 0 || row_count > max_rows || lines[line_count - 2] != StringFormat("END|%d", row_count) || ArrayResize(bars, row_count) != row_count) return(false); string acquisition[]; if(StringSplit(lines[5], '|', acquisition) != 6 || acquisition[0] != "ACQUISITION_RAW" || acquisition[1] != "SERVER_TIME" || acquisition[4] != "GUARD") { reason = "study_forecast.raw.acquisition"; return(false); } const datetime capture_from = StringToTime(acquisition[2]); const datetime capture_to = StringToTime(acquisition[3]); const datetime guard_time = StringToTime(acquisition[5]); if(capture_from <= 0 || capture_to <= capture_from || guard_time < capture_to || (string)capture_from != acquisition[2] || (string)capture_to != acquisition[3] || guard_time <= 0 || (string)guard_time != acquisition[5]) { reason = "study_forecast.raw.acquisition_time"; return(false); } long previous_successor = 0; for(int i = 0; i < row_count; i++) { SDPCCClosedHistoryRow row; if(!DPCCDecodeClosedHistoryRow(lines[6 + i], i, row, reason)) { reason = StringFormat("study_forecast.raw.row.%d:", i) + reason; return(false); } if(row.bar.raw_version != (version2 ? 2 : version3 ? 3 : 1)) {reason = "study_forecast.raw.row_version"; return(false);} if(quote_basis_version) row.bar.chart_quote_mode = (identity[4] == "BID" ? SYMBOL_CHART_MODE_BID : SYMBOL_CHART_MODE_LAST); if(row.bar.server_open < capture_from || row.bar.server_open >= capture_to || (i > 0 && row.bar.server_open <= bars[i - 1].server_open) || (i > 0 && previous_successor != row.bar.server_open) || (i == row_count - 1 && row.successor_open_server != (long)guard_time)) { reason = StringFormat("study_forecast.raw.successor.%d", i); return(false); } bars[i] = row.bar; previous_successor = row.successor_open_server; } raw_sha = (compute_sha ? actual : ""); successor_open_server = previous_successor; source_server = identity[1]; source_symbol = identity[2]; source_timeframe = identity[3]; return(true); } //+------------------------------------------------------------------+ //| Function DPCCReadStudyForecastRaw | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastRaw(const string path, const string expected, SDPCCStudyForecastBar &bars[], string &raw_sha, long &successor_open_server, string &reason, string &source_server, string &source_symbol, string &source_timeframe, const bool common_files = false, const string expected_indicator_params = "", const bool verify_expected_sha = true) { string indicator_params = ""; return(DPCCReadStudyForecastRawEx(path, expected, bars, raw_sha, successor_open_server, reason, source_server, source_symbol, source_timeframe, indicator_params, common_files, expected_indicator_params, verify_expected_sha)); } //+------------------------------------------------------------------+ //| Function DPCCReadStudyForecastRaw | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastRaw(const string path, const string expected, SDPCCStudyForecastBar &bars[], string &raw_sha, long &successor_open_server, string &reason, const bool common_files = false) { string source_server = "", source_symbol = "", source_timeframe = ""; return(DPCCReadStudyForecastRaw(path, expected, bars, raw_sha, successor_open_server, reason, source_server, source_symbol, source_timeframe, common_files)); } //+------------------------------------------------------------------+ //| Read pinned raw history without exposing its final successor | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastRaw(const string path, const string expected, SDPCCStudyForecastBar &bars[], string &raw_sha, string &reason, const bool common_files = false) { long successor_open_server = 0; return(DPCCReadStudyForecastRaw(path, expected, bars, raw_sha, successor_open_server, reason, common_files)); } //--- Read authenticated source identity from the retained raw-history header. bool DPCCReadStudyForecastRawIdentity(const string path, const string expected, string &server, string &symbol, string &timeframe, string &reason) { server = ""; symbol = ""; timeframe = ""; uchar bytes[]; long maximum_bytes = 0; string text = "", actual = ""; if(!DPCCResolverLowerSha256(expected) || !DPCCReadStudyForecastRawBytes(path, false, bytes, maximum_bytes, reason) || !DPCCSHA256Lower(bytes, actual, reason) || actual != expected || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), maximum_bytes, text, reason)) { reason = "study_forecast.raw_identity.sha_or_encoding"; return(false); } string lines[]; if(StringFind(text, "\r") >= 0 || StringSplit(text, '\n', lines) < 9 || (lines[0] != "DPCC_FRESH_MARKET_RAW_V1" && lines[0] != "DPCC_FRESH_MARKET_RAW_V2" && lines[0] != "DPCC_FRESH_MARKET_RAW_V3")) { reason = "study_forecast.raw_identity.header"; return(false); } string fields[]; const bool version2 = (lines[0] == "DPCC_FRESH_MARKET_RAW_V2"); const bool version3 = (lines[0] == "DPCC_FRESH_MARKET_RAW_V3"); const bool quote_basis_version = (version2 || version3); if(StringSplit(lines[2], '|', fields) != (quote_basis_version ? 5 : 4) || fields[0] != "IDENTITY" || StringLen(fields[1]) == 0 || StringLen(fields[2]) == 0 || StringLen(fields[3]) == 0 || (quote_basis_version && (!DPCCStudyForecastTimeframeName(fields[3]) || (version2 && fields[3] != "PERIOD_H1") || (fields[4] != "BID" && fields[4] != "LAST")))) { reason = "study_forecast.raw_identity.fields"; return(false); } if(quote_basis_version) { long current_basis = -1; if(!SymbolInfoInteger(fields[2], SYMBOL_CHART_MODE, current_basis) || current_basis != (long)(fields[4] == "BID" ? SYMBOL_CHART_MODE_BID : SYMBOL_CHART_MODE_LAST)) {reason = "study_forecast.raw_identity.quote_basis_changed"; return(false);} } server = fields[1]; symbol = fields[2]; timeframe = fields[3]; reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCBuildStudyForecastSplits builds chronological server-time | //| splits | //+------------------------------------------------------------------+ bool DPCCBuildStudyForecastSplits(const SDPCCStudyForecastBar &bars[], int &fit[], int &holdout[], string &reason, const bool use_all_history = false, const datetime range_start = 0, const datetime range_end = 0) { const datetime fit_from = StringToTime("2025.01.01 00:00"); const datetime fit_to = StringToTime("2026.01.01 00:00"); const datetime holdout_to = StringToTime("2026.02.01 00:00"); if(fit_from <= 0 || fit_to <= fit_from || holdout_to <= fit_to || ArraySize(bars) < 18) { reason = "study_forecast.split.configuration"; return(false); } if(ArrayResize(fit, 0) < 0 || ArrayResize(holdout, 0) < 0) { reason = "study_forecast.split.allocate"; return(false); } if(use_all_history) { for(int i = 5; i + 12 < ArraySize(bars); i++) { if(bars[i].server_open < (long)range_start || bars[i].server_open >= (long)range_end || bars[i + 12].server_open >= (long)range_end) continue; const int n = ArraySize(fit); if(ArrayResize(fit, n + 1) != n + 1) { reason = "study_forecast.split.fit_allocate"; return(false); } fit[n] = i; } if(ArraySize(fit) == 0) { reason = "study_forecast.split.all_history_empty"; return(false); } reason = ""; return(true); } for(int i = 5; i + 12 < ArraySize(bars) && (use_all_history ? (bars[i].server_open >= (long)range_start && bars[i].server_open < (long)range_end && bars[i + 12].server_open < (long)range_end) : true); i++) { const long decision = bars[i].server_open; const long successor = bars[i + 12].server_open; if(decision >= fit_from && decision < fit_to && successor <= fit_to) { const int n = ArraySize(fit); if(ArrayResize(fit, n + 1) != n + 1) { reason = "study_forecast.split.fit_allocate"; return(false); } fit[n] = i; } if(decision >= fit_to && decision < holdout_to && successor <= holdout_to) { //--- A successor index is used only after the raw reader verified every row link. const int n = ArraySize(holdout); if(ArrayResize(holdout, n + 1) != n + 1) { reason = "study_forecast.split.holdout_allocate"; return(false); } holdout[n] = i; } } if(ArraySize(fit) == 0 || ArraySize(holdout) == 0) { reason = "study_forecast.split.empty"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Aggregate one H1 source pass through a reusable daily tick | //| buffer | //+------------------------------------------------------------------+ bool DPCCCaptureHistoricalPriceTicks(const string symbol, const ENUM_SYMBOL_CHART_MODE basis, const MqlRates &rates[], const int rows, SDPCCHistoricalPriceTicks &prices[], string &reason) { if(rows <= 0 || ArraySize(rates) != rows + 1 || (basis != SYMBOL_CHART_MODE_BID && basis != SYMBOL_CHART_MODE_LAST) || rates[0].time <= 0 || (long)rates[rows].time > LONG_MAX / 1000 || ArrayResize(prices, rows) != rows) {reason = "source_capture.tick_arguments"; return(false);} const long period_seconds = PeriodSeconds(PERIOD_H1); if(period_seconds <= 0 || (long)rates[rows - 1].time > LONG_MAX / 1000 - period_seconds) {reason = "source_capture.tick_period"; return(false);} bool invalid[]; if(ArrayResize(invalid, rows) != rows) {reason = "source_capture.tick_capacity"; return(false);} ArrayInitialize(invalid, false); for(int row = 0; row < rows; row++) ZeroMemory(prices[row]); MqlTick ticks[]; const long finish = (long)rates[rows].time * 1000; const long chunk_msc = 24 * 60 * 60 * 1000; const uint known_flags = TICK_FLAG_BID | TICK_FLAG_ASK | TICK_FLAG_LAST | TICK_FLAG_VOLUME | TICK_FLAG_BUY | TICK_FLAG_SELL; int first_row = 0; for(long from = (long)rates[0].time * 1000; from < finish;) { if(IsStopped()) {ArrayFree(ticks); reason = "source_capture.tick_stopped"; return(false);} const long until = (finish - from > chunk_msc ? from + chunk_msc - 1 : finish - 1); while(first_row < rows && (long)rates[first_row + 1].time * 1000 <= from) first_row++; ResetLastError(); const int copied = CopyTicksRange(symbol, ticks, COPY_TICKS_ALL, (ulong)from, (ulong)until); const int copy_error = GetLastError(); bool chunk_valid = (copied >= 0 && copy_error == 0 && copied <= ArraySize(ticks)); long previous = from; for(int i = 0; chunk_valid && i < copied; i++) { if(ticks[i].time_msc < from || ticks[i].time_msc > until || ticks[i].time_msc < previous || (long)ticks[i].time != ticks[i].time_msc / 1000 || ticks[i].flags == 0 || (ticks[i].flags & ~known_flags) != 0) chunk_valid = false; previous = ticks[i].time_msc; } if(!chunk_valid) { for(int row = first_row; row < rows && (long)rates[row].time * 1000 <= until; row++) invalid[row] = true; } else { int row = first_row; for(int i = 0; i < copied && row < rows; i++) { const MqlTick observed = ticks[i]; while(row < rows && observed.time_msc >= (long)rates[row + 1].time * 1000) row++; if(row >= rows) break; if(observed.time_msc < (long)rates[row].time * 1000) continue; //--- A quote in an unrepresented H1 interval cannot extend its preceding candle. if(observed.time_msc >= ((long)rates[row].time + period_seconds) * 1000) continue; const double price = (basis == SYMBOL_CHART_MODE_BID ? observed.bid : observed.last); if(!MathIsValidNumber(price) || price <= 0) {invalid[row] = true; continue;} if(prices[row].count == LONG_MAX) {invalid[row] = true; continue;} if(prices[row].count == 0) { prices[row].time_msc = observed.time_msc; for(int c = 0; c < 4; c++) prices[row].ohlc[c] = price; } prices[row].ohlc[1] = MathMax(prices[row].ohlc[1], price); prices[row].ohlc[2] = MathMin(prices[row].ohlc[2], price); prices[row].ohlc[3] = price; prices[row].count++; } } from = until + 1; } ArrayFree(ticks); for(int row = 0; row < rows; row++) { if(invalid[row] || prices[row].count == 0) ZeroMemory(prices[row]); else prices[row].available = true; } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Admit complete price targets while retaining original bar | //| indices | //+------------------------------------------------------------------+ bool DPCCPriceTargetWindowAvailable(const SDPCCStudyForecastBar &bars[], const int decision, const int horizon, const ENUM_SYMBOL_CHART_MODE basis, const bool h1_ohlc_targets = false, const ENUM_TIMEFRAMES timeframe = PERIOD_H1) { const long period_seconds = PeriodSeconds(timeframe); if(decision < 1 || horizon <= 0 || decision > ArraySize(bars) - horizon || period_seconds <= 0) return(false); for(int h = 0; h < horizon; h++) { const SDPCCStudyForecastBar row = bars[decision + h]; if(row.chart_quote_mode != basis || row.server_open <= 0 || row.server_open > LONG_MAX / 1000) return(false); if(h1_ohlc_targets) { if(row.raw_version != 3 || !row.price_bar.available || row.price_bar.time_msc != row.server_open * 1000 || !MathIsValidNumber(row.ohlc[0]) || !MathIsValidNumber(row.ohlc[1]) || !MathIsValidNumber(row.ohlc[2]) || !MathIsValidNumber(row.ohlc[3]) || row.ohlc[0] <= 0 || row.ohlc[1] < MathMax(row.ohlc[0], row.ohlc[3]) || row.ohlc[2] > MathMin(row.ohlc[0], row.ohlc[3])) return(false); } else if(row.raw_version != 2 || !row.price_ticks.available || row.price_ticks.count <= 0 || row.server_open > LONG_MAX / 1000 - period_seconds || row.price_ticks.time_msc < row.server_open * 1000 || row.price_ticks.time_msc >= (row.server_open + period_seconds) * 1000) return(false); } return(true); } //+------------------------------------------------------------------+ //| Keep Market and Study on the same eligible historical split | //+------------------------------------------------------------------+ bool DPCCFilterStudyForecastPriceTargets(const SDPCCStudyForecastBar &bars[], const ENUM_SYMBOL_CHART_MODE basis, const bool h1_ohlc_targets, int &fit[], int &holdout[], string &reason, const ENUM_TIMEFRAMES timeframe = PERIOD_H1) { reason = ""; if(ArraySize(bars) == 0 || (basis != SYMBOL_CHART_MODE_BID && basis != SYMBOL_CHART_MODE_LAST)) { reason = "study_forecast.price_split.arguments"; return(false); } int eligible_fit = 0, eligible_holdout = 0; for(int i = 0; i < ArraySize(fit); i++) if(DPCCPriceTargetWindowAvailable(bars, fit[i], 12, basis, h1_ohlc_targets, timeframe)) fit[eligible_fit++] = fit[i]; for(int i = 0; i < ArraySize(holdout); i++) if(DPCCPriceTargetWindowAvailable(bars, holdout[i], 12, basis, h1_ohlc_targets, timeframe)) holdout[eligible_holdout++] = holdout[i]; if(ArrayResize(fit, eligible_fit) != eligible_fit || ArrayResize(holdout, eligible_holdout) != eligible_holdout) { reason = "study_forecast.price_split.capacity"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCCaptureStudyForecastRaw captures aligned closed native bars | //+------------------------------------------------------------------+ bool DPCCCaptureStudyForecastRaw(const string symbol, const ENUM_TIMEFRAMES timeframe, const datetime capture_from, const datetime capture_to, string &raw_path, string &raw_sha, uint &row_count, string &reason, const bool capture_price_ticks = false, const ENUM_APPLIED_PRICE rsi_price = PRICE_CLOSE, const ENUM_APPLIED_PRICE cci_price = PRICE_TYPICAL, const ENUM_APPLIED_PRICE macd_price = PRICE_CLOSE, const bool capture_h1_ohlc_targets = false, const bool common_files = false) { //--- Require a half-open server-time interval and a safe wire identity. raw_path = ""; raw_sha = ""; row_count = 0; reason = ""; const string server = AccountInfoString(ACCOUNT_SERVER); const int max_rows = (capture_h1_ohlc_targets ? DPCC_RAW_V3_MAX_ROWS : DPCC_RAW_LEGACY_MAX_ROWS); const long max_bytes = (capture_h1_ohlc_targets ? DPCC_RAW_V3_MAX_BYTES : DPCC_RAW_LEGACY_MAX_BYTES); if(capture_price_ticks && capture_h1_ohlc_targets) { reason = "source_capture.conflicting_target_sources"; return(false); } const string forbidden = "|\r\n\\/:*?\"<>"; if(symbol == "" || server == "" || capture_from <= 0 || capture_to <= capture_from || PeriodSeconds(timeframe) <= 0) { reason = "source_capture.arguments"; return(false); } if(!DPCCStudyForecastAppliedPriceValid((long)rsi_price) || !DPCCStudyForecastAppliedPriceValid((long)cci_price) || !DPCCStudyForecastAppliedPriceValid((long)macd_price)) { reason = "source_capture.applied_price"; return(false); } for(int i = 0; i < StringLen(forbidden); i++) if(StringFind(symbol, StringSubstr(forbidden, i, 1)) >= 0) { reason = "source_capture.symbol_encoding"; return(false); } if(StringFind(server, "|") >= 0 || StringFind(server, "\r") >= 0 || StringFind(server, "\n") >= 0) { reason = "source_capture.server_encoding"; return(false); } //--- Copy the requested rows and exactly one observed successor guard. MqlRates requested[], rates[]; ArraySetAsSeries(requested, false); ArraySetAsSeries(rates, false); const ulong history_started = GetTickCount64(); int count = -1; do { ResetLastError(); count = CopyRates(symbol, timeframe, capture_from, capture_to - 1, requested); if(count > 0 && SeriesInfoInteger(symbol, timeframe, SERIES_SYNCHRONIZED)) break; if(IsStopped()) break; Sleep(25); } while(GetTickCount64() - history_started < 3000); if(count <= 0 || count > max_rows || !SeriesInfoInteger(symbol, timeframe, SERIES_SYNCHRONIZED) || SeriesInfoInteger(symbol, timeframe, SERIES_FIRSTDATE) <= 0 || SeriesInfoInteger(symbol, timeframe, SERIES_FIRSTDATE) > (long)capture_from || Bars(symbol, timeframe, capture_from, capture_to - 1) != count) { reason = StringFormat("source_capture.history_incomplete:%d:%d", count, GetLastError()); return(false); } const datetime forming_open = iTime(symbol, timeframe, 0); const int first_shift = iBarShift(symbol, timeframe, requested[0].time, true); const int last_shift = iBarShift(symbol, timeframe, requested[count - 1].time, true); if(forming_open <= 0 || capture_to > forming_open || last_shift < 1 || first_shift - last_shift + 1 != count || CopyRates(symbol, timeframe, last_shift - 1, count + 1, rates) != count + 1) { reason = "source_capture.closed_range_or_successor"; return(false); } if(rates[count].time < capture_to) { reason = "source_capture.right_boundary_unproven"; return(false); } for(int i = 0; i < count; i++) if(rates[i].time != requested[i].time || rates[i].time < capture_from || rates[i].time >= capture_to || rates[i].time >= forming_open || rates[i + 1].time <= rates[i].time) { reason = StringFormat("source_capture.chronology:%d", i); return(false); } //--- Keep native indicator handles scoped to this acquisition. int handles[4]; ArrayInitialize(handles, INVALID_HANDLE); handles[0] = iRSI(symbol, timeframe, RSIPeriod, rsi_price); handles[1] = iCCI(symbol, timeframe, CCIPeriod, cci_price); handles[2] = iATR(symbol, timeframe, ATRPeriod); handles[3] = iMACD(symbol, timeframe, FastPeriod, SlowPeriod, SignalPeriod, macd_price); double rsi[], cci[], atr[], macd[], signal[]; bool captured = false; do { bool valid_handles = true; for(int i = 0; i < 4; i++) if(handles[i] == INVALID_HANDLE) valid_handles = false; if(!valid_handles) { reason = StringFormat("source_capture.indicator_handle:%d", GetLastError()); break; } const ulong indicators_started = GetTickCount64(); bool aligned = false; do { bool ready = true; for(int i = 0; i < 4; i++) if(BarsCalculated(handles[i]) <= first_shift) ready = false; if(ready) { const int nr = CopyBuffer(handles[0], 0, last_shift, count, rsi); const int nc = CopyBuffer(handles[1], 0, last_shift, count, cci); const int na = CopyBuffer(handles[2], 0, last_shift, count, atr); const int nm = CopyBuffer(handles[3], 0, last_shift, count, macd); const int ns = CopyBuffer(handles[3], 1, last_shift, count, signal); aligned = (nr == count && nc == count && na == count && nm == count && ns == count); } if(aligned || IsStopped()) break; Sleep(25); } while(GetTickCount64() - indicators_started < 3000); if(!aligned || iTime(symbol, timeframe, 0) != forming_open || iTime(symbol, timeframe, first_shift) != rates[0].time || iTime(symbol, timeframe, last_shift) != rates[count - 1].time || iTime(symbol, timeframe, last_shift - 1) != rates[count].time) { reason = "source_capture.indicator_incomplete_or_history_shift"; break; } //--- V2 retains compact tick targets under the same artifact SHA as the native bars. SDPCCHistoricalPriceTicks price_ticks[]; long quote_mode = -1; if(capture_h1_ohlc_targets && (!SymbolInfoInteger(symbol, SYMBOL_CHART_MODE, quote_mode) || (quote_mode != SYMBOL_CHART_MODE_BID && quote_mode != SYMBOL_CHART_MODE_LAST))) { reason = "source_capture.bar_ohlc_basis"; break; } if(capture_price_ticks && (timeframe != PERIOD_H1 || !SymbolInfoInteger(symbol, SYMBOL_CHART_MODE, quote_mode) || (quote_mode != SYMBOL_CHART_MODE_BID && quote_mode != SYMBOL_CHART_MODE_LAST) || !DPCCCaptureHistoricalPriceTicks(symbol, (ENUM_SYMBOL_CHART_MODE)quote_mode, rates, count, price_ticks, reason))) { if(reason == "") reason = "source_capture.tick_basis"; break; } const string raw_version = (capture_h1_ohlc_targets ? "DPCC_FRESH_MARKET_RAW_V3\n" : capture_price_ticks ? "DPCC_FRESH_MARKET_RAW_V2\n" : "DPCC_FRESH_MARKET_RAW_V1\n"); string payload = raw_version; payload += "SCOPE|ACTUAL_TERMINAL_CLOSED_HISTORY|NO_UTC_ADMISSION|NO_DATA_READINESS\n"; payload += "IDENTITY|" + server + "|" + symbol + "|" + EnumToString(timeframe); if(capture_price_ticks || capture_h1_ohlc_targets) payload += (quote_mode == SYMBOL_CHART_MODE_BID ? "|BID" : "|LAST"); payload += "\n"; payload += "WAIT_LIMIT_MS|3000\n"; payload += DPCCStudyForecastIndicatorParams(rsi_price, cci_price, macd_price) + "\n"; payload += "ACQUISITION_RAW|SERVER_TIME|" + (string)capture_from + "|" + (string)capture_to + "|GUARD|" + (string)rates[count].time + "\n"; bool valid_rows = true; for(int i = 0; i < count; i++) { const double values[5] = {rsi[i], cci[i], atr[i], macd[i], signal[i]}; for(int c = 0; c < 5; c++) if(!MathIsValidNumber(values[c]) || values[c] == EMPTY_VALUE) valid_rows = false; if(!valid_rows) { reason = StringFormat("source_capture.indicator_number:%d", i); break; } string row = StringFormat("ROW|%d|%s|%I64d", i, (string)rates[i].time, (long)rates[i + 1].time); row += "|" + DoubleToString(rates[i].open, 16) + "|" + DoubleToString(rates[i].high, 16) + "|" + DoubleToString(rates[i].low, 16) + "|" + DoubleToString(rates[i].close, 16); row += StringFormat("|%I64d|%d|%I64d", rates[i].tick_volume, rates[i].spread, rates[i].real_volume); for(int c = 0; c < 5; c++) row += "|" + DoubleToString(values[c], 16); if(capture_price_ticks) { row += "|TICKS_H1_V1|" + (string)(int)price_ticks[i].available + "|" + (string)price_ticks[i].time_msc + "|" + (string)price_ticks[i].count; for(int c = 0; c < 4; c++) row += "|" + DoubleToString(price_ticks[i].ohlc[c], 16); } else if(capture_h1_ohlc_targets) { if((long)rates[i].time <= 0 || (long)rates[i].time > LONG_MAX / 1000) { reason = StringFormat("source_capture.bar_ohlc_anchor:%d", i); valid_rows = false; break; } row += (timeframe == PERIOD_H1 ? "|H1_OHLC_V1|" : "|BAR_OHLC_V1|") + (string)((long)rates[i].time * 1000); } SDPCCClosedHistoryRow decoded; if(!DPCCDecodeClosedHistoryRow(row, i, decoded, reason)) { reason = StringFormat("source_capture.row:%d:", i) + reason; valid_rows = false; break; } payload += row + "\n"; } if(!valid_rows) break; payload += StringFormat("END|%d\n", count); //--- Persist a unique UTF-8 artifact and hash the exact read-back bytes. uchar bytes[]; const int encoded = StringToCharArray(payload, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(encoded <= 1 || (long)encoded - 1 > max_bytes || ArrayResize(bytes, encoded - 1) != encoded - 1) { reason = "source_capture.utf8"; break; } string path = ""; for(int attempt = 0; attempt < 16; attempt++) { path = StringFormat("DPCC_%s_%s_%I64u_%I64u_%d.dpcraw", symbol, EnumToString(timeframe), (ulong)TimeLocal(), GetMicrosecondCount(), attempt); if(!FileIsExist(path, common_files ? FILE_COMMON : 0)) break; path = ""; } if(path == "") { reason = "source_capture.unique_path"; break; } ResetLastError(); const int writer = FileOpen(path, FILE_WRITE | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(writer == INVALID_HANDLE) { reason = StringFormat("source_capture.file_open:%d", GetLastError()); break; } const uint written = FileWriteArray(writer, bytes, 0, ArraySize(bytes)); FileFlush(writer); FileClose(writer); if(written != (uint)ArraySize(bytes)) { reason = "source_capture.file_write"; break; } uchar persisted[]; string memory_sha = "", persisted_sha = ""; if(!DPCCSHA256Lower(bytes, memory_sha, reason) || !DPCCReadArtifactBytes(path, common_files, max_bytes, persisted, reason) || !DPCCSHA256Lower(persisted, persisted_sha, reason)) break; if(ArraySize(persisted) != ArraySize(bytes) || persisted_sha != memory_sha) { reason = "source_capture.persisted_hash"; break; } raw_path = path; raw_sha = persisted_sha; row_count = (uint)count; reason = ""; captured = true; } while(false); for(int i = 0; i < 4; i++) if(handles[i] != INVALID_HANDLE) IndicatorRelease(handles[i]); return(captured); } //+------------------------------------------------------------------+ //| Function DPCCStudyForecastAdd | //+------------------------------------------------------------------+ bool DPCCStudyForecastAdd(CArrayObj *list, CLayerDescription *description) { if(list && description && list.Add(description)) return(true); if(description) delete description; return(false); } //+------------------------------------------------------------------+ //| Function DPCCStudyForecastLayer | //+------------------------------------------------------------------+ CLayerDescription *DPCCStudyForecastLayer(const uint type, const uint count) { CLayerDescription *description = new CLayerDescription(); if(description && !DPCCSetLayerBaseDefaults(description, type, count, 100000)) { delete description; return(NULL); } return(description); } //+------------------------------------------------------------------+ //| Function DPCCCreateStudyForecastGraphs | //+------------------------------------------------------------------+ bool DPCCCreateStudyForecastGraphs(CNet &market_net, CNet &target_net, string &reason, const bool include_teacher = true, const bool price_head = false) { CArrayObj *market = new CArrayObj(); CArrayObj *target = (include_teacher ? new CArrayObj() : NULL); if(!market || (include_teacher && !target)) { if(market) delete market; if(target) delete target; reason = "study_forecast.graph_allocation"; return(false); } market.FreeMode(true); if(target) target.FreeMode(true); bool ok = true; CLayerDescription *d = DPCCStudyForecastLayer(defNeuronBaseOCL, 45); ok = DPCCStudyForecastAdd(market, d); if(ok) ok = DPCCStudyForecastAdd(market, DPCCStudyForecastLayer(defNeuronBatchNormOCL, 45)); if(ok) { d = DPCCStudyForecastLayer(defNeuronDropoutOCL, 45); if(d) d.probability = .1f; ok = DPCCStudyForecastAdd(market, d); } if(ok) { d = DPCCStudyForecastLayer(defNeuronCogDriverData, 9); if(d) { d.window = DPCC_ENCODER_INPUT_WINDOW; d.probability = 1; uint units[] = {DPCC_ENCODER_STATE_STACK, DPCC_ENCODER_STATE_STACK, 8}; if(ArrayCopy(d.units, units) != 3) { delete d; d = NULL; } } ok = DPCCStudyForecastAdd(market, d); } if(ok) { d = DPCCStudyForecastLayer(defNeuronCogDriverRankTCM, 16); if(d) { d.window = 85; d.variables = 9; uint units[] = {DPCC_ENCODER_STATE_STACK, 4}; if(ArrayCopy(d.units, units) != 2) { delete d; d = NULL; } } ok = DPCCStudyForecastAdd(market, d); } if(ok) { d = DPCCStudyForecastLayer(defNeuronOMPBOCL, 9); if(d) { d.window = 16; d.layers = 5; uint units[] = {256, 64}; if(ArrayCopy(d.units, units) != 2) { delete d; d = NULL; } } ok = DPCCStudyForecastAdd(market, d); } if(ok) { d = DPCCStudyForecastLayer(price_head ? defNeuronScenarioForecastPriceOCL : defNeuronScenarioForecast, 21); if(d) { d.variables = 5; d.window = 16; d.window_out = 12; d.layers = 9; } ok = DPCCStudyForecastAdd(market, d); } if(ok && price_head) { d = DPCCStudyForecastLayer(defNeuronScenarioDecoderOCL, 21 * 12 * 4); if(d) { d.window = 9 * 16; d.window_out = 64; d.layers = 2; d.step = 1; d.variables = 1; d.batch = 1; uint units[] = {21, 9, 12, 16, 64, defScenarioDecoderPriceABI}; if(ArrayCopy(d.units, units) != 6) { delete d; d = NULL; } } ok = DPCCStudyForecastAdd(market, d); } if(include_teacher) { if(ok) ok = DPCCStudyForecastAdd(target, DPCCStudyForecastLayer(defNeuronBaseOCL, 108)); if(ok) { d = DPCCStudyForecastLayer(defNeuronPeriodNorm, 1); if(d) { d.window = 12; d.variables = 9; } ok = DPCCStudyForecastAdd(target, d); } for(uint i = 0; i < 3 && ok; i++) { d = DPCCStudyForecastLayer(defNeuronConvOCL, 12); if(d) { d.window = (i == 0 ? 3 : (i == 1 ? 16 : 32)); d.step = (i == 0 ? 1 : d.window); d.window_out = (i == 1 ? 32 : 16); d.layers = 9; d.activation = (i < 2 ? GELU : None); } ok = DPCCStudyForecastAdd(target, d); } if(ok) { d = DPCCStudyForecastLayer(defNeuronPeriodNorm, 12); if(d) { d.window = 16; d.variables = 9; } ok = DPCCStudyForecastAdd(target, d); } } const bool created = ok && market.Total() == (price_head ? 8 : 7) && ((!include_teacher) || (target.Total() == 6 && target_net.Create(target))) && market_net.Create(market); delete market; delete target; if(!created || (include_teacher && !target_net.SetOpenCLChecked(market_net.GetOpenCL()))) { reason = "study_forecast.fresh_graph"; return(false); } CNeuronOMPBOCL *ompb = dynamic_cast(market_net.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(market_net.Layer(6)); if(!ompb || !forecast || !forecast.SetRecoveryAge(256) || !ompb.SetSourceAnchor(false) || !ompb.SetMode(OMPB_BYPASS) || !market_net.TrainMode(true) || (include_teacher && (!target_net.TrainMode(false) || !target_net.SetWeightsUpdate(false)))) { reason = "study_forecast.fresh_policy"; return(false); } ompb.TrainMode(false); if(price_head) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(forecast); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(market_net.Layer(7)); if(!price || price.Type() != defNeuronScenarioForecastPriceOCL || price.ABI() != defScenarioForecastPriceABI || price.Scenarios() != 21 || price.GetTopK() != 5 || price.Variables() != 9 || price.Horizon() != 12 || price.Dimension() != 16 || !decoder || decoder.Type() != defNeuronScenarioDecoderOCL || decoder.ABI() != defScenarioDecoderPriceABI || decoder.Scenarios() != 21 || decoder.Horizon() != 12 || decoder.DecoderEmbedding() != 64 || decoder.Neurons() != 21 * 12 * 4 || market_net.Layer(8) != NULL || !market_net.TrainMode(false) || !market_net.SetWeightsUpdate(false) || !price.SetCodebookUpdate(false) || !price.SetPricePhase(defForecastPriceVerify) || !decoder.SetPricePhase(defForecastPriceVerify)) { reason = "study_forecast.price_graph.geometry_or_phase"; return(false); } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Typed DatasetTimeIndex metadata and CPU integer bar storage | //+------------------------------------------------------------------+ struct SDPCCDatasetTimezoneMapping { string path; string sha256; string id; }; struct SDPCCDatasetTimeIndex { long schema_version; bool example_only; string time_index_id; string dataset_id; string data_source; SDPCCDatasetTimezoneMapping timezone_mapping; long bar_index[]; long start_utc[]; long end_utc[]; }; //+------------------------------------------------------------------+ //| DPCCValidateDatasetTimeIndex validates the supplied typed index | //+------------------------------------------------------------------+ bool DPCCValidateDatasetTimeIndex(const SDPCCDatasetTimeIndex &index, const string expected_dataset_id, string &reason) { //--- Validate metadata before indexing any bar array. reason = ""; if(index.schema_version != 1) reason = "schema_version"; else if(StringLen(index.time_index_id) == 0) reason = "time_index_id"; else if(StringLen(index.dataset_id) == 0) reason = "dataset_id"; else if(StringLen(index.data_source) == 0) reason = "data_source"; else if(StringLen(index.timezone_mapping.path) == 0) reason = "timezone_mapping.path"; else if(StringLen(index.timezone_mapping.id) == 0) reason = "timezone_mapping.id"; else if(StringLen(expected_dataset_id) == 0) reason = "expected_dataset_id"; else if(index.dataset_id != expected_dataset_id) reason = "dataset_id"; else if(StringLen(index.timezone_mapping.sha256) != 64) reason = "timezone_mapping.sha256"; if(StringLen(reason) != 0) return(false); //--- Require the exact lowercase ASCII hexadecimal SHA representation. for(int i = 0; i < 64; i++) { ushort code = StringGetCharacter(index.timezone_mapping.sha256, i); if(!((code >= '0' && code <= '9') || (code >= 'a' && code <= 'f'))) { reason = "timezone_mapping.sha256"; return(false); } } //--- ArraySize is already int; no bar or UTC value is narrowed. const int count = ArraySize(index.bar_index); if(count <= 0) reason = "bars"; else if(ArraySize(index.start_utc) != count) reason = "bars.start_utc.length"; else if(ArraySize(index.end_utc) != count) reason = "bars.end_utc.length"; if(StringLen(reason) != 0) return(false); //--- Compare 64-bit values directly; gaps and half-open adjacency are valid. for(int i = 0; i < count; i++) { if(index.bar_index[i] != (long)i) reason = StringFormat("bars[%d].bar_index", i); else if(index.start_utc[i] < 0) reason = StringFormat("bars[%d].start_utc", i); else if(index.end_utc[i] < 0 || index.end_utc[i] <= index.start_utc[i]) reason = StringFormat("bars[%d].end_utc", i); else if(i > 0 && index.start_utc[i] < index.end_utc[i - 1]) reason = StringFormat("bars[%d].start_utc", i); if(StringLen(reason) != 0) return(false); } //--- This verdict covers typed metadata only; example_only is preserved. return(true); } //+------------------------------------------------------------------+ //| Typed META-08 architecture metadata | //+------------------------------------------------------------------+ struct SDPCCDecoderArchitecture { string architecture; uint embedding; string activation; string backward_profile; }; struct SDPCCArchitectureContract { SDPCCDecoderArchitecture model_decoder; SDPCCDenoiserArchitecture model_denoiser; SDPCCParallelExecution model_parallel; SDPCCDecoderArchitecture training_decoder; SDPCCDenoiserArchitecture training_denoiser; SDPCCParallelExecution training_parallel; }; //+------------------------------------------------------------------+ //| DPCCArchitectureSHA256Encoding checks lowercase ASCII digest | //+------------------------------------------------------------------+ bool DPCCArchitectureSHA256Encoding(const string digest) { if(StringLen(digest) != 64) return(false); for(int i = 0; i < 64; i++) { ushort code = StringGetCharacter(digest, i); if(!((code >= '0' && code <= '9') || (code >= 'a' && code <= 'f'))) return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCValidateDecoderArchitecture validates typed fields | //+------------------------------------------------------------------+ bool DPCCValidateDecoderArchitecture(const SDPCCDecoderArchitecture &value, const string path, string &reason) { if(value.architecture != "SHARED_VD_E4") reason = path + ".architecture"; else if(value.embedding == 0) reason = path + ".embedding"; else if(value.activation != "GELU") reason = path + ".activation"; else if(value.backward_profile != "DPCC_EXACT_CHAIN_V1") reason = path + ".backward_profile"; if(StringLen(reason) != 0) return(false); return(true); } //+------------------------------------------------------------------+ //| DPCCValidateDenoiserArchitecture validates typed fields | //+------------------------------------------------------------------+ bool DPCCValidateDenoiserArchitecture(const SDPCCDenoiserArchitecture &value, const string path, string &reason) { if(value.architecture != "CONDITIONED_USHAPE_TENSOR_V1") reason = path + ".architecture"; else if(value.embedding == 0) reason = path + ".embedding"; else if(value.heads == 0) reason = path + ".heads"; else if(value.key_dimension != 4 && value.key_dimension != 8 && value.key_dimension != 16 && value.key_dimension != 32 && value.key_dimension != 64) reason = path + ".key_dimension"; else if(value.depth != 1) reason = path + ".depth"; else if(value.padding_mask != true) reason = path + ".padding_mask"; else if(value.graph_id != "DPCC_DENOISER_GRAPH_V1") reason = path + ".graph_id"; else if(value.normalization != "INPUT_AFFINE_ONLY") reason = path + ".normalization"; else if(value.feedforward_activation != "TANH") reason = path + ".feedforward_activation"; else if(value.temporal_mask != false) reason = path + ".temporal_mask"; else if(value.attention_backward != "DPCC_MHFlashAttentionGradient") reason = path + ".attention_backward"; else if(!DPCCArchitectureSHA256Encoding(value.graph_sha256)) reason = path + ".graph_sha256"; else if(value.backward_profile != "DPCC_EXACT_CHAIN_V1") reason = path + ".backward_profile"; if(StringLen(reason) != 0) return(false); //--- Widen both operands before multiplication to reject uint wrap aliases. const ulong width = (ulong)value.heads * (ulong)value.key_dimension; if((ulong)value.embedding != width) { reason = path + ".embedding"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCValidateParallelExecution validates typed fields | //+------------------------------------------------------------------+ bool DPCCValidateParallelExecution(const SDPCCParallelExecution &value, const string path, string &reason) { if(value.version != 1) reason = path + ".version"; else if(value.program != "DPCC_TENSOR_V1") reason = path + ".program"; else if(value.training_plan_batch == 0) reason = path + ".training_plan_batch"; else if(value.deployment_plan_batch == 0) reason = path + ".deployment_plan_batch"; else if(value.proposal_slots != 9) reason = path + ".proposal_slots"; else if(value.logical_heads == 0) reason = path + ".logical_heads"; else if(value.key_dimension == 0) reason = path + ".key_dimension"; else if(value.shared_projection_weights != true) reason = path + ".shared_projection_weights"; else if(value.self_attention_fold != "M_X_HEAD") reason = path + ".self_attention_fold"; else if(value.cross_attention_fold != "M_X_QUERY_SHARED_KV") reason = path + ".cross_attention_fold"; else if(value.activation_lifetime != "UNTIL_BACKWARD_AND_UPDATE") reason = path + ".activation_lifetime"; else if(value.binary_layer_version != 3) reason = path + ".binary_layer_version"; else if(value.legacy_slice_use != "REFERENCE_ONLY") reason = path + ".legacy_slice_use"; if(StringLen(reason) != 0) return(false); return(true); } //+------------------------------------------------------------------+ //| DPCCCompareDecoderArchitecture requires exact copies | //+------------------------------------------------------------------+ bool DPCCCompareDecoderArchitecture(const SDPCCDecoderArchitecture &model, const SDPCCDecoderArchitecture &training, const string path, string &reason) { if(training.architecture != model.architecture) reason = path + ".architecture"; else if(training.embedding != model.embedding) reason = path + ".embedding"; else if(training.activation != model.activation) reason = path + ".activation"; else if(training.backward_profile != model.backward_profile) reason = path + ".backward_profile"; return(StringLen(reason) == 0); } //+------------------------------------------------------------------+ //| DPCCCompareDenoiserArchitecture requires exact copies | //+------------------------------------------------------------------+ bool DPCCCompareDenoiserArchitecture(const SDPCCDenoiserArchitecture &model, const SDPCCDenoiserArchitecture &training, const string path, string &reason) { if(training.architecture != model.architecture) reason = path + ".architecture"; else if(training.embedding != model.embedding) reason = path + ".embedding"; else if(training.heads != model.heads) reason = path + ".heads"; else if(training.key_dimension != model.key_dimension) reason = path + ".key_dimension"; else if(training.depth != model.depth) reason = path + ".depth"; else if(training.padding_mask != model.padding_mask) reason = path + ".padding_mask"; else if(training.graph_id != model.graph_id) reason = path + ".graph_id"; else if(training.normalization != model.normalization) reason = path + ".normalization"; else if(training.feedforward_activation != model.feedforward_activation) reason = path + ".feedforward_activation"; else if(training.temporal_mask != model.temporal_mask) reason = path + ".temporal_mask"; else if(training.attention_backward != model.attention_backward) reason = path + ".attention_backward"; else if(training.graph_sha256 != model.graph_sha256) reason = path + ".graph_sha256"; else if(training.backward_profile != model.backward_profile) reason = path + ".backward_profile"; return(StringLen(reason) == 0); } //+------------------------------------------------------------------+ //| DPCCCompareParallelExecution requires exact copies | //+------------------------------------------------------------------+ bool DPCCCompareParallelExecution(const SDPCCParallelExecution &model, const SDPCCParallelExecution &training, const string path, string &reason) { if(training.version != model.version) reason = path + ".version"; else if(training.program != model.program) reason = path + ".program"; else if(training.training_plan_batch != model.training_plan_batch) reason = path + ".training_plan_batch"; else if(training.deployment_plan_batch != model.deployment_plan_batch) reason = path + ".deployment_plan_batch"; else if(training.proposal_slots != model.proposal_slots) reason = path + ".proposal_slots"; else if(training.logical_heads != model.logical_heads) reason = path + ".logical_heads"; else if(training.key_dimension != model.key_dimension) reason = path + ".key_dimension"; else if(training.shared_projection_weights != model.shared_projection_weights) reason = path + ".shared_projection_weights"; else if(training.self_attention_fold != model.self_attention_fold) reason = path + ".self_attention_fold"; else if(training.cross_attention_fold != model.cross_attention_fold) reason = path + ".cross_attention_fold"; else if(training.activation_lifetime != model.activation_lifetime) reason = path + ".activation_lifetime"; else if(training.binary_layer_version != model.binary_layer_version) reason = path + ".binary_layer_version"; else if(training.legacy_slice_use != model.legacy_slice_use) reason = path + ".legacy_slice_use"; return(StringLen(reason) == 0); } //+------------------------------------------------------------------+ //| DPCCValidateArchitectureContract validates META-08 typed pair | //+------------------------------------------------------------------+ bool DPCCValidateArchitectureContract(const SDPCCArchitectureContract &contract, const string expected_graph_sha256, string &reason) { //--- Validate Model schema fields before comparing Training copies. reason = ""; if(!DPCCValidateDecoderArchitecture(contract.model_decoder, "model.composition.decoder", reason)) return(false); if(!DPCCValidateDenoiserArchitecture(contract.model_denoiser, "model.composition.denoiser", reason)) return(false); if(!DPCCValidateParallelExecution(contract.model_parallel, "model.parallel_execution", reason)) return(false); //--- Equal architecture copies inherit every validated Model constant and dimension. if(!DPCCCompareDecoderArchitecture(contract.model_decoder, contract.training_decoder, "training.decoder.architecture", reason)) return(false); if(!DPCCCompareDenoiserArchitecture(contract.model_denoiser, contract.training_denoiser, "training.denoiser.architecture", reason)) return(false); if(!DPCCValidateParallelExecution(contract.training_parallel, "training.parallel_execution", reason)) return(false); if(!DPCCCompareParallelExecution(contract.model_parallel, contract.training_parallel, "training.parallel_execution", reason)) return(false); //--- Bind logical heads and key width to the shared parallel program. if(contract.model_denoiser.heads != contract.model_parallel.logical_heads) reason = "model.parallel_execution.logical_heads"; else if(contract.model_denoiser.key_dimension != contract.model_parallel.key_dimension) reason = "model.parallel_execution.key_dimension"; if(StringLen(reason) != 0) return(false); //--- The caller resolves the graph; this gate compares only its typed digest. if(!DPCCArchitectureSHA256Encoding(expected_graph_sha256)) reason = "expected_graph_sha256"; else if(contract.model_denoiser.graph_sha256 != expected_graph_sha256) reason = "model.composition.denoiser.graph_sha256"; return(StringLen(reason) == 0); } //+------------------------------------------------------------------+ //| DPCCBuildCosineDiffusionSchedule builds canonical host arrays | //+------------------------------------------------------------------+ bool DPCCBuildCosineDiffusionSchedule(const uint J, const double s, const double beta_cap, double &beta[], double &alpha_bar[], double &sigma[], string &reason) { //--- Empty any prior caller data before validating this request. ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = ""; //--- Validate inputs and the dynamic-array index range before arithmetic/allocation. if(J == 0 || J >= 2147483647) reason = "J"; else if(!MathIsValidNumber(s) || s <= 0.0) reason = "s"; else if(!MathIsValidNumber(beta_cap) || beta_cap <= 0.0 || beta_cap >= 1.0) reason = "beta_cap"; if(StringLen(reason) != 0) return(false); const int count = (int)J + 1; if(ArrayResize(beta, count) != count || ArrayResize(alpha_bar, count) != count || ArrayResize(sigma, count) != count) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "allocation"; return(false); } beta[0] = 0.0; alpha_bar[0] = 1.0; sigma[0] = 0.0; //--- The normalized cosine cumulative base determines each capped beta. const double pi = 3.14159265358979323846; const double denominator = 1.0 + s; if(!MathIsValidNumber(denominator) || denominator <= 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } double previous_base = 0.0; double previous_alpha_bar = 1.0; for(int j = 1; j < count; j++) { const double previous_fraction = (((double)(j - 1) / (double)J) + s) / denominator; const double current_fraction = (((double)j / (double)J) + s) / denominator; const double previous_cosine = MathCos(previous_fraction * pi * 0.5); const double current_cosine = MathCos(current_fraction * pi * 0.5); const double base_previous = previous_cosine * previous_cosine; const double base_current = current_cosine * current_cosine; if(!MathIsValidNumber(base_previous) || !MathIsValidNumber(base_current) || base_previous <= 0.0 || base_current < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } //--- Use f(0) on the first interval and the previous cosine base thereafter. if(j == 1) previous_base = base_previous; double beta_value = 1.0 - base_current / previous_base; if(!MathIsValidNumber(beta_value) || beta_value < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } if(beta_value > beta_cap) beta_value = beta_cap; const double current_alpha_bar = previous_alpha_bar * (1.0 - beta_value); if(!MathIsValidNumber(current_alpha_bar) || current_alpha_bar < 0.0 || current_alpha_bar > 1.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } double sigma_value = 0.0; if(j > 1) { const double posterior_denominator = 1.0 - current_alpha_bar; const double posterior_numerator = beta_value * (1.0 - previous_alpha_bar); if(!MathIsValidNumber(posterior_denominator) || !MathIsValidNumber(posterior_numerator) || posterior_denominator <= 0.0 || posterior_numerator < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } const double variance = posterior_numerator / posterior_denominator; if(!MathIsValidNumber(variance) || variance < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } sigma_value = MathSqrt(variance); } if(!MathIsValidNumber(sigma_value) || sigma_value < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } beta[j] = beta_value; alpha_bar[j] = current_alpha_bar; sigma[j] = sigma_value; previous_base = base_current; previous_alpha_bar = current_alpha_bar; } //--- Guard the complete result, including values written by recurrence. for(int i = 0; i < count; i++) { if(!MathIsValidNumber(beta[i]) || !MathIsValidNumber(alpha_bar[i]) || !MathIsValidNumber(sigma[i]) || sigma[i] < 0.0) { ArrayResize(beta, 0); ArrayResize(alpha_bar, 0); ArrayResize(sigma, 0); reason = "numeric"; return(false); } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Host-side typed source for the CTX-03 Global[45] mapping | //+------------------------------------------------------------------+ struct SDPCCGlobalContextSource { float account_features[]; double limits[]; double volume_scale; double price_scale; double money_scale; double point; double current_bid; double current_ask; double current_last; double tick_size; double volume_min; double volume_step; double initial_margin_per_lot; double margin_currency_rate; double tick_value_loss; double price_close_mean; double price_close_std; string chart_price_basis; int trade_mode; }; //+------------------------------------------------------------------+ //| DPCCBuildGlobalContext maps typed inputs to exact CTX-03 slots | //+------------------------------------------------------------------+ bool DPCCBuildGlobalContext(const SDPCCGlobalContextSource &source, float &global[], string &reason) { //--- Clear stale output before validating any source field. ArrayResize(global, 0); reason = ""; //--- Validate ABI arrays and every value before reading indexed fields. if(ArraySize(source.account_features) != 13) reason = "account_features.length"; else if(ArraySize(source.limits) != 24) reason = "limits.length"; if(StringLen(reason) != 0) return(false); for(int i = 0; i < 13; i++) { if(!MathIsValidNumber((double)source.account_features[i])) { reason = StringFormat("account_features[%d]", i); return(false); } } for(int i = 0; i < 24; i++) { if(!MathIsValidNumber(source.limits[i])) { reason = StringFormat("limits[%d]", i); return(false); } } //--- Validate scalar finiteness before positivity or transform checks. if(!MathIsValidNumber(source.volume_scale)) reason = "volume_scale"; else if(!MathIsValidNumber(source.price_scale)) reason = "price_scale"; else if(!MathIsValidNumber(source.money_scale)) reason = "money_scale"; else if(!MathIsValidNumber(source.point)) reason = "point"; else if(!MathIsValidNumber(source.current_bid)) reason = "current_bid"; else if(!MathIsValidNumber(source.current_ask)) reason = "current_ask"; else if(!MathIsValidNumber(source.current_last)) reason = "current_last"; else if(!MathIsValidNumber(source.tick_size)) reason = "tick_size"; else if(!MathIsValidNumber(source.volume_min)) reason = "volume_min"; else if(!MathIsValidNumber(source.volume_step)) reason = "volume_step"; else if(!MathIsValidNumber(source.initial_margin_per_lot)) reason = "initial_margin_per_lot"; else if(!MathIsValidNumber(source.margin_currency_rate)) reason = "margin_currency_rate"; else if(!MathIsValidNumber(source.tick_value_loss)) reason = "tick_value_loss"; else if(!MathIsValidNumber(source.price_close_mean)) reason = "price_close_mean"; else if(!MathIsValidNumber(source.price_close_std)) reason = "price_close_std"; if(StringLen(reason) != 0) return(false); //--- Enforce fixed scales, protocol values, and the quote contract. if(source.volume_scale <= 0.0) reason = "volume_scale"; else if(source.price_scale <= 0.0) reason = "price_scale"; else if(source.money_scale <= 0.0) reason = "money_scale"; else if(source.point <= 0.0) reason = "point"; else if(source.price_close_std <= 0.0) reason = "price_close_std"; else if(source.tick_size <= 0.0) reason = "tick_size"; else if(source.volume_step <= 0.0) reason = "volume_step"; else if(source.chart_price_basis != "BID" && source.chart_price_basis != "LAST") reason = "chart_price_basis"; else if(source.trade_mode < 0 || source.trade_mode > 4) reason = "trade_mode"; else if(source.current_ask < source.current_bid) reason = "current_ask"; else if(source.limits[20] < 0.0 || source.limits[20] > 511.0 || MathFloor(source.limits[20]) != source.limits[20]) reason = "limits[20]"; else if((((int)source.limits[20]) & (~511)) != 0) reason = "limits[20]"; else if(source.limits[22] != 2.0) reason = "limits[22]"; else if(source.limits[23] != 0.0) reason = "limits[23]"; else if(source.money_scale != source.limits[21]) reason = "money_scale"; if(StringLen(reason) != 0) return(false); //--- Build all 45 values in double precision from the CTX-03 table. double values[45]; for(int i = 0; i < 13; i++) values[i] = (double)source.account_features[i]; const int active_mask = (int)source.limits[20]; const double mask_bit[9] = {1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0}; for(int i = 0; i <= 10; i++) { int bit = i; if(i >= 7 && i <= 8) bit = 7; else if(i >= 9) bit = 8; const bool enabled = (((active_mask & (int)mask_bit[bit]) != 0)); values[13 + i] = (enabled ? source.limits[i] : 0.0); } values[13] /= source.volume_scale; values[14] /= source.money_scale; values[15] = (active_mask & 4) != 0 ? source.limits[2] : 0.0; values[16] /= source.money_scale; values[17] /= source.money_scale; values[18] /= source.money_scale; values[19] /= source.money_scale; values[20] = values[20] * source.point / source.price_scale; values[21] = (((active_mask & 128) != 0) ? source.limits[8] : 0.0) * source.point / source.price_scale; values[22] = values[22] * source.point / source.price_scale; values[23] = (((active_mask & 256) != 0) ? source.limits[10] : 0.0) * source.point / source.price_scale; values[24] = source.limits[11] / source.money_scale; values[25] = source.limits[12] / source.money_scale; values[26] = source.limits[13] * source.point / source.price_scale; values[27] = source.limits[14] / source.volume_scale; values[28] = source.limits[15] / source.price_scale; values[29] = source.limits[16] / source.money_scale; values[30] = source.limits[17]; values[31] = source.limits[18]; values[32] = source.limits[19] / source.price_scale; values[33] = source.limits[20] / 511.0; values[34] = source.limits[21] / source.money_scale; values[35] = source.limits[22] / 2.0; values[36] = 0.0; values[37] = (source.current_ask - source.current_bid) / source.price_scale; values[38] = source.tick_size / source.price_scale; values[39] = source.volume_min / source.volume_scale; values[40] = source.volume_step / source.volume_scale; values[41] = source.initial_margin_per_lot * source.margin_currency_rate * source.volume_scale / source.money_scale; values[42] = source.tick_value_loss * source.volume_scale / source.money_scale; const double native_quote = (source.chart_price_basis == "BID" ? source.current_bid : source.current_last); values[43] = (native_quote - source.price_close_mean) / source.price_close_std; values[44] = (double)source.trade_mode / 4.0; //--- Reject non-finite double results and float32 overflow before success. for(int i = 0; i < 45; i++) { if(!MathIsValidNumber(values[i])) { reason = StringFormat("global[%d]", i); return(false); } if(i >= 13) { const float converted = (float)values[i]; if(!MathIsValidNumber((double)converted)) { reason = StringFormat("global[%d]", i); return(false); } } } //--- Publish the complete float32 vector only after every slot is valid. if(ArrayResize(global, 45) != 45) { ArrayResize(global, 0); reason = "global.allocation"; return(false); } for(int i = 0; i < 13; i++) global[i] = source.account_features[i]; for(int i = 13; i < 45; i++) global[i] = (float)values[i]; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Packs the causal decoder section from one admitted context | //+------------------------------------------------------------------+ bool DPCCPackDecoderInput(const SDPCCShape &shape, const float &context_without_noise[], float &causal_values[], string &reason) { ArrayResize(causal_values, 0); reason = ""; SDPCCLayout context_layout, rollout_layout; uint tokens = 0; if(!DPCCCompositeLayouts(shape, context_layout, rollout_layout, tokens) || context_layout.offsets[10] > (ulong)INT_MAX || context_layout.lengths[0] > (ulong)INT_MAX || ArraySize(context_without_noise) != (int)context_layout.offsets[10]) { reason = "decoder.causal_values.context_shape"; return(false); } const int count = (int)context_layout.lengths[0]; for(int i = 0; i < count; i++) if(!MathIsValidNumber((double)context_without_noise[i])) { reason = "decoder.causal_values.nonfinite"; return(false); } if(ArrayResize(causal_values, count) != count) { ArrayResize(causal_values, 0); reason = "decoder.causal_values.capacity"; return(false); } for(int i = 0; i < count; i++) causal_values[i] = context_without_noise[i]; return(true); } //+------------------------------------------------------------------+ //| Exact float bits for comparing observed-path context slices | //+------------------------------------------------------------------+ //| DPCCBuildHistoricalRolloutContext joins one observed H1 path | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalRolloutContext(const SDPCCShape &source_shape, const float &context_without_noise[], const float &target_native_ohlc[], const long decision_server_time, const long decision_ordinal, const long &target_open_server[], const long &target_close_server[], const long &target_ordinals[], SDPCCShape &rollout_shape, float &rollout_context[], string &reason) { //--- Keep both outputs unpublished until every source field passes. ZeroMemory(rollout_shape); ArrayResize(rollout_context, 0); reason = ""; SDPCCLayout source_layout, unused_rollout, one_path_layout; uint tokens = 0; SDPCCShape candidate_shape = source_shape; candidate_shape.K = 1; candidate_shape.M = 1; candidate_shape.Depth = 0; if(!DPCCCompositeLayouts(source_shape, source_layout, unused_rollout, tokens) || !DPCCBuildRolloutLayout(candidate_shape, one_path_layout) || source_layout.offsets[10] > (ulong)INT_MAX || one_path_layout.total > (ulong)INT_MAX || ArraySize(context_without_noise) != (int)source_layout.offsets[10] || ArraySize(target_native_ohlc) != (int)(source_shape.H * 4) || ArraySize(target_open_server) != (int)source_shape.H || ArraySize(target_close_server) != (int)source_shape.H || ArraySize(target_ordinals) != (int)source_shape.H) { reason = "historical_rollout.shape"; return(false); } //--- Join the decision to exact source ordinals and ordered bar intervals. if(decision_server_time <= 0 || decision_ordinal < 0 || decision_ordinal > LONG_MAX - ((long)source_shape.H - 1) || target_open_server[0] != decision_server_time) { reason = "historical_rollout.time_join"; return(false); } for(uint h = 0; h < source_shape.H; h++) if(target_ordinals[h] != decision_ordinal + (long)h || target_open_server[h] <= 0 || target_close_server[h] <= target_open_server[h] || (h > 0 && target_open_server[h] < target_close_server[h - 1])) { reason = "historical_rollout.time_join"; return(false); } //--- Validate native observations and every retained context value. for(int i = 0; i < ArraySize(context_without_noise); i++) if(!MathIsValidNumber((double)context_without_noise[i])) { reason = "historical_rollout.context_numeric"; return(false); } for(uint h = 0; h < source_shape.H; h++) { const int base = (int)h * 4; const float open = target_native_ohlc[base]; const float high = target_native_ohlc[base + 1]; const float low = target_native_ohlc[base + 2]; const float close = target_native_ohlc[base + 3]; if(!MathIsValidNumber((double)open) || !MathIsValidNumber((double)high) || !MathIsValidNumber((double)low) || !MathIsValidNumber((double)close) || open <= 0 || high < open || high < close || high < low || low > open || low > close) { reason = "historical_rollout.ohlc"; return(false); } } //--- Execution and calendar are K-indexed; accept only a common slice. const uint indexed_sections[] = {7, 9}; for(int section_index = 0; section_index < 2; section_index++) { const uint section = indexed_sections[section_index]; const int stride = (section == 7 ? 24 : (int)source_shape.E * 6); for(uint k = 1; k < source_shape.K; k++) for(uint h = 0; h < source_shape.H; h++) for(int i = 0; i < stride; i++) { UDPCCObservedFloatBits first, other; first.value = context_without_noise[(int)source_layout.offsets[section] + (int)h * stride + i]; other.value = context_without_noise[(int)source_layout.offsets[section] + ((int)k * (int)source_shape.H + ( int)h) * stride + i]; if(first.bits != other.bits) { reason = "historical_rollout.scenario_context"; return(false); } } } //--- Build the complete one-path vector locally, then publish it. float candidate[]; if(ArrayResize(candidate, (int)one_path_layout.total) != (int)one_path_layout.total) { reason = "historical_rollout.capacity"; return(false); } ArrayInitialize(candidate, 0.0f); for(int i = 0; i < ArraySize(target_native_ohlc); i++) candidate[(int)one_path_layout.offsets[0] + i] = target_native_ohlc[i]; for(uint section = 1; section <= 3; section++) for(int i = 0; i < (int)one_path_layout.lengths[section]; i++) candidate[(int)one_path_layout.offsets[section] + i] = context_without_noise[(int)source_layout.offsets[section + 3] + i]; const uint from_sections[] = {7, 8, 9}; for(uint section = 4; section <= 6; section++) for(int i = 0; i < (int)one_path_layout.lengths[section]; i++) candidate[(int)one_path_layout.offsets[section] + i] = context_without_noise[(int)source_layout.offsets[from_sections[section - 4]] + i]; candidate[(int)one_path_layout.offsets[7]] = 1.0f; if(ArrayResize(rollout_context, (int)one_path_layout.total) != (int)one_path_layout.total) { ArrayResize(rollout_context, 0); reason = "historical_rollout.capacity"; return(false); } for(int i = 0; i < ArraySize(candidate); i++) rollout_context[i] = candidate[i]; rollout_shape = candidate_shape; return(true); } //+------------------------------------------------------------------+ //| Packs one admitted bar and its matching native context sections | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalRolloutStepContext(const SDPCCShape &source_shape, const float &source_context[], const uint step, SDPCCShape &step_shape, float &step_context[], string &reason) { ZeroMemory(step_shape); ArrayResize(step_context, 0); reason = ""; SDPCCLayout source_layout, step_layout; SDPCCShape candidate_shape = source_shape; candidate_shape.H = 1; candidate_shape.M = 1; candidate_shape.Depth = 0; if(source_shape.M != 1 || step >= source_shape.H || !DPCCBuildRolloutLayout(source_shape, source_layout) || !DPCCBuildRolloutLayout(candidate_shape, step_layout) || source_layout.total > (ulong)INT_MAX || step_layout.total > (ulong)INT_MAX || ArraySize(source_context) != (int)source_layout.total) { reason = "historical_rollout_step.shape"; return(false); } for(int i = 0; i < ArraySize(source_context); i++) if(!MathIsValidNumber((double)source_context[i])) { reason = "historical_rollout_step.context_numeric"; return(false); } float candidate[]; if(ArrayResize(candidate, (int)step_layout.total) != (int)step_layout.total) { reason = "historical_rollout_step.capacity"; return(false); } ArrayInitialize(candidate, 0.0f); for(uint k = 0; k < source_shape.K; k++) for(uint field = 0; field < 4; field++) candidate[(int)step_layout.offsets[0] + (int)k * 4 + (int)field] = source_context[(int)source_layout.offsets[0] + ((int)k * (int)source_shape.H + (int)step) * 4 + (int)field]; for(uint section = 1; section <= 3; section++) for(int i = 0; i < (int)step_layout.lengths[section]; i++) candidate[(int)step_layout.offsets[section] + i] = source_context[(int)source_layout.offsets[section] + i]; for(uint k = 0; k < source_shape.K; k++) for(uint field = 0; field < 24; field++) candidate[(int)step_layout.offsets[4] + (int)k * 24 + (int)field] = source_context[(int)source_layout.offsets[4] + ((int)k * (int)source_shape.H + (int)step) * 24 + ( int)field]; candidate[(int)step_layout.offsets[5]] = source_context[(int)source_layout.offsets[5] + (int)step]; for(uint k = 0; k < source_shape.K; k++) for(uint field = 0; field < source_shape.E * 6; field++) candidate[(int)step_layout.offsets[6] + (int)k * (int)source_shape.E * 6 + (int)field] = source_context[(int)source_layout.offsets[6] + ((int)k * (int)source_shape.H + (int)step) * (int)source_shape.E * 6 + (int)field]; candidate[(int)step_layout.offsets[7]] = source_context[(int)source_layout.offsets[7]]; if(ArrayResize(step_context, (int)step_layout.total) != (int)step_layout.total) { ArrayResize(step_context, 0); reason = "historical_rollout_step.capacity"; return(false); } for(int i = 0; i < ArraySize(candidate); i++) step_context[i] = candidate[i]; step_shape = candidate_shape; return(true); } //+------------------------------------------------------------------+ //| Applies the host training floor without changing the native flag | //+------------------------------------------------------------------+ bool DPCCTrainingFloorReached(const double balance, const double equity) { if(!MathIsValidNumber(balance) || !MathIsValidNumber(equity)) return(true); return(balance < 50.0 || equity < 50.0); } //+------------------------------------------------------------------+ //| Canonical native policy layout for one decoder/account state | //+------------------------------------------------------------------+ struct SDPCCPolicyLayout { uint scenario_count; uint account_offset; uint plan_count; uint actor_count; uint critic_count; }; //+------------------------------------------------------------------+ //| DPCCPolicyLayout checks KH4, Account13 and physical H6 | //| capacities | //+------------------------------------------------------------------+ bool DPCCPolicyLayout(const SDPCCShape &shape, SDPCCPolicyLayout &layout, string &reason) { reason = ""; if(shape.K == 0 || shape.H == 0 || shape.NActions != 6 || shape.StateSize != 13) { reason = "policy.shape"; return(false); } const ulong limit = ulong(UINT_MAX) / sizeof(float); const ulong scenarios = (ulong)shape.K * shape.H; if(scenarios > (limit - 13) / 4 || (ulong)shape.H > limit / 6) { reason = "policy.capacity"; return(false); } const ulong actor = scenarios * 4 + 13; const ulong plan = (ulong)shape.H * 6; if(actor + plan > limit || (actor + plan + 1) * 32 > limit || (plan + 1) * 33 > limit) { reason = "policy.dense_capacity"; return(false); } SDPCCPolicyLayout candidate; candidate.scenario_count = (uint)(scenarios * 4); candidate.account_offset = candidate.scenario_count; candidate.plan_count = (uint)plan; candidate.actor_count = (uint)actor; candidate.critic_count = (uint)(actor + shape.NActions); layout = candidate; return(true); } //+------------------------------------------------------------------+ //| DPCCPackPolicyInput packs KH4, Account13 and first executed | //| action | //+------------------------------------------------------------------+ bool DPCCPackPolicyInput(const SDPCCShape &shape, CBufferFloat *scenarios, const float &account13[], CBufferFloat *physical_plan, CBufferFloat *destination, string &reason) { //--- Packing is an explicit host boundary for CNet's ordinary input API. SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason)) return(false); if(CheckPointer(scenarios) == POINTER_INVALID || CheckPointer(destination) == POINTER_INVALID || destination == scenarios || destination == physical_plan || destination.GetIndex() >= 0 || scenarios.Total() != (int)layout.scenario_count || ArraySize(account13) != 13) { reason = "policy.pack_input"; return(false); } const bool critic = (physical_plan != NULL); if(critic && (CheckPointer(physical_plan) == POINTER_INVALID || physical_plan.Total() != (int)layout.plan_count)) { reason = "policy.pack_plan"; return(false); } if(scenarios.GetIndex() >= 0 && !scenarios.BufferRead()) { reason = "policy.pack_scenario_read"; return(false); } if(critic && physical_plan.GetIndex() >= 0 && !physical_plan.BufferRead()) { reason = "policy.pack_plan_read"; return(false); } //--- Decoder index ((k*H+h)*4+c); plan index (h*6+a) is unchanged. const uint count = (critic ? layout.critic_count : layout.actor_count); float packed[]; if(ArrayResize(packed, (int)count) != (int)count) { reason = "policy.pack_capacity"; return(false); } for(uint i = 0; i < count; i++) { const float value = (i < layout.scenario_count ? scenarios.At((int)i) : (i < layout.actor_count ? account13[i - layout.account_offset] : physical_plan.At((int)(i - layout.actor_count)))); if(!MathIsValidNumber(value)) { reason = StringFormat("policy.pack_nonfinite.%u", i); return(false); } packed[i] = value; } if(!destination.AssignArray(packed)) { reason = "policy.pack_publish"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCPolicyNetPreflight checks resident shapes without allocation | //+------------------------------------------------------------------+ bool DPCCPolicyNetPreflight(CNet &net, const SDPCCShape &shape, const bool critic, string &reason) { SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason)) return(false); const uint types[] = {defNeuronBaseOCL, defNeuronConvOCL, (critic ? defNeuronBaseOCL : defNeuronConvOCL), defNeuronACSRM }; const uint counts[] = {(critic ? layout.critic_count : layout.actor_count), 32, (critic ? 32 : layout.plan_count), 1 }; const int layers = (critic ? 4 : 3); if(!net.IsOpenCLTransferReady() || CheckPointer(net.GetOpenCL()) == POINTER_INVALID || net.Layer(layers) != NULL) { reason = "policy.net_context_or_layers"; return(false); } for(int i = 0; i < layers; i++) { const int expected_activation = (i == 1 ? GELU : (!critic && i == 2 ? SIGMOID : None)); CNeuronBaseOCL *layer = net.Layer(i); if(CheckPointer(layer) == POINTER_INVALID || layer.Type() != (int)types[i] || layer.GetOpenCL() != net.GetOpenCL() || layer.Activation() != expected_activation || layer.getOutput().Total() != (int)counts[i] || layer.getGradient().Total() != (int)counts[i]) { reason = StringFormat("policy.net_layer.%d", i); return(false); } } if(critic) { CNeuronACSRM *head = dynamic_cast(net.Layer(3)); CBufferFloat *quantiles = NULL, *probabilities = NULL; if(!head || !head.GetDistributionBuffers(quantiles, probabilities) || !quantiles || !probabilities || quantiles.Total() != 32 || probabilities.Total() != 32) { reason = "policy.net_distribution"; return(false); } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCValidatePolicyNet admits complete creation or restored | //| graphs | //+------------------------------------------------------------------+ bool DPCCValidatePolicyNet(CNet &net, const SDPCCShape &shape, const bool critic, string &reason) { if(!DPCCPolicyNetPreflight(net, shape, critic, reason)) return(false); SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason)) return(false); const uint counts[] = {(critic ? layout.critic_count : layout.actor_count), 32, layout.plan_count}; const int last = (critic ? 1 : 2); for(int i = 1; i <= last; i++) { CNeuronConvOCL *convolution = dynamic_cast(net.Layer(i)); if(CheckPointer(convolution) == POINTER_INVALID) { reason = StringFormat("policy.net_convolution_type.%d", i); return(false); } CLayerDescription *description = convolution.GetLayerInfo(); const uint width = counts[i - 1]; const bool valid = description != NULL && description.count == 1 && description.window == width && description.step == width && description.window_out == counts[i] && description.layers == 1 && description.batch == 1 && description.optimization == ADAM && convolution.GetWeightsConv() != NULL && convolution.GetWeightsConv().Total() == (int)((width + 1) * counts[i]); delete description; if(!valid) { reason = StringFormat("policy.net_convolution.%d", i); return(false); } } if(critic) { //--- Native FQF requires an explicit Base(32) predecessor and two dense matrices. CLayerDescription *bridge = net.Layer(2).GetLayerInfo(); const bool descriptor = bridge != NULL && bridge.type == defNeuronBaseOCL && bridge.count == 32 && bridge.activation == None && bridge.batch == 1 && bridge.optimization == ADAM; delete bridge; if(!descriptor) { reason = "policy.net_critic_bridge"; return(false); } for(int i = 1; i <= 2; i++) { CBufferFloat *weights = net.Layer(i).getWeights(); if(CheckPointer(weights) == POINTER_INVALID || weights.Total() != 33 * 32 || weights.GetIndex() < 0 || weights.GetOpenCL() != net.GetOpenCL()) { reason = StringFormat("policy.net_critic_dense_weights.%d", i); return(false); } } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCCreatePolicyNet creates a fresh Actor or independent Critic | //+------------------------------------------------------------------+ bool DPCCCreatePolicyNet(CNet &net, const SDPCCShape &shape, const bool critic, COpenCLMy *context, string &reason) { SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason)) return(false); if(net.Layer(0) != NULL || (context != NULL && CheckPointer(context) == POINTER_INVALID)) { reason = "policy.destination_or_context"; return(false); } CArrayObj descriptions; descriptions.FreeMode(true); const uint counts[] = {(critic ? layout.critic_count : layout.actor_count), 32, (critic ? 32 : layout.plan_count), 1 }; const int layers = (critic ? 4 : 3); for(int i = 0; i < layers; i++) { CLayerDescription *description = new CLayerDescription; if(!description) { reason = "policy.description_allocation"; return(false); } const uint type = (i == 0 || (critic && i == 2) ? defNeuronBaseOCL : (critic && i == 3 ? defNeuronACSRM : defNeuronConvOCL)); DPCCSetLayerBaseDefaults(description, type, (type == defNeuronConvOCL ? 1 : counts[i]), 1); if(type == defNeuronConvOCL) { description.window = counts[i - 1]; description.step = counts[i - 1]; description.window_out = counts[i]; description.layers = 1; description.activation = (i == 1 ? GELU : (!critic && i == 2 ? SIGMOID : None)); } else description.window_out = 32; if(!descriptions.Add(description)) { delete description; reason = "policy.description_add"; return(false); } } if(!net.Create(GetPointer(descriptions))) { net.Clear(); reason = "policy.net_create"; return(false); } if(context != NULL && !net.SetOpenCLChecked(context)) { net.Clear(); reason = "policy.context_bind"; return(false); } if(!DPCCValidatePolicyNet(net, shape, critic, reason)) { net.Clear(); return(false); } if(!net.TrainMode(true)) { net.Clear(); reason = "policy.train_mode"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCBuildOneBarReward materializes actual open-to-open equity | //| PnL | //+------------------------------------------------------------------+ bool DPCCBuildOneBarReward(const double previous_open_equity, const double next_open_equity, float &reward, string &reason) { reward = 0.0; reason = ""; //--- Validate both actual equity marks before subtraction. if(!MathIsValidNumber(previous_open_equity) || !MathIsValidNumber(next_open_equity)) {reason = "policy.reward_non_finite_input"; return(false);} //--- Reject a non-finite delta before publishing it to the TD owner. const double candidate = next_open_equity - previous_open_equity; if(!MathIsValidNumber(candidate)) {reason = "policy.reward_non_finite_result"; return(false);} const float candidate_reward = (float)candidate; if(!MathIsValidNumber((double)candidate_reward)) {reason = "policy.reward_non_finite_result"; return(false);} reward = candidate_reward; return(true); } //+------------------------------------------------------------------+ //| DPCCBuildOneBarTDTargets materializes separate scalar Q targets | //+------------------------------------------------------------------+ bool DPCCBuildOneBarTDTargets(const float reward, const float gamma, const float bootstrap_mask, const float z1_next, const float z2_next, float &y1, float &y2, string &reason) { //--- Clear outputs before validating any caller-provided values. y1 = 0.0f; y2 = 0.0f; reason = ""; //--- Validate the scalar transition inputs and explicit bootstrap contract. if(!MathIsValidNumber((double)reward) || !MathIsValidNumber((double)gamma) || !MathIsValidNumber((double)bootstrap_mask) || !MathIsValidNumber((double)z1_next) || !MathIsValidNumber((double)z2_next)) { reason = "policy.td_non_finite_input"; return(false); } if(gamma < 0.0f || gamma > 1.0f) { reason = "policy.td_gamma_range"; return(false); } if(bootstrap_mask != 0.0f && bootstrap_mask != 1.0f) { reason = "policy.td_bootstrap_mask_binary"; return(false); } //--- Compute in double, then reject values outside finite MQL float range. const double float_max = 3.4028234663852885981170418348451692544e+38; const double target1 = (double)reward + (double)gamma * (double)bootstrap_mask * (double)z1_next; const double target2 = (double)reward + (double)gamma * (double)bootstrap_mask * (double)z2_next; if(!MathIsValidNumber(target1) || !MathIsValidNumber(target2) || MathAbs(target1) > float_max || MathAbs(target2) > float_max) { reason = "policy.td_non_finite_output"; return(false); } const float candidate1 = (float)target1; const float candidate2 = (float)target2; if(!MathIsValidNumber((double)candidate1) || !MathIsValidNumber((double)candidate2)) { reason = "policy.td_non_finite_output"; return(false); } y1 = candidate1; y2 = candidate2; return(true); } //+------------------------------------------------------------------+ //| DPCCUpdatePolicyCritic learns a scalar realized rollout return | //+------------------------------------------------------------------+ bool DPCCUpdatePolicyCritic(CNet &critic, const SDPCCShape &shape, const float realized_return, const float probability_weight, const float value_weight, string &reason) { //--- Caller has forwarded the pending state and its recorded physical plan. if(!DPCCPolicyNetPreflight(critic, shape, true, reason)) return(false); if(!MathIsValidNumber(realized_return) || !MathIsValidNumber(probability_weight) || !MathIsValidNumber(value_weight) || !critic.WeightsUpdateEnabled()) { reason = "policy.critic_target_or_frozen"; return(false); } CNeuronACSRM *head = dynamic_cast(critic.Layer(3)); if(!head.calcDistributionTargetGradients(critic.Layer(2), realized_return, probability_weight, value_weight)) { reason = "policy.critic_distribution_gradient"; return(false); } if(!critic.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, -1, true)) { reason = "policy.critic_backward"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCActorGradientDifferent dispatches the existing library | //| kernel | //+------------------------------------------------------------------+ bool DPCCActorGradientDifferent(COpenCLMy *context, CBufferFloat *left, CBufferFloat *right, CBufferFloat *output, const uint count, string &reason) { if(CheckPointer(context) == POINTER_INVALID || CheckPointer(left) == POINTER_INVALID || CheckPointer(right) == POINTER_INVALID || CheckPointer(output) == POINTER_INVALID || count == 0 || left.GetOpenCL() != context || right.GetOpenCL() != context || output.GetOpenCL() != context || left.GetIndex() < 0 || right.GetIndex() < 0 || output.GetIndex() < 0 || count > (uint)INT_MAX || left.Total() < (int)count || right.Total() < (int)count || output.Total() < (int)count) { reason = "policy.actor_gradient.difference_buffers"; return(false); } const int buffers[] = {left.GetIndex(), right.GetIndex(), output.GetIndex()}; for(int i = 0; i < 3; i++) if(!context.SetArgumentBuffer(def_k_MatrixDif, i, buffers[i])) { reason = StringFormat("policy.actor_gradient.difference_buffer.%u", i); return(false); } const int zero = 0; if(!context.SetArgument(def_k_MatrixDif, def_k_dif_multiplyer, 1.0f) || !context.SetArgument(def_k_MatrixDif, def_k_dif_shift_in1, zero) || !context.SetArgument(def_k_MatrixDif, def_k_dif_shift_in2, zero) || !context.SetArgument(def_k_MatrixDif, def_k_dif_shift_out, zero)) { reason = "policy.actor_gradient.difference_arguments"; return(false); } uint offset[2] = {0, 0}; uint global[2] = {1, count}; if(!context.Execute(def_k_MatrixDif, 2, offset, global)) { reason = "policy.actor_gradient.difference_execute"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCActorGradientGate dispatches the registered library kernel | //+------------------------------------------------------------------+ bool DPCCActorGradientGate(COpenCLMy *context, CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *gate, CBufferFloat *output, const uint count, string &reason) { if(CheckPointer(context) == POINTER_INVALID || CheckPointer(input1) == POINTER_INVALID || CheckPointer(input2) == POINTER_INVALID || CheckPointer(gate) == POINTER_INVALID || CheckPointer(output) == POINTER_INVALID || count == 0 || input1.GetOpenCL() != context || input2.GetOpenCL() != context || gate.GetOpenCL() != context || output.GetOpenCL() != context || input1.GetIndex() < 0 || input2.GetIndex() < 0 || gate.GetIndex() < 0 || output.GetIndex() < 0 || count > (uint)INT_MAX || input1.Total() < (int)count || input2.Total() < (int)count || gate.Total() < (int)count || output.Total() < (int)count) { reason = "policy.actor_gradient.gate_buffers"; return(false); } const int buffers[] = {input1.GetIndex(), input2.GetIndex(), gate.GetIndex(), output.GetIndex(), output.GetIndex()}; for(int i = 0; i < 5; i++) if(!context.SetArgumentBuffer(def_k_GateElementMul, i, buffers[i])) { reason = StringFormat("policy.actor_gradient.gate_buffer.%u", i); return(false); } if(!context.SetArgument(def_k_GateElementMul, def_k_gem_flags_enabled, 0)) { reason = "policy.actor_gradient.gate_flags"; return(false); } uint offset[1] = {0}; uint global[1] = {count}; if(!context.Execute(def_k_GateElementMul, 1, offset, global)) { reason = "policy.actor_gradient.gate_execute"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCPrepareDiscountedPlanGradient builds the shared full-H loss | //+------------------------------------------------------------------+ bool DPCCPrepareDiscountedPlanGradient(CNet &actor, const SDPCCShape &shape, const int output_layer, const float &teacher_plan[], const float &valid_steps[], const float gamma, string &reason) { reason = ""; CNeuronDPCCActorOCL *dpcc_head = dynamic_cast(actor.Layer(output_layer)); if(CheckPointer(dpcc_head) != POINTER_INVALID) return(dpcc_head.PrepareOracleGradient(shape.H, shape.NActions, teacher_plan, valid_steps, gamma, reason)); SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason) || actor.Layer(output_layer) == NULL) return(false); COpenCLMy *context = actor.GetOpenCL(); CBufferFloat *destination = actor.Layer(output_layer).getGradient(); CBufferFloat *actor_values = actor.Layer(output_layer).getOutput(); if(CheckPointer(context) == POINTER_INVALID || CheckPointer(destination) == POINTER_INVALID || CheckPointer(actor_values) == POINTER_INVALID || destination.GetOpenCL() != context || actor_values.GetOpenCL() != context || destination.GetIndex() < 0 || actor_values.GetIndex() < 0 || destination.Total() != (int)layout.plan_count || actor_values.Total() != (int)layout.plan_count) { reason = "policy.oracle.gradient_buffers"; return(false); } if(!destination.Fill(0.0f)) { reason = "policy.oracle.gradient_clear"; return(false); } if(ArraySize(teacher_plan) != (int)layout.plan_count || ArraySize(valid_steps) != (int)shape.H || !MathIsValidNumber(gamma) || gamma < 0.0f || gamma > 1.0f) { reason = "policy.oracle.target_shape_or_gamma"; return(false); } for(uint i = 0; i < layout.plan_count; i++) if(!MathIsValidNumber(teacher_plan[i]) || teacher_plan[i] < 0.0f || teacher_plan[i] > 1.0f) { reason = StringFormat("policy.oracle.target_value.%u", i); return(false); } double discounts[]; if(ArrayResize(discounts, (int)shape.H) != (int)shape.H) { reason = "policy.oracle.discount_capacity"; return(false); } double discount = 1.0; double normalization = 0.0; for(uint h = 0; h < shape.H; h++) { if(valid_steps[h] != 0.0f && valid_steps[h] != 1.0f) { reason = StringFormat("policy.oracle.valid_step.%u", h); return(false); } discounts[h] = discount; normalization += (double)valid_steps[h] * discount; discount *= (double)gamma; } if(!MathIsValidNumber(normalization) || normalization <= 0.0) { reason = "policy.oracle.no_discounted_valid_steps"; return(false); } if(!actor_values.BufferRead()) { reason = "policy.oracle.actor_output_read"; return(false); } for(uint i = 0; i < layout.plan_count; i++) { const float action = actor_values.At((int)i); if(!MathIsValidNumber(action) || action < 0.0f || action > 1.0f) { reason = StringFormat("policy.oracle.actor_output.%u", i); return(false); } } const int plan_size = (int)layout.plan_count; float weight_values[], one_values[], zero_values[]; if(ArrayResize(weight_values, plan_size) != plan_size || ArrayResize(one_values, plan_size) != plan_size || ArrayResize(zero_values, plan_size) != plan_size) { reason = "policy.oracle.scratch_capacity"; return(false); } for(int i = 0; i < plan_size; i++) { const uint horizon = (uint)i / shape.NActions; weight_values[i] = (float)((double)valid_steps[horizon] * discounts[horizon] / normalization); one_values[i] = 1.0f; zero_values[i] = 0.0f; } CBufferFloat target_buffer, weight_buffer, one_buffer, zero_buffer; CBufferFloat error_buffer, weighted_buffer, one_minus_buffer, factor_buffer; if(!target_buffer.AssignArray(teacher_plan) || !weight_buffer.AssignArray(weight_values) || !one_buffer.AssignArray(one_values) || !zero_buffer.AssignArray(zero_values) || !error_buffer.BufferInit(layout.plan_count, 0.0f) || !weighted_buffer.BufferInit(layout.plan_count, 0.0f) || !one_minus_buffer.BufferInit(layout.plan_count, 0.0f) || !factor_buffer.BufferInit(layout.plan_count, 0.0f)) { reason = "policy.oracle.scratch_init"; return(false); } CBufferFloat *scratch[] = {GetPointer(target_buffer), GetPointer(weight_buffer), GetPointer(one_buffer), GetPointer(zero_buffer), GetPointer(error_buffer), GetPointer(weighted_buffer), GetPointer(one_minus_buffer), GetPointer(factor_buffer) }; for(int i = 0; i < 8; i++) if(!scratch[i].BufferCreate(context) || !scratch[i].BufferWrite()) { reason = StringFormat("policy.oracle.scratch_opencl.%d", i); return(false); } if(!DPCCActorGradientDifferent(context, GetPointer(target_buffer), actor_values, GetPointer(error_buffer), layout.plan_count, reason) || !DPCCActorGradientGate(context, GetPointer(error_buffer), GetPointer(zero_buffer), GetPointer(weight_buffer), GetPointer(weighted_buffer), layout.plan_count, reason) || !DPCCActorGradientDifferent(context, GetPointer(one_buffer), actor_values, GetPointer(one_minus_buffer), layout.plan_count, reason) || !DPCCActorGradientGate(context, actor_values, GetPointer(zero_buffer), GetPointer(one_minus_buffer), GetPointer(factor_buffer), layout.plan_count, reason) || !DPCCActorGradientGate(context, GetPointer(weighted_buffer), GetPointer(zero_buffer), GetPointer(factor_buffer), destination, layout.plan_count, reason)) { destination.Fill(0.0f); if(reason == "") reason = "policy.oracle.gradient_dispatch"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCPrepareOracleActorGradient validates the legacy dense graph | //+------------------------------------------------------------------+ bool DPCCPrepareOracleActorGradient(CNet &actor, const SDPCCShape &shape, const float &teacher_plan[], const float &valid_steps[], const float gamma, string &reason) { return(DPCCPolicyNetPreflight(actor, shape, false, reason) && DPCCPrepareDiscountedPlanGradient(actor, shape, 2, teacher_plan, valid_steps, gamma, reason)); } //+------------------------------------------------------------------+ //| DPCCPreparePolicyActorGradient routes frozen Critic SRM on | //| device | //+------------------------------------------------------------------+ bool DPCCPreparePolicyActorGradient(CNet &actor, CNet &critic, const SDPCCShape &shape, const float upstream, const int risk_type, const float risk_parameter, const float mean_weight, string &reason) { //--- Caller forwards Actor and selected Critic on the same state and Actor plan. SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(shape, layout, reason) || !DPCCPolicyNetPreflight(actor, shape, false, reason) || !DPCCPolicyNetPreflight(critic, shape, true, reason)) return(false); if(actor.GetOpenCL() != critic.GetOpenCL() || !MathIsValidNumber(upstream) || !MathIsValidNumber(risk_parameter) || !MathIsValidNumber(mean_weight)) { reason = "policy.actor_gradient_context_or_upstream"; return(false); } COpenCLMy *context = actor.GetOpenCL(); CBufferFloat value, gradient, state_gradient; CBufferFloat action_gradient, action_one, action_zero, action_factor; CBufferFloat *scratch[] = {GetPointer(value), GetPointer(gradient), GetPointer(state_gradient)}; const uint sizes[] = {1, 1, layout.actor_count}; const float values[] = {0, upstream, 0}; for(int i = 0; i < 3; i++) { if(!scratch[i].BufferInit(sizes[i], values[i])) { reason = StringFormat("policy.actor_gradient_scratch_init.%d", i); return(false); } if(!scratch[i].BufferCreate(context)) { reason = StringFormat("policy.actor_gradient_scratch_create.%d", i); return(false); } if(!scratch[i].BufferWrite()) { reason = StringFormat("policy.actor_gradient_scratch_write.%d", i); return(false); } } CBufferFloat *action_scratch[] = {GetPointer(action_gradient), GetPointer(action_one), GetPointer(action_zero), GetPointer(action_factor) }; for(int i = 0; i < 4; i++) { if(!action_scratch[i].BufferInit(layout.plan_count, 0.0f)) { reason = StringFormat("policy.actor_action_scratch_init.%d", i); return(false); } if(!action_scratch[i].BufferCreate(context)) { reason = StringFormat("policy.actor_action_scratch_create.%d", i); return(false); } } if(!action_one.Fill(1.0f) || !action_zero.Fill(0.0f) || !action_gradient.Fill(0.0f) || !action_factor.Fill(0.0f)) { reason = "policy.actor_action_scratch_fill"; return(false); } CNeuronACSRM *head = dynamic_cast(critic.Layer(3)); if(!head.SRMForward(GetPointer(value), 1, risk_type, risk_parameter, mean_weight)) { reason = "policy.actor_srm_forward"; return(false); } if(!head.SRMGradient(GetPointer(gradient), 1, risk_type, risk_parameter, mean_weight)) { reason = "policy.actor_srm_gradient"; return(false); } //--- Preserve the selected Critic's prior write permission on every backward outcome. const bool previous = critic.WeightsUpdateEnabled(); if(!critic.SetWeightsUpdate(false)) { reason = "policy.actor_freeze_critic"; return(false); } const bool propagated = critic.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, -1, false); const bool restored = critic.SetWeightsUpdate(previous); if(!propagated || !restored) { reason = "policy.actor_critic_backward_or_restore"; return(false); } //--- Split the Critic gradient into the Actor state and executed action tail. CBufferFloat *source = critic.Layer(0).getGradient(); CBufferFloat *destination = actor.Layer(2).getGradient(); CBufferFloat *actor_values = actor.Layer(2).getOutput(); if(source.GetIndex() < 0 || destination.GetIndex() < 0 || actor_values.GetIndex() < 0 || source.GetOpenCL() != context || destination.GetOpenCL() != context || actor_values.GetOpenCL() != context || source.Total() != (int)layout.critic_count || destination.Total() != (int)layout.plan_count || actor_values.Total() != (int)layout.plan_count || state_gradient.Total() != (int)layout.actor_count || action_gradient.Total() != (int)layout.plan_count || action_one.Total() != (int)layout.plan_count || action_zero.Total() != (int)layout.plan_count || action_factor.Total() != (int)layout.plan_count || action_gradient.GetOpenCL() != context || action_one.GetOpenCL() != context || action_zero.GetOpenCL() != context || action_factor.GetOpenCL() != context) { reason = "policy.actor_gradient_buffers"; return(false); } if(!destination.Fill(0.0f)) { reason = "policy.actor_gradient_clear"; return(false); } const int arguments[] = {state_gradient.GetIndex(), (int)layout.actor_count, action_gradient.GetIndex(), (int)shape.NActions, state_gradient.GetIndex(), 0, destination.GetIndex(), 0, source.GetIndex() }; for(uint argument = 0; argument < 9; argument++) { const bool bound = (argument % 2 == 0 ? context.SetArgumentBuffer(def_k_DeconcatenateMatrix, argument, arguments[argument]) : context.SetArgument(def_k_DeconcatenateMatrix, argument, arguments[argument])); if(!bound) { reason = StringFormat("policy.actor_gradient_argument.%u", argument); return(false); } } uint offset[2] = {0, 0}; uint global[2] = {1, MathMax(layout.actor_count, shape.NActions)}; if(!context.Execute(def_k_DeconcatenateMatrix, 2, offset, global)) { reason = "policy.actor_gradient_split"; return(false); } //--- Build exact Sigmoid derivatives on-device with existing elementwise kernels. CNeuronBaseOCL *actor_output = actor.Layer(2); if(!DPCCActorGradientDifferent(context, GetPointer(action_one), actor_values, destination, layout.plan_count, reason)) { if(reason == "") reason = "policy.actor_sigmoid_one_minus"; destination.Fill(0.0f); return(false); } if(!DPCCActorGradientGate(context, actor_values, GetPointer(action_zero), destination, GetPointer(action_factor), layout.plan_count, reason)) { if(reason == "") reason = "policy.actor_sigmoid_derivative"; destination.Fill(0.0f); return(false); } if(!DPCCActorGradientGate(context, GetPointer(action_gradient), GetPointer(action_zero), GetPointer(action_factor), destination, layout.plan_count, reason)) { if(reason == "") reason = "policy.actor_sigmoid_chain"; destination.Fill(0.0f); return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCOnlineActorCritic selects the successful Actor event source | //+------------------------------------------------------------------+ int DPCCOnlineActorCritic(const ulong event_index, const int selector, const uint seed) { if(selector == 1) return(0); if(selector == 2) return(1); if(selector != 0) return(-1); return((int)((event_index * (ulong)2654435761 + (ulong)seed) & 1)); } //+------------------------------------------------------------------+ //| DPCCOnlineCrossedTarget alternates targets from the seeded start | //+------------------------------------------------------------------+ int DPCCOnlineCrossedTarget(const ulong event_index, const uint seed) { return((int)((event_index + (ulong)seed) & 1)); } //+------------------------------------------------------------------+ //| Online policy transition inputs and completed mutation flags | //+------------------------------------------------------------------+ struct SDPCCOnlinePolicyStep { float reward, gamma, bootstrap_mask, sample_u1, sample_u2; float probability_weight, value_weight, actor_upstream, risk_parameter, mean_weight, target_tau; int risk_type, actor_critic, refresh_critic_target; bool update_actor, refresh_actor_target; }; struct SDPCCOnlinePolicyResult { float z1, z2, y1, y2; bool mutation_started, q1_updated, q2_updated, actor_updated, actor_target_refreshed, critic_target_refreshed; }; //+------------------------------------------------------------------+ //| DPCCPolicyTuplePreflight admits six distinct native policy nets | //+------------------------------------------------------------------+ bool DPCCPolicyTuplePreflight(CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCShape &shape, string &reason) { CNet *nets[] = {GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; for(int i = 0; i < 6; i++) { if(!DPCCValidatePolicyNet(nets[i], shape, i != 0 && i != 3, reason)) return(false); if(nets[i].GetOpenCL() != actor.GetOpenCL()) { reason = "policy.online.tuple_context"; return(false); } for(int j = 0; j < i; j++) { if(nets[i] == nets[j]) { reason = "policy.online.tuple_alias"; return(false); } const int current_layers = (i == 0 || i == 3 ? 3 : 4); const int other_layers = (j == 0 || j == 3 ? 3 : 4); for(int a = 0; a < current_layers; a++) for(int b = 0; b < other_layers; b++) if(nets[i].Layer(a) == nets[j].Layer(b)) { reason = "policy.online.tuple_layer_alias"; return(false); } } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCSyncPolicyTargets averages selected crossed native targets | //+------------------------------------------------------------------+ bool DPCCSyncPolicyTargets(CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCShape &shape, const bool refresh_actor, const int crossed_critic, string &reason, const float tau = 1.0f) { //--- Reject the complete contract before either selected target changes. if(!DPCCPolicyTuplePreflight(actor, q1, q2, target_actor, target_q1, target_q2, shape, reason)) return(false); if(!MathIsValidNumber(tau) || tau < 0.0f || tau > 1.0f || crossed_critic < -1 || crossed_critic > 1 || target_actor.WeightsUpdateEnabled() || target_q1.WeightsUpdateEnabled() || target_q2.WeightsUpdateEnabled()) { reason = "policy.online.target_sync_contract"; return(false); } //--- Preserve crossed pairing for hard initialization and Polyak updates. if(refresh_actor && !target_actor.WeightsUpdate(GetPointer(actor), tau)) { reason = "policy.online.target_actor_copy"; return(false); } CNet *destination = (crossed_critic == 0 ? GetPointer(target_q1) : GetPointer(target_q2)); CNet *source = (crossed_critic == 0 ? GetPointer(q2) : GetPointer(q1)); if(crossed_critic >= 0 && !destination.WeightsUpdate(source, tau)) { reason = "policy.online.target_critic_copy"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCInitializePolicyTargets initializes and freezes shadow nets | //+------------------------------------------------------------------+ bool DPCCInitializePolicyTargets(CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCShape &shape, string &reason) { if(!DPCCPolicyTuplePreflight(actor, q1, q2, target_actor, target_q1, target_q2, shape, reason)) return(false); CNet *targets[] = {GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2)}; for(int i = 0; i < 3; i++) { if(!targets[i].TrainMode(false)) { reason = StringFormat("policy.online.target_inference.%d", i); return(false); } if(!targets[i].SetWeightsUpdate(false)) { reason = StringFormat("policy.online.target_freeze.%d", i); return(false); } } if(!DPCCSyncPolicyTargets(actor, q1, q2, target_actor, target_q1, target_q2, shape, true, 0, reason, 1.0f)) return(false); return(DPCCSyncPolicyTargets(actor, q1, q2, target_actor, target_q1, target_q2, shape, false, 1, reason, 1.0f)); } //+------------------------------------------------------------------+ //| DPCCSamplePolicyTarget samples the native return distribution | //+------------------------------------------------------------------+ bool DPCCSamplePolicyTarget(CNet &critic, const float u, float &value, string &reason) { value = 0.0f; CNeuronACSRM *head = dynamic_cast(critic.Layer(3)); if(CheckPointer(head) == POINTER_INVALID || !MathIsValidNumber(u) || u <= 0.0f || u >= 1.0f) { reason = "policy.online.sample_input"; return(false); } if(!head.ValidateDistributionForSRM(1, defSRM_MEAN, 0.1f, 0.0f, false)) { reason = "policy.online.sample_distribution"; return(false); } CBufferFloat sample, random; CBufferFloat *buffers[] = {GetPointer(sample), GetPointer(random)}; const float values[] = {0.0f, u}; for(int i = 0; i < 2; i++) { if(!buffers[i].BufferInit(1, values[i])) { reason = StringFormat("policy.online.sample_init.%d", i); return(false); } if(!buffers[i].BufferCreate(critic.GetOpenCL())) { reason = StringFormat("policy.online.sample_create.%d", i); return(false); } if(!buffers[i].BufferWrite()) { reason = StringFormat("policy.online.sample_write.%d", i); return(false); } } if(!head.Sample(GetPointer(sample), GetPointer(random), 1)) { reason = "policy.online.sample_dispatch"; return(false); } if(!sample.BufferRead() || !MathIsValidNumber(sample.At(0))) { reason = "policy.online.sample_read"; return(false); } value = sample.At(0); reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCUpdateOnlinePolicy applies a caller-supplied one-bar tuple | //+------------------------------------------------------------------+ bool DPCCUpdateOnlinePolicy(CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCShape &shape, CBufferFloat *scenarios, const float &account13[], CBufferFloat *behavior_plan, CBufferFloat *next_scenarios, const float &next_account13[], const SDPCCOnlinePolicyStep &step, SDPCCOnlinePolicyResult &result, string &reason) { //--- Invalid inputs are rejected before trainable parameters can change. ZeroMemory(result); reason = ""; float checked_y1 = 0.0f, checked_y2 = 0.0f; if(!DPCCBuildOneBarTDTargets(step.reward, step.gamma, step.bootstrap_mask, 0.0f, 0.0f, checked_y1, checked_y2, reason)) return(false); if(!DPCCPolicyTuplePreflight(actor, q1, q2, target_actor, target_q1, target_q2, shape, reason)) return(false); if(!actor.WeightsUpdateEnabled() || !q1.WeightsUpdateEnabled() || !q2.WeightsUpdateEnabled() || target_actor.WeightsUpdateEnabled() || target_q1.WeightsUpdateEnabled() || target_q2.WeightsUpdateEnabled() || step.actor_critic < 0 || step.actor_critic > 1 || step.refresh_critic_target < -1 || step.refresh_critic_target > 1 || !MathIsValidNumber(step.target_tau) || step.target_tau < 0.0f || step.target_tau > 1.0f || !MathIsValidNumber(step.probability_weight) || step.probability_weight < 0.0f || !MathIsValidNumber(step.value_weight) || step.value_weight < 0.0f) { reason = "policy.online.permission_or_weights"; return(false); } //--- Native backprop may return success while inference-mode layers skip writes. const int actor_layers = 3, critic_layers = 4; for(int i = 0; i < critic_layers; i++) if(!q1.Layer(i).TrainMode() || !q2.Layer(i).TrainMode()) { reason = "policy.online.critic_inference_mode"; return(false); } if(step.update_actor) for(int i = 0; i < actor_layers; i++) if(!actor.Layer(i).TrainMode()) { reason = "policy.online.actor_inference_mode"; return(false); } for(int i = 0; i < 3; i++) { CNet *target = (i == 0 ? GetPointer(target_actor) : (i == 1 ? GetPointer(target_q1) : GetPointer(target_q2))); if(target.Layer(1).TrainMode()) { reason = "policy.online.target_training"; return(false); } } if(step.bootstrap_mask == 1.0f && (!MathIsValidNumber(step.sample_u1) || step.sample_u1 <= 0.0f || step.sample_u1 >= 1.0f || !MathIsValidNumber(step.sample_u2) || step.sample_u2 <= 0.0f || step.sample_u2 >= 1.0f)) { reason = "policy.online.sample_range"; return(false); } if(step.update_actor) { const bool risk_valid = step.risk_type == defSRM_MEAN || ((step.risk_type == defSRM_CVAR || step.risk_type == defSRM_MEAN_CVAR) && step.risk_parameter > 0.0f && step.risk_parameter <= 1.0f) || (step.risk_type == defSRM_EXPONENTIAL && step.risk_parameter >= 0.0f) || (step.risk_type == defSRM_DUAL_POWER && step.risk_parameter >= 1.0f); if(!risk_valid || !MathIsValidNumber(step.actor_upstream) || !MathIsValidNumber(step.risk_parameter) || !MathIsValidNumber(step.mean_weight) || (step.risk_type == defSRM_MEAN_CVAR && (step.mean_weight < 0.0f || step.mean_weight > 1.0f))) { reason = "policy.online.actor_risk"; return(false); } } CBufferFloat pending_input, behavior_input, next_input, target_input, current_input; if(CheckPointer(behavior_plan) == POINTER_INVALID || !DPCCPackPolicyInput(shape, scenarios, account13, behavior_plan, GetPointer(behavior_input), reason)) { if(reason == "") reason = "policy.online.behavior_plan"; return(false); } for(uint i = 0; i < shape.NActions; i++) if(behavior_plan.At((int)i) < 0.0f || behavior_plan.At((int)i) > 1.0f) { reason = "policy.online.behavior_range"; return(false); } if(step.update_actor && !DPCCPackPolicyInput(shape, scenarios, account13, NULL, GetPointer(pending_input), reason)) return(false); //--- Frozen targets bootstrap from the next observation; terminal tuples skip it. if(step.bootstrap_mask == 1.0f) { if(!DPCCPackPolicyInput(shape, next_scenarios, next_account13, NULL, GetPointer(next_input), reason)) return(false); if(!target_actor.feedForward(GetPointer(next_input), 1, false, (CBufferFloat*)NULL)) { reason = "policy.online.target_actor_forward"; return(false); } if(!DPCCPackPolicyInput(shape, next_scenarios, next_account13, target_actor.Layer(2).getOutput(), GetPointer(target_input), reason)) return(false); CNet *targets[] = {GetPointer(target_q1), GetPointer(target_q2)}; const float random_values[] = {step.sample_u1, step.sample_u2}; float samples[2]; for(int i = 0; i < 2; i++) { if(!targets[i].feedForward(GetPointer(target_input), 1, false, (CBufferFloat*)NULL)) { reason = StringFormat("policy.online.target_critic_forward.%d", i); return(false); } if(!DPCCSamplePolicyTarget(targets[i], random_values[i], samples[i], reason)) return(false); } result.z1 = samples[0]; result.z2 = samples[1]; } if(!DPCCBuildOneBarTDTargets(step.reward, step.gamma, step.bootstrap_mask, result.z1, result.z2, result.y1, result.y2, reason)) return(false); //--- Q updates use the recorded behavior tuple, never a recomputed action. CNet *critics[] = {GetPointer(q1), GetPointer(q2)}; const float td_targets[] = {result.y1, result.y2}; for(int i = 0; i < 2; i++) { if(!critics[i].feedForward(GetPointer(behavior_input), 1, false, (CBufferFloat*)NULL)) { reason = StringFormat("policy.online.behavior_critic_forward.%d", i); return(false); } result.mutation_started = true; if(!DPCCUpdatePolicyCritic(critics[i], shape, td_targets[i], step.probability_weight, step.value_weight, reason)) return(false); if(i == 0) result.q1_updated = true; else result.q2_updated = true; } //--- Actor uses the selected updated Q on a fresh policy action at the pending state. if(step.update_actor) { if(!actor.feedForward(GetPointer(pending_input), 1, false, (CBufferFloat*)NULL)) { reason = "policy.online.current_actor_forward"; return(false); } if(!DPCCPackPolicyInput(shape, scenarios, account13, actor.Layer(2).getOutput(), GetPointer(current_input), reason)) return(false); CNet *selected = critics[step.actor_critic]; if(!selected.feedForward(GetPointer(current_input), 1, false, (CBufferFloat*)NULL)) { reason = "policy.online.current_critic_forward"; return(false); } if(!DPCCPreparePolicyActorGradient(actor, selected, shape, step.actor_upstream, step.risk_type, step.risk_parameter, step.mean_weight, reason)) return(false); if(!actor.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, -1, true)) { reason = "policy.online.actor_backward"; return(false); } result.actor_updated = true; } //--- Completed flags survive failure so callers cannot replay a partly applied tuple. if(step.refresh_actor_target) { if(!DPCCSyncPolicyTargets(actor, q1, q2, target_actor, target_q1, target_q2, shape, true, -1, reason, step.target_tau)) return(false); result.actor_target_refreshed = true; } if(step.refresh_critic_target >= 0) { if(!DPCCSyncPolicyTargets(actor, q1, q2, target_actor, target_q1, target_q2, shape, false, step.refresh_critic_target, reason, step.target_tau)) return(false); result.critic_target_refreshed = true; } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCModelCount | //+------------------------------------------------------------------+ bool DPCCModelCount(const ulong value, uint &count) { count = 0; if(value == 0 || value > INT_MAX || value > ulong(UINT_MAX) / sizeof(float)) return(false); count = (uint)value; return(true); } //+------------------------------------------------------------------+ //| Function DPCCModelDescribe | //+------------------------------------------------------------------+ bool DPCCModelDescribe(CLayerDescription *description, const SDPCCConfig &config, const uint type, const uint optimizer_batch) { if(CheckPointer(description) == POINTER_INVALID || optimizer_batch == 0) return(false); SDPCCShape shape = config.shape; uint count = 0; if(shape.K == 0 || shape.V == 0 || shape.H == 0 || shape.D == 0 || shape.M == 0 || shape.J == 0 || shape.E == 0 || shape.PlanBatch == 0 || shape.NActions != 6 || shape.StateSize != 13) return(false); if(type == defNeuronScenarioDecoderOCL) {if((ulong)shape.K > ulong(INT_MAX) / shape.H / 4 || !DPCCModelCount((ulong)shape.K * shape.H * 4, count))return( false);} else if(type == defNeuronPlanDenoiserOCL) {if(shape.PlanBatch != 1 || !DPCCModelCount((ulong)shape.H * 6, count))return(false);} else if(type == defNeuronDPCCOCL) count = 6; else return(false); if(!DPCCSetLayerBaseDefaults(description, type, count, optimizer_batch)) return(false); description.window = 6; description.window_out = 13; description.variables = shape.V; description.step = shape.H; description.layers = (type == defNeuronPlanDenoiserOCL ? shape.Depth : 0); if(ArrayResize(description.windows, 9) != 9 || ArrayResize(description.units, 5) != 5 || ArrayResize( description.heads, 1) != 1) return(false); uint windows[] = {2, shape.K, shape.V, shape.H, shape.D, shape.M, shape.J, shape.E, shape.PlanBatch}; uint units[] = {shape.Embedding, shape.Heads, shape.KeyDimension, shape.NActions, shape.StateSize}; for(uint i = 0; i < 9; i++) description.windows[i] = windows[i]; for(uint i = 0; i < 5; i++) description.units[i] = units[i]; description.heads[0] = shape.Heads; return(true); } //+------------------------------------------------------------------+ //| Function DPCCModelCreatePair | //+------------------------------------------------------------------+ bool DPCCModelCreatePair(CNet &net, const SDPCCConfig &config, const uint type, const uint input_count, const uint optimizer_batch, COpenCLMy *context, string &reason) { reason = ""; if(input_count == 0 || input_count > INT_MAX || input_count > UINT_MAX / sizeof(float)) { reason = "model.input_capacity"; return(false); } if(context != NULL && CheckPointer(context) == POINTER_INVALID) { reason = "model.invalid_context"; return(false); } //--- Do not discard an existing checkpoint/model as a side effect of factory creation. if(net.Layer(0) != NULL) { reason = "model.destination_not_empty"; return(false); } CArrayObj descriptions; CLayerDescription *input_description = new CLayerDescription; CLayerDescription *model = new CLayerDescription; if(!input_description || !model) { delete input_description; delete model; reason = "model.description_allocation"; return(false); } input_description.type = defNeuronBaseOCL; input_description.count = input_count; input_description.batch = optimizer_batch; input_description.activation = None; input_description.optimization = ADAM; if(!DPCCModelDescribe(model, config, type, optimizer_batch)) {delete input_description; delete model; reason = "model.description_contract"; return(false);} if(!descriptions.Add(input_description)) { delete input_description; delete model; reason = "model.input_add"; return(false); } if(!descriptions.Add(model)) { delete model; reason = "model.layer_add"; return(false); } if(!net.Create(GetPointer(descriptions)) || (CheckPointer(context) != POINTER_INVALID && !net.SetOpenCLChecked(context))) {net.Clear(); reason = "model.create_or_context"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCCreateDecoderNet | //+------------------------------------------------------------------+ bool DPCCCreateDecoderNet(CNet &net, const SDPCCConfig &config, const SDPCCDecoderComposition &composition, const uint optimizer_batch, COpenCLMy *context, CNeuronScenarioDecoderOCL *&decoder, string &reason) { decoder = NULL; uint count = 0; //--- Widen incrementally; no four-uint wrap before checking tensor capacity. ulong value = config.shape.K; const uint factors[] = {config.shape.V, config.shape.H, config.shape.D}; for(uint i = 0; i < 3; i++) {if(factors[i] == 0 || value > ulong(INT_MAX) / factors[i]) {reason = "model.decoder_input_shape"; return(false); } value *= factors[i];} if(!DPCCModelCount(value, count) || !DPCCModelCreatePair(net, config, defNeuronScenarioDecoderOCL, count, optimizer_batch, context, reason)) return(false); CNeuronScenarioDecoderOCL *candidate = dynamic_cast(net.Layer(1)); if(!candidate || candidate.Type() != defNeuronScenarioDecoderOCL || !candidate.Configure(config, composition)) {net.Clear(); reason = "model.decoder_configure"; return(false);} decoder = candidate; return(true); } //+------------------------------------------------------------------+ //| Function DPCCCreateDenoiserNet | //+------------------------------------------------------------------+ bool DPCCCreateDenoiserNet(CNet &net, const SDPCCConfig &config, const SDPCCDenoiserComposition &model_composition, const SDPCCDenoiserComposition &training_composition, const uint optimizer_batch, COpenCLMy *context, CNeuronPlanDenoiserOCL *&denoiser, string &reason) { denoiser = NULL; uint count = 0; if(config.shape.PlanBatch != 1 || !DPCCModelCount((ulong)config.shape.H * 6, count)) {reason = "model.denoiser_single_plan"; return(false);} if(!DPCCModelCreatePair(net, config, defNeuronPlanDenoiserOCL, count, optimizer_batch, context, reason)) return(false); CNeuronPlanDenoiserOCL *candidate = dynamic_cast(net.Layer(1)); if(!candidate || candidate.Type() != defNeuronPlanDenoiserOCL || !candidate.Configure(config, model_composition, training_composition)) {net.Clear(); reason = "model.denoiser_configure"; return(false);} denoiser = candidate; return(true); } //+------------------------------------------------------------------+ //| Function DPCCCreateActorNet | //+------------------------------------------------------------------+ bool DPCCCreateActorNet(CNet &net, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder_composition, const SDPCCDenoiserComposition &model_composition, const SDPCCDenoiserComposition &training_composition, const double &beta[], const double &alpha_bar[], const double &sigma[], const uint optimizer_batch, COpenCLMy *context, CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; if(!DPCCModelCreatePair(net, config, defNeuronDPCCOCL, 13, optimizer_batch, context, reason)) return(false); CNeuronDPCCOCL *candidate = dynamic_cast(net.Layer(1)); if(!candidate || candidate.Type() != defNeuronDPCCOCL || !candidate.Configure(config, decoder_composition, model_composition, training_composition, beta, alpha_bar, sigma)) {net.Clear(); reason = "model.actor_configure"; return(false);} net.TrainMode(false); net.SetWeightsUpdate(false); actor = candidate; return(true); } //+------------------------------------------------------------------+ //| Create a caller-owned CNet directly on the borrowed context | //+------------------------------------------------------------------+ bool DPCCCreateNetInContext(CNet *&net, CArrayObj &descriptions, COpenCLMy *context, string &reason) { reason = ""; if(net != NULL || CheckPointer(context) == POINTER_INVALID || descriptions.Total() < 2) { reason = "model.context.destination_or_graph"; return(false); } //--- Only Base layers transfer; the final graph is created on the shared context. CLayerDescription input_layer, output; if(!DPCCSetLayerBaseDefaults(GetPointer(input_layer), defNeuronBaseOCL, 1, 1)) { reason = "model.context.input_defaults"; return(false); } if(!DPCCSetLayerBaseDefaults(GetPointer(output), defNeuronBaseOCL, 1, 1)) { reason = "model.context.output_defaults"; return(false); } CArrayObj bootstrap; bootstrap.FreeMode(false); if(!bootstrap.Add(GetPointer(input_layer))) { reason = "model.context.input_add"; return(false); } if(!bootstrap.Add(GetPointer(output))) { reason = "model.context.output_add"; return(false); } CNet *created = new CNet; if(!created) { reason = "model.context.net_allocation"; return(false); } if(!created.Create(GetPointer(bootstrap))) reason = "model.context.bootstrap"; else if(!created.SetOpenCLChecked(context)) reason = "model.context.transfer"; else if(!created.Create(GetPointer(descriptions))) reason = "model.context.create"; else { net = created; return(true); } delete created; return(false); } //+------------------------------------------------------------------+ //| Create an ABI3 Actor; the caller owns the published CNet | //+------------------------------------------------------------------+ bool DPCCCreateAction6ActorNet(CNet *&net, const SDPCCRuntimeConfig &runtime, CNet *market, CNet *q1, CNet *q2, CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; reason = ""; //--- External networks and their context remain caller-owned. if(net != NULL) { reason = "model.action6.destination_not_empty"; return(false); } if(CheckPointer(market) == POINTER_INVALID || CheckPointer(q1) == POINTER_INVALID || CheckPointer(q2) == POINTER_INVALID || market == q1 || market == q2 || q1 == q2) { reason = "model.action6.policy_networks"; return(false); } COpenCLMy *context = market.GetOpenCL(); if(CheckPointer(context) == POINTER_INVALID || !market.IsOpenCLTransferReady() || market.WeightsUpdateEnabled() || q1.GetOpenCL() != context || q2.GetOpenCL() != context || !NNQCriticGraph(q1) || !NNQCriticGraph(q2)) { reason = "model.action6.policy_context_or_critic_graph"; return(false); } for(int i = 0; i < 8; i++) { CNeuronBaseOCL *layer = market.Layer(i); if(CheckPointer(layer) == POINTER_INVALID || layer.GetOpenCL() != context) { reason = "model.action6.market_graph"; return(false); } } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(market.Layer(7)); const SDPCCShape shape = runtime.financial.shape; if(!forecast || !decoder || market.Layer(8) != NULL || forecast.Type() != defNeuronScenarioForecastPriceOCL || forecast.ABI() != defScenarioForecastPriceABI || decoder.Type() != defNeuronScenarioDecoderOCL || decoder.ABI() != defScenarioDecoderPriceABI || forecast.Scenarios() != shape.K || forecast.Variables() != shape.V || forecast.Horizon() < shape.H || forecast.Dimension() == 0 || decoder.Scenarios() != forecast.Scenarios() || decoder.Horizon() != forecast.Horizon() || !DPCCValidateFixedConfig(shape, runtime.financial)) { reason = "model.action6.market_geometry_or_profile"; return(false); } CNet *critics[] = {q1, q2}; for(uint i = 0; i < (uint)ArraySize(critics); i++) { CNeuronScenarioCrossAttention *cross = dynamic_cast(critics[i].Layer(1)); if(!cross || cross.Scenarios() != forecast.Scenarios() || cross.Variables() != forecast.Variables() || cross.Horizon() != forecast.Horizon() || cross.Latent() != forecast.Dimension()) { reason = "model.action6.critic_geometry"; return(false); } } //--- CNet dispatches this descriptor to CNeuronDPCCOCL::InitAction6. CArrayObj descriptions; descriptions.FreeMode(true); CLayerDescription *input_layer = new CLayerDescription; CLayerDescription *model = new CLayerDescription; if(CheckPointer(input_layer) == POINTER_INVALID || CheckPointer(model) == POINTER_INVALID) { delete input_layer; delete model; reason = "model.action6.description_allocation"; return(false); } if(!DPCCSetLayerBaseDefaults(input_layer, defNeuronBaseOCL, 13, 1)) { delete input_layer; delete model; reason = "model.action6.input_defaults"; return(false); } if(!DPCCSetLayerBaseDefaults(model, defNeuronDPCCOCL, 6, 1)) { delete input_layer; delete model; reason = "model.action6.layer_defaults"; return(false); } model.window = 6; model.window_out = shape.Embedding; model.layers = shape.J; model.step = 1; model.variables = 1; const uint units[] = {shape.H, shape.M, shape.Heads, shape.K, shape.V, forecast.Horizon(), forecast.Dimension(), 3 }; if(ArrayResize(model.windows, 0) != 0 || ArrayResize(model.heads, 0) != 0 || ArrayCopy(model.units, units) != 8) { delete input_layer; delete model; reason = "model.action6.descriptor"; return(false); } if(!descriptions.Add(input_layer)) { delete input_layer; delete model; reason = "model.action6.input_add"; return(false); } if(!descriptions.Add(model)) { delete model; reason = "model.action6.layer_add"; return(false); } //--- Configure and bind the fresh graph before publishing its ownership. CNet *created = NULL; if(!DPCCCreateNetInContext(created, descriptions, context, reason)) return(false); CNeuronDPCCOCL *candidate = dynamic_cast(created.Layer(1)); if(!candidate || candidate.Type() != defNeuronDPCCOCL || created.Layer(2) != NULL) reason = "model.action6.created_graph"; else if(!candidate.Configure(runtime)) reason = StringFormat("model.action6.configure.status_%d", (int)candidate.LastStatus()); else if(!candidate.BindPolicyNetworks(market, q1, q2)) reason = "model.action6.bind_policy_networks"; else if(!created.TrainMode(false) || !created.SetWeightsUpdate(false)) reason = "model.action6.initial_mode"; else { //--- The caller deletes the result before its borrowed Market/Q/context owners. net = created; actor = candidate; return(true); } delete created; return(false); } //+------------------------------------------------------------------+ //| Create one independent Price Critic from the frozen Market axes | //+------------------------------------------------------------------+ bool DPCCCreateAction6CriticNet(CNet *&net, CNet *market, string &reason) { reason = ""; //--- Preserve StackSize from the AC-SRM donor's SkillAddCross descriptor. const uint stack_size = 24 * 21; if(net != NULL || CheckPointer(market) == POINTER_INVALID || !market.IsOpenCLTransferReady() || market.WeightsUpdateEnabled()) { reason = "model.price_q.destination_or_market"; return(false); } COpenCLMy *context = market.GetOpenCL(); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(market.Layer(6)); if(CheckPointer(context) == POINTER_INVALID || !forecast || forecast.Type() != defNeuronScenarioForecastPriceOCL || forecast.ABI() != defScenarioForecastPriceABI || forecast.GetOpenCL() != context || forecast.TrainMode() || forecast.CodebookUpdate() || forecast.Scenarios() == 0 || forecast.Variables() == 0 || forecast.Horizon() == 0 || forecast.Dimension() == 0 || forecast.Dimension() > (uint)INT_MAX / 5 || forecast.GetTopK() == 0 || forecast.GetTopK() > forecast.Scenarios()) { reason = "model.price_q.forecast_contract"; return(false); } //--- Native Cross maps five account/action queries to one distributional SRM head. CArrayObj descriptions; descriptions.FreeMode(true); const uint types[] = {defNeuronBaseOCL, defNeuronScenarioCrossAttentionPriceOCL, defNeuronConvOCL, defNeuronBaseOCL, defNeuronACSRM }; const uint counts[] = {19, forecast.Scenarios(), 1, 32, 1}; for(uint i = 0; i < 5; i++) { CLayerDescription *description = new CLayerDescription; if(!description) { reason = "model.price_q.description_allocation"; return(false); } if(!DPCCSetLayerBaseDefaults(description, types[i], counts[i], 1)) { delete description; reason = "model.price_q.description_defaults"; return(false); } if(i == 1) { description.window = forecast.Dimension(); description.window_out = forecast.Horizon(); description.layers = forecast.Variables(); description.variables = 5; description.step = stack_size; description.probability = (float)forecast.GetTopK(); } else if(i == 2) { description.window = 5 * forecast.Dimension(); description.step = description.window; description.window_out = forecast.Dimension(); description.layers = 1; description.activation = GELU; } else if(i == 4) description.window_out = 32; if(!descriptions.Add(description)) { delete description; reason = "model.price_q.description_add"; return(false); } } //--- Each invocation initializes fresh weights; only the OpenCL context is borrowed. CNet *created = NULL; if(!DPCCCreateNetInContext(created, descriptions, context, reason)) return(false); CNeuronScenarioCrossAttentionPriceOCL *cross = dynamic_cast(created.Layer(1)); if(!NNQCriticGraph(created) || created.GetOpenCL() != context || !cross || cross.Type() != defNeuronScenarioCrossAttentionPriceOCL || cross.ABI() != defScenarioCrossAttentionPriceABI || cross.Scenarios() != forecast.Scenarios() || cross.Variables() != forecast.Variables() || cross.Horizon() != forecast.Horizon() || cross.Latent() != forecast.Dimension() || cross.HistorySize() != stack_size || cross.HistoryTopK() != forecast.GetTopK()) reason = "model.price_q.created_graph"; else if(!created.TrainMode(false) || !created.SetWeightsUpdate(false)) reason = "model.price_q.initial_mode"; else { //--- The caller owns this CNet and retains Market's context until it is deleted. net = created; return(true); } delete created; return(false); } //+------------------------------------------------------------------+ //| Fit one Price Critic distribution to a caller-provided return | //+------------------------------------------------------------------+ bool DPCCUpdateAction6Critic(CNet &critic, CNet &market, const float &account_action19[], const float target, const float probability_weight, const float value_weight, string &reason) { reason = ""; if(GetPointer(critic) == GetPointer(market) || !NNQCriticGraph(GetPointer(critic)) || !market.IsOpenCLTransferReady() || market.WeightsUpdateEnabled() || !critic.WeightsUpdateEnabled() || critic.GetOpenCL() != market.GetOpenCL() || ArraySize(account_action19) != 19 || !MathIsValidNumber(target) || !MathIsValidNumber(probability_weight) || probability_weight < 0 || !MathIsValidNumber(value_weight) || value_weight < 0) { reason = "model.price_q.update_contract"; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(market.Layer(6)); CNeuronScenarioCrossAttentionPriceOCL *cross = dynamic_cast(critic.Layer(1)); CNeuronBaseOCL *base = critic.Layer(3); CNeuronACSRM *head = dynamic_cast(critic.Layer(4)); if(!forecast || forecast.Type() != defNeuronScenarioForecastPriceOCL || forecast.ABI() != defScenarioForecastPriceABI || !cross || cross.Type() != defNeuronScenarioCrossAttentionPriceOCL || cross.ABI() != defScenarioCrossAttentionPriceABI || !cross.ValidPolicyForecast(forecast) || !head || !NNQValid(critic.GetOpenCL(), base.getOutput(), 32) || !NNQValid(critic.GetOpenCL(), base.getGradient(), 32)) { reason = "model.price_q.update_price_stream"; return(false); } for(int i = 1; i < 5; i++) if(!critic.Layer(i).TrainMode()) { reason = "model.price_q.update_frozen_layer"; return(false); } //--- Validate and pack the recorded Account13/Action6 tuple before touching the Critic. CBufferFloat query; for(int i = 0; i < 19; i++) { if(!MathIsValidNumber(account_action19[i])) { reason = "model.price_q.update_input_nonfinite"; return(false); } if(!query.Add(account_action19[i])) { reason = "model.price_q.update_input_allocation"; return(false); } } if(!critic.feedForward(GetPointer(query), 1, false, GetPointer(market), 6)) { reason = "model.price_q.update_forward"; return(false); } //--- The native AC-SRM target prepares the Base32 boundary for its specialized backward. if(!head.calcDistributionTargetGradients(base, target, probability_weight, value_weight)) { reason = "model.price_q.update_distribution_gradient"; return(false); } if(!critic.backPropGradient(GetPointer(market), 6, -1, false)) { reason = "model.price_q.update_backward"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Update the ABI3 Actor from its selected native SRM gradient | //+------------------------------------------------------------------+ bool DPCCUpdateAction6Actor(CNet &actor, const ulong decision_id, const uint critic_index, string &reason) { reason = ""; if(IsStopped()) { reason = "model.action6.update_stopped"; return(false); } CNeuronDPCCOCL *top = dynamic_cast(actor.Layer(1)); if(!actor.IsOpenCLTransferReady() || !actor.WeightsUpdateEnabled() || !top || actor.Layer(2) != NULL || top.Type() != defNeuronDPCCOCL || top.ActorABI() != 3 || top.GetOpenCL() != actor.GetOpenCL() || !top.TrainMode() || !top.HasValidDecision() || top.LastStatus() != DPCC_OK || decision_id == 0 || critic_index > 1) { reason = "model.action6.update_contract"; return(false); } //--- Binding consumes policy state; any subsequent failure requires caller restoration. if(!top.BindSelectedPolicyGradient(decision_id, critic_index, -1.0f)) { reason = "model.action6.update_policy_gradient_non_retryable"; return(false); } if(IsStopped()) { reason = "model.action6.update_stopped_after_bind_non_retryable"; return(false); } //--- The saved decision owns its condition; this pass applies one accumulated Adam update. if(!actor.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 1, false)) { reason = "model.action6.update_backward_non_retryable"; return(false); } if(top.LastStatus() != DPCC_OK || top.HasValidDecision()) { reason = "model.action6.update_incomplete_non_retryable"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Fit the current Action6 to the factual-horizon Oracle operation | //+------------------------------------------------------------------+ bool DPCCUpdateAction6OracleActor(CNet &actor, const ulong decision_id, const float &teacher_action6[], string &reason) { reason = ""; if(IsStopped()) { reason = "model.action6.oracle_stopped"; return(false); } CNeuronDPCCOCL *top = dynamic_cast(actor.Layer(1)); if(!actor.IsOpenCLTransferReady() || !actor.WeightsUpdateEnabled() || !top || actor.Layer(2) != NULL || top.Type() != defNeuronDPCCOCL || top.ActorABI() != 3 || top.GetOpenCL() != actor.GetOpenCL() || !top.TrainMode() || !top.HasValidDecision() || top.LastStatus() != DPCC_OK || decision_id == 0 || ArraySize(teacher_action6) != 6) { reason = "model.action6.oracle_contract"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(teacher_action6[i])) { reason = "model.action6.oracle_target_nonfinite"; return(false); } //--- Preserve the selected Actor branch while preparing its physical Action6 loss. uint selected = UINT_MAX; { reason = "model.action6.oracle_selected_candidate"; return(false); } COpenCLMy *context = actor.GetOpenCL(); CBufferFloat *current = top.getOutput(); if(!NNQValid(context, current, 6)) { reason = "model.action6.oracle_buffers"; return(false); } CBufferFloat target; if(!target.AssignArray(teacher_action6)) { reason = "model.action6.oracle_target_allocation"; return(false); } if(!target.BufferCreate(context)) { reason = "model.action6.oracle_target_context"; return(false); } if(!target.BufferWrite()) { reason = "model.action6.oracle_target_write"; return(false); } CBufferFloat gradient; if(!gradient.BufferInit(6, 0)) { reason = "model.action6.oracle_gradient_allocation"; return(false); } if(!gradient.BufferCreate(context)) { reason = "model.action6.oracle_gradient_context"; return(false); } if(!DPCCActorGradientDifferent(context, GetPointer(target), current, GetPointer(gradient), 6, reason)) return(false); if(IsStopped()) { reason = "model.action6.oracle_stopped_before_bind"; return(false); } //--- The owned reverse unroll applies projection and denoiser derivatives once. if(!top.BindAction6Gradient(decision_id, selected, GetPointer(gradient))) { reason = "model.action6.oracle_gradient_non_retryable"; return(false); } if(IsStopped()) { reason = "model.action6.oracle_stopped_after_bind_non_retryable"; return(false); } if(!actor.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 1, false)) { reason = "model.action6.oracle_backward_non_retryable"; return(false); } if(top.LastStatus() != DPCC_OK || top.HasValidDecision()) { reason = "model.action6.oracle_update_incomplete_non_retryable"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Builds a discounted full-plan Action6 Oracle gradient | //+------------------------------------------------------------------+ bool DPCCSelectClosestAction6OracleCandidate(CNeuronDPCCOCL &actor, const float &teacher_plan[], const float &valid_steps[], const float gamma, uint &selected_candidate, bool &found_candidate, string &reason) { selected_candidate = UINT_MAX; found_candidate = false; reason = ""; const int horizon = ArraySize(valid_steps); const long row = (long)horizon * 6; if(horizon <= 0 || row <= 0 || row > INT_MAX || ArraySize(teacher_plan) != (int)row || !MathIsValidNumber(gamma) || gamma < 0.0f || gamma > 1.0f) { reason = "policy.action6_oracle.nearest_shape"; return(false); } COpenCLMy *context = actor.GetOpenCL(); CBufferFloat *plans = NULL; uchar eligible[]; uint plan_horizon = 0; if(CheckPointer(context) == POINTER_INVALID || !actor.GetOracleCandidateMask(eligible, plan_horizon) || !actor.GetCheckedPlans(plans) || CheckPointer(plans) == POINTER_INVALID || plan_horizon != (uint)horizon || ArraySize(eligible) <= 0 || plans.GetOpenCL() != context || plans.GetIndex() < 0) { reason = StringFormat( "policy.action6_oracle.nearest_candidates context=%d horizon=%u/%d eligible=%d plans=%d plans_context=%d plan_index=%d", (CheckPointer(context) != POINTER_INVALID ? 1 : 0), plan_horizon, horizon, ArraySize(eligible), (CheckPointer(plans) != POINTER_INVALID ? 1 : 0), (CheckPointer(plans) != POINTER_INVALID && plans.GetOpenCL() == context ? 1 : 0), (CheckPointer(plans) != POINTER_INVALID ? plans.GetIndex() : -1)); return(false); } const uint candidate_count = (uint)ArraySize(eligible); const ulong total = (ulong)candidate_count * (ulong)row; if(total == 0 || total > INT_MAX || plans.Total() != (int)total) { reason = "policy.action6_oracle.nearest_candidate_shape"; return(false); } float candidates[]; if(ArrayResize(candidates, (int)total) != (int)total || !context.BufferRead(plans.GetIndex(), candidates, 0, 0, (uint)total)) { reason = "policy.action6_oracle.nearest_candidate_read"; return(false); } double discounts[]; if(ArrayResize(discounts, horizon) != horizon) { reason = "policy.action6_oracle.nearest_discount_capacity"; return(false); } double discount = 1.0; double normalization = 0.0; for(int h = 0; h < horizon; h++) { if((valid_steps[h] != 0.0f && valid_steps[h] != 1.0f) || !MathIsValidNumber((double)discount)) { reason = StringFormat("policy.action6_oracle.nearest_mask.%d", h); return(false); } discounts[h] = discount; normalization += (double)valid_steps[h] * discount; discount *= (double)gamma; } if(!MathIsValidNumber(normalization) || normalization <= 0.0) { reason = "policy.action6_oracle.nearest_no_valid_horizon"; return(false); } for(int i = 0; i < (int)row; i++) if(!MathIsValidNumber((double)teacher_plan[i])) { reason = StringFormat("policy.action6_oracle.nearest_target_nonfinite.%d", i); return(false); } bool found = false; double best_distance = 0.0; for(uint candidate = 0; candidate < candidate_count; candidate++) { if(eligible[candidate] > 1) { reason = StringFormat("policy.action6_oracle.nearest_eligibility.%d", candidate); return(false); } if(eligible[candidate] == 0) continue; double distance = 0.0; for(int h = 0; h < horizon; h++) { if(valid_steps[h] == 0.0f) continue; for(int coordinate = 0; coordinate < 6; coordinate++) { const ulong index = (ulong)candidate * (ulong)row + (ulong)h * 6 + (ulong)coordinate; const float value = candidates[(int)index]; if(!MathIsValidNumber((double)value)) { reason = StringFormat("policy.action6_oracle.nearest_candidate_nonfinite.%d.%d", candidate, h * 6 + coordinate); return(false); } const double delta = (double)value - (double)teacher_plan[h * 6 + coordinate]; distance += discounts[h] * delta * delta; } } if(!MathIsValidNumber(distance)) { reason = StringFormat("policy.action6_oracle.nearest_distance_nonfinite.%d", candidate); return(false); } //--- Stable tie-break: the first eligible candidate wins an exact tie. if(!found || distance < best_distance) { found = true; best_distance = distance; selected_candidate = candidate; } } found_candidate = found; return(true); } //+------------------------------------------------------------------+ //| Function DPCCPrepareAction6OraclePlanGradient | //+------------------------------------------------------------------+ bool DPCCPrepareAction6OraclePlanGradient(CNeuronDPCCOCL &actor, const uint selected_candidate, const float &teacher_plan[], const float &valid_steps[], const float gamma, CBufferFloat *gradient, string &reason) { reason = ""; const int horizon = ArraySize(valid_steps); const long count = (long)horizon * 6; COpenCLMy *context = actor.GetOpenCL(); CBufferFloat *plans = NULL; uint plan_candidates = 0, plan_horizon = 0; if(horizon <= 0 || count <= 0 || count > INT_MAX || ArraySize(teacher_plan) != (int)count || !MathIsValidNumber(gamma) || gamma < 0.0f || gamma > 1.0f) { reason = "policy.action6_oracle.gradient_shape"; return(false); } const bool oracle_fallback = (!actor.HasValidDecision() && actor.LastStatus() == DPCC_NO_FEASIBLE_CANDIDATE); const bool oracle_decision = (actor.HasValidDecision() && actor.LastStatus() == DPCC_OK); uchar eligible[]; bool plan_state = (oracle_fallback || oracle_decision) && actor.GetOracleCandidateMask(eligible, plan_horizon); plan_candidates = (uint)ArraySize(eligible); plan_state = plan_state && plan_horizon == (uint)horizon && selected_candidate < plan_candidates && eligible[selected_candidate] == 1; if(!actor.GetCheckedPlans(plans) || !plan_state || CheckPointer(context) == POINTER_INVALID || CheckPointer(plans) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || plan_horizon != (uint)horizon || plan_candidates == 0 || plans == gradient || plans.GetOpenCL() != context || gradient.GetOpenCL() != context || plans.GetIndex() < 0 || gradient.GetIndex() < 0 || plans.GetIndex() == gradient.GetIndex() || plans.Total() <= 0 || (ulong)plans.Total() % (ulong)count != 0 || (ulong)plans.Total() / (ulong)count != (ulong)plan_candidates || selected_candidate >= plan_candidates || gradient.Total() != (int)count) { reason = "policy.action6_oracle.gradient_contract"; return(false); } const ulong source_offset = (ulong)selected_candidate * (ulong)count; if(source_offset > (ulong)UINT_MAX / sizeof(float) || source_offset + (ulong)count > (ulong)plans.Total()) { reason = "policy.action6_oracle.selected_plan_range"; return(false); } float current_values[]; if(ArrayResize(current_values, (int)count) != (int)count || !context.BufferRead(plans.GetIndex(), current_values, (uint)(source_offset * sizeof(float)), 0, (uint)count)) { reason = "policy.action6_oracle.selected_plan_read"; return(false); } for(int i = 0; i < (int)count; i++) if(!MathIsValidNumber(current_values[i])) { reason = StringFormat("policy.action6_oracle.selected_plan_nonfinite.%d", i); return(false); } double discounts[]; if(ArrayResize(discounts, horizon) != horizon) { reason = "policy.action6_oracle.discount_capacity"; return(false); } double discount = 1.0; double normalization = 0.0; for(int h = 0; h < horizon; h++) { if(valid_steps[h] != 0.0f && valid_steps[h] != 1.0f) { reason = StringFormat("policy.action6_oracle.valid_step.%d", h); return(false); } discounts[h] = discount; normalization += (double)valid_steps[h] * discount; discount *= (double)gamma; } if(!MathIsValidNumber(normalization) || normalization <= 0.0) { reason = "policy.action6_oracle.no_discounted_valid_steps"; return(false); } float weights[], zeros[]; if(ArrayResize(weights, (int)count) != (int)count || ArrayResize(zeros, (int)count) != (int)count) { reason = "policy.action6_oracle.weight_capacity"; return(false); } for(int i = 0; i < (int)count; i++) { if(!MathIsValidNumber(teacher_plan[i])) { reason = StringFormat("policy.action6_oracle.target_nonfinite.%d", i); return(false); } const int h = i / 6; weights[i] = (float)((double)valid_steps[h] * discounts[h] / normalization); zeros[i] = 0.0f; if(!MathIsValidNumber((double)weights[i])) { reason = StringFormat("policy.action6_oracle.weight_nonfinite.%d", i); return(false); } } CBufferFloat target, weight, zero, error, current; if(!target.AssignArray(teacher_plan) || !current.AssignArray(current_values) || !weight.AssignArray(weights) || !zero.AssignArray(zeros) || !error.BufferInit((uint)count, 0.0f)) { reason = "policy.action6_oracle.scratch_init"; return(false); } CBufferFloat *buffers[] = {GetPointer(target), GetPointer(weight), GetPointer(zero), GetPointer(error), GetPointer( current)}; for(int i = 0; i < ArraySize(buffers); i++) if(!buffers[i].BufferCreate(context) || !buffers[i].BufferWrite()) { reason = StringFormat("policy.action6_oracle.scratch_opencl.%d", i); return(false); } for(int i = 0; i < ArraySize(buffers); i++) { const int index = buffers[i].GetIndex(); if(index < 0 || index == plans.GetIndex() || index == gradient.GetIndex()) { reason = "policy.action6_oracle.scratch_alias"; return(false); } for(int j = 0; j < i; j++) if(index == buffers[j].GetIndex()) { reason = "policy.action6_oracle.scratch_alias"; return(false); } } if(!DPCCActorGradientDifferent(context, GetPointer(target), GetPointer(current), GetPointer(error), (uint)count, reason) || !DPCCActorGradientGate(context, GetPointer(error), GetPointer(zero), GetPointer(weight), gradient, (uint)count, reason)) { if(reason == "") reason = "policy.action6_oracle.gradient_dispatch"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Applies one discounted Oracle update across the complete H plan | //+------------------------------------------------------------------+ bool DPCCUpdateAction6OraclePlanActor(CNet &actor, const ulong decision_id, const float &teacher_plan[], const float &valid_steps[], const float gamma, string &reason) { reason = ""; CNeuronDPCCOCL *top = dynamic_cast(actor.Layer(1)); const int horizon = ArraySize(valid_steps); const long plan_count = (long)horizon * 6; if(IsStopped() || !actor.IsOpenCLTransferReady() || !actor.WeightsUpdateEnabled() || CheckPointer(top) == POINTER_INVALID || actor.Layer(2) != NULL || top.Type() != defNeuronDPCCOCL || top.ActorABI() != 3 || top.GetOpenCL() != actor.GetOpenCL() || !top.TrainMode() || decision_id == 0 || horizon <= 0 || plan_count > INT_MAX || ArraySize(teacher_plan) != (int)plan_count || !MathIsValidNumber(gamma) || gamma < 0.0f || gamma > 1.0f) { reason = "model.action6.oracle_plan_contract"; return(false); } const bool oracle_fallback = (!top.HasValidDecision() && top.LastStatus() == DPCC_NO_FEASIBLE_CANDIDATE); const bool oracle_decision = (top.HasValidDecision() && top.LastStatus() == DPCC_OK); if(!oracle_fallback && !oracle_decision) { reason = "model.action6.oracle_plan_decision"; return(false); } uint selected = UINT_MAX; bool candidate_found = false; if(!DPCCSelectClosestAction6OracleCandidate(*top, teacher_plan, valid_steps, gamma, selected, candidate_found, reason)) return(false); if(!candidate_found) { const bool cleaned = (oracle_fallback ? top.AbortNoFeasibleAction6Unroll() : top.CancelGeneration()); if(!cleaned) { reason = "model.action6.oracle_plan_no_candidate_cleanup"; return(false); } reason = ""; return(true); } CBufferFloat gradient; if(!gradient.BufferInit((uint)plan_count, 0.0f)) { reason = "model.action6.oracle_plan_gradient_allocation"; return(false); } if(!gradient.BufferCreate(actor.GetOpenCL())) { reason = "model.action6.oracle_plan_gradient_context"; return(false); } if(!DPCCPrepareAction6OraclePlanGradient(*top, selected, teacher_plan, valid_steps, gamma, GetPointer(gradient), reason)) return(false); if(IsStopped()) { reason = "model.action6.oracle_plan_stopped_before_bind"; return(false); } if(!top.BindFullPlanGradient(decision_id, selected, GetPointer(gradient))) { reason = "model.action6.oracle_plan_gradient_non_retryable"; return(false); } if(IsStopped()) { reason = "model.action6.oracle_plan_stopped_after_bind_non_retryable"; return(false); } if(!actor.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 1, false)) { reason = "model.action6.oracle_plan_backward_non_retryable"; return(false); } if(top.LastStatus() != DPCC_OK || top.HasValidDecision()) { reason = "model.action6.oracle_plan_update_incomplete_non_retryable"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCAssembleActor | //+------------------------------------------------------------------+ bool DPCCAssembleActor(CNet &net, CNeuronScenarioDecoderOCL *decoder, CNeuronPlanDenoiserOCL *denoiser, CNeuronBatchNormOCL *normalizer, CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; reason = ""; CNeuronDPCCOCL *candidate = dynamic_cast(net.Layer(1)); //--- Refuse invalid borrowed objects before touching the existing destination. if(CheckPointer(candidate) == POINTER_INVALID || CheckPointer(decoder) == POINTER_INVALID || CheckPointer(denoiser) == POINTER_INVALID || CheckPointer(normalizer) == POINTER_INVALID) {reason = "model.actor_preflight_pointer"; return(false);} if(candidate.Type() != defNeuronDPCCOCL || decoder.Type() != defNeuronScenarioDecoderOCL || denoiser.Type() != defNeuronPlanDenoiserOCL || normalizer.Type() != defNeuronBatchNormOCL || decoder.GetOpenCL() != net.GetOpenCL() || denoiser.GetOpenCL() != net.GetOpenCL() || normalizer.GetOpenCL() != net.GetOpenCL()) {reason = "model.actor_preflight_contract"; return(false);} //--- Native import revokes publication and can fail after partial parameter copy. //--- Preserve that failure state: the caller must recreate or reload this net. if(!candidate.ImportPreparedComponents(decoder, denoiser, normalizer)) {reason = "model.actor_import"; return(false);} net.TrainMode(false); net.SetWeightsUpdate(false); actor = candidate; return(true); } //+------------------------------------------------------------------+ //| Function DPCCAssembleSelectedCheckpoints | //+------------------------------------------------------------------+ bool DPCCAssembleSelectedCheckpoints(CNet &net, const SDPCCConfig &training, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder_composition, const SDPCCDenoiserComposition &composition, const double &beta[], const double &alpha[], const double &sigma[], const SDPCCCheckpointArtifact &selected_decoder, const SDPCCCheckpointArtifact &selected_denoiser, const string expected_binding, const float &price_norm[], const ulong maximum_bytes, CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; reason = ""; if(net.Layer(0) != NULL || selected_decoder.binding_sha256 != expected_binding || selected_denoiser.binding_sha256 != expected_binding || selected_decoder.common_files != selected_denoiser.common_files || selected_decoder.initial_manifest_path != selected_denoiser.initial_manifest_path || selected_decoder.initial_manifest_sha256 != selected_denoiser.initial_manifest_sha256 || selected_decoder.cursor.stage != 1 || selected_denoiser.cursor.stage != 2 || selected_decoder.cursor.decoder_updates == 0 || selected_denoiser.cursor.denoiser_updates == 0 || ArraySize( price_norm) != 16) {reason = "actor.selected_owner_binding"; return(false);} SDPCCCheckpointArtifact actual_decoder, actual_denoiser; const bool common = selected_decoder.common_files; if(!DPCCReadCheckpointArtifact(selected_decoder.manifest_path, common, selected_decoder.manifest_sha256, actual_decoder, reason) || !DPCCReadCheckpointArtifact(selected_denoiser.manifest_path, common, selected_denoiser.manifest_sha256, actual_denoiser, reason) || DPCCCheckpointManifest(actual_decoder) != DPCCCheckpointManifest(selected_decoder) || DPCCCheckpointManifest(actual_denoiser) != DPCCCheckpointManifest(selected_denoiser)) return(false); string sha = ""; if(!DPCCCheckpointFileSHA(actual_decoder.decoder_path, common, maximum_bytes, sha, reason) || sha != actual_decoder.decoder_sha256 || !DPCCCheckpointFileSHA(actual_denoiser.denoiser_path, common, maximum_bytes, sha, reason) || sha != actual_denoiser.denoiser_sha256) {reason = "actor.selected_native_sha"; return(false);} CNeuronDPCCOCL *prepared_candidate = NULL; if(!DPCCCreateActorNet(net, config, decoder_composition, composition, composition, beta, alpha, sigma, 1, NULL, prepared_candidate, reason)) return(false); CNet decoder_net, denoiser_net; CNeuronScenarioDecoderOCL *decoder = NULL; CNeuronPlanDenoiserOCL *denoiser = NULL; SDPCCConfig decoder_config = training; decoder_config.shape.Depth = 0; if(!DPCCCreateDecoderNet(decoder_net, decoder_config, decoder_composition, 1, net.GetOpenCL(), decoder, reason) || !DPCCCreateDenoiserNet(denoiser_net, training, composition, composition, 1, net.GetOpenCL(), denoiser, reason)) return(false); const ulong decoder_contract = decoder.ContractSignature(), denoiser_contract = denoiser.ContractSignature(); float error = 0, undefined = 0, forecast = 0; datetime time = 0; if(!decoder_net.Load(actual_decoder.decoder_path, error, undefined, forecast, time, common) || !denoiser_net.Load(actual_denoiser.denoiser_path, error, undefined, forecast, time, common)) {reason = "actor.native_load"; return(false);} decoder = dynamic_cast(decoder_net.Layer(1)); denoiser = dynamic_cast(denoiser_net.Layer(1)); if(!decoder || !denoiser || decoder_net.Layer(2) != NULL || denoiser_net.Layer(2) != NULL || decoder.Type() != defNeuronScenarioDecoderOCL || denoiser.Type() != defNeuronPlanDenoiserOCL || decoder.GetOpenCL() != net.GetOpenCL() || denoiser.GetOpenCL() != net.GetOpenCL() || decoder.ContractSignature() != decoder_contract || denoiser.ContractSignature() != denoiser_contract || decoder.GenerationPhase() != 0 || denoiser.GenerationPhase() != 0 || decoder.UpdateCount() != actual_decoder.cursor.decoder_updates || denoiser.UpdateCount() != actual_denoiser.cursor.denoiser_updates || decoder.TrainableStateFingerprint() != actual_decoder.decoder_state || denoiser.TrainableStateFingerprint() != actual_denoiser.denoiser_state) {reason = "actor.loaded_native_full_state"; return(false);} for(uint role = 0; role < 2; role++) { CNet *donor = (role == 0 ? GetPointer(decoder_net) : GetPointer(denoiser_net)); CLayerDescription *input_info = donor.Layer(0).GetLayerInfo(), *layer = donor.Layer(1).GetLayerInfo(); const uint count = (role == 0 ? training.shape.K * training.shape.V * training.shape.H*training.shape.D : training.shape.H * 6); const bool descriptor = input_info != NULL && layer != NULL && input_info.type == defNeuronBaseOCL && input_info.count == count && input_info.batch == 1 && input_info.optimization == ADAM && layer.batch == 1 && layer.optimization == ADAM && layer.layers == (role == 0 ? 0 : 1) /* native public descriptor is fixed, graph depth is serialized separately */ && ArraySize(layer.windows) == 9 && layer.windows[8] == 1; delete layer; delete input_info; if(!descriptor) { reason = "actor.loaded_native_descriptor"; return(false); } } const ulong df = decoder.TrainableStateFingerprint(), nf = denoiser.TrainableStateFingerprint(); //--- Restore the actual admitted Model financial profile while retaining native L=1 geometry. SDPCCConfig donor_config = config; donor_config.shape.PlanBatch = training.shape.PlanBatch; donor_config.shape.Depth = training.shape.Depth; if(!denoiser.Configure(donor_config, composition, composition) || denoiser.TrainableStateFingerprint() != nf || denoiser.UpdateCount() != actual_denoiser.cursor.denoiser_updates || denoiser.ContractSignature() != denoiser_contract) {reason = "actor.loaded_denoiser_configure"; return(false);} CNeuronBatchNormOCL normalizer; if(!normalizer.Init(0, 0, net.GetOpenCL(), uint(4), uint(2), ADAM)) { reason = "actor.normalizer_init"; return(false); } CBufferFloat *options = normalizer.getBatchOptions(); if(!options || options.Total() != 36 || !options.Fill(0)) { reason = "actor.normalizer_capacity"; return(false); } for(int i = 0; i < 4; i++) { const float mean = price_norm[i * 4], std = price_norm[i * 4 + 1], variance = std * std; if(!MathIsValidNumber(mean) || !MathIsValidNumber(std) || std <= 0 || !MathIsValidNumber(variance) || variance <= 0 || price_norm[i * 4 + 2] != 1 || price_norm[i * 4 + 3] != 0 || !options.Update(i * 9, mean) || !options.Update(i * 9 + 1, variance) || !options.Update(i * 9 + 3, 1)) {reason = "actor.normalizer_causal_gain1"; return(false);} } if(!options.BufferWrite() || !normalizer.SetBatchCount(1)) { reason = "actor.normalizer_upload"; return(false); } decoder_net.TrainMode(false); denoiser_net.TrainMode(false); normalizer.TrainMode(false); if(!DPCCAssembleActor(net, decoder, denoiser, GetPointer(normalizer), prepared_candidate, reason) || decoder.TrainableStateFingerprint() != df || denoiser.TrainableStateFingerprint() != nf) {if(StringLen(reason) == 0)reason = "actor.import_donor_mutation"; return(false);} if(!DPCCCheckpointFileSHA(actual_decoder.decoder_path, common, maximum_bytes, sha, reason) || sha != actual_decoder.decoder_sha256 || !DPCCCheckpointFileSHA(actual_denoiser.denoiser_path, common, maximum_bytes, sha, reason) || sha != actual_denoiser.denoiser_sha256) {reason = "actor.selected_native_changed"; return(false);} actor = prepared_candidate; return(true); } //+------------------------------------------------------------------+ //| Pure ACCOUNT13 transform shared by historical and live callers | //+------------------------------------------------------------------+ bool DPCCBuildAccount13Features(const double balance, const double previous_balance, const double equity, const double previous_equity, const double buy_lots, const double sell_lots, const double buy_profit, const double sell_profit, const double position_discount, const long server_time, float &features[], string &reason) { ArrayFree(features); reason = ""; const double inputs[] = {balance, previous_balance, equity, previous_equity, buy_lots, sell_lots, buy_profit, sell_profit, position_discount }; for(int i = 0; i < ArraySize(inputs); i++) if(!MathIsValidNumber(inputs[i])) { reason = "account13.nonfinite_input"; return(false); } if(previous_balance <= 0 || previous_equity <= 0) { reason = "account13.previous_reference_nonpositive"; return(false); } const int year_seconds = (int)(D'2024.01.01' - D'2023.01.01'); const int month_seconds = PeriodSeconds(PERIOD_MN1); const int week_seconds = PeriodSeconds(PERIOD_W1); const int day_seconds = PeriodSeconds(PERIOD_D1); if(year_seconds <= 0 || month_seconds <= 0 || week_seconds <= 0 || day_seconds <= 0) { reason = "account13.period_seconds"; return(false); } float values[13]; values[0] = (float)(balance / 10000.0); values[1] = (float)((balance - previous_balance) / previous_balance); values[2] = (float)(equity / previous_balance); values[3] = (float)((equity - previous_equity) / previous_equity); values[4] = (float)buy_lots; values[5] = (float)sell_lots; values[6] = (float)(buy_profit / previous_balance); values[7] = (float)(sell_profit / previous_balance); values[8] = (float)(position_discount / previous_balance); double phase = (double)server_time / (double)year_seconds; values[9] = (float)MathSin(phase != 0 ? 2.0 * M_PI*phase : 0); phase = (double)server_time / (double)month_seconds; values[10] = (float)MathCos(phase != 0 ? 2.0 * M_PI*phase : 0); phase = (double)server_time / (double)week_seconds; values[11] = (float)MathSin(phase != 0 ? 2.0 * M_PI*phase : 0); phase = (double)server_time / (double)day_seconds; values[12] = (float)MathSin(phase != 0 ? 2.0 * M_PI*phase : 0); for(int i = 0; i < ArraySize(values); i++) if(!MathIsValidNumber((double)values[i])) { reason = "account13.float_range"; return(false); } if(ArrayResize(features, ArraySize(values)) != ArraySize(values)) { ArrayFree(features); reason = "account13.output_capacity"; return(false); } for(int i = 0; i < ArraySize(values); i++) features[i] = values[i]; return(true); } //+------------------------------------------------------------------+ //| Metadata for one isolated simulated net position | //+------------------------------------------------------------------+ struct SDPCCSimulatedPositionMetadata { bool initialized; long snapshot_bar, snapshot_server, snapshot_time; string time_domain; long bar_server_open, bar_server_close, bar_open_time, bar_close_time; long first_open_bar, first_open_server, first_open_time; float signed_lots; }; //+------------------------------------------------------------------+ //| Calculation modes implemented by the native DPCC rollout kernel | //+------------------------------------------------------------------+ bool DPCCSupportedHistoricalCalculationMode(const ENUM_SYMBOL_CALC_MODE calculation_mode) { switch(calculation_mode) { case SYMBOL_CALC_MODE_FOREX: case SYMBOL_CALC_MODE_FOREX_NO_LEVERAGE: case SYMBOL_CALC_MODE_FUTURES: case SYMBOL_CALC_MODE_CFD: case SYMBOL_CALC_MODE_CFDINDEX: case SYMBOL_CALC_MODE_CFDLEVERAGE: case SYMBOL_CALC_MODE_EXCH_STOCKS: case SYMBOL_CALC_MODE_EXCH_FUTURES: case SYMBOL_CALC_MODE_EXCH_FUTURES_FORTS: case SYMBOL_CALC_MODE_EXCH_BONDS: case SYMBOL_CALC_MODE_EXCH_STOCKS_MOEX: case SYMBOL_CALC_MODE_EXCH_BONDS_MOEX: case SYMBOL_CALC_MODE_SERV_COLLATERAL: return(true); } return(false); } bool DPCCSupportedHistoricalCalculationModeValue(const float calculation_mode) { if(!MathIsValidNumber((double)calculation_mode) || calculation_mode < 0.0f || calculation_mode > (float)INT_MAX || MathFloor((double)calculation_mode) != (double)calculation_mode) return(false); return(DPCCSupportedHistoricalCalculationMode((ENUM_SYMBOL_CALC_MODE)(int)calculation_mode)); } //+------------------------------------------------------------------+ //| Builds Account13 from exact forex snapshots and indexed bar | //| fields | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalAccount13(const long bar_ordinal, const long server_open, const long server_close, const long open_time, const long close_time, const long snapshot_server, const ENUM_SYMBOL_CALC_MODE calculation_mode, const float &previous_state[], const float ¤t_state[], const SDPCCSimulatedPositionMetadata &previous_position, SDPCCSimulatedPositionMetadata &position, float &features[], string &reason, const string time_domain = "UTC_SECONDS_V1") { ZeroMemory(position); ArrayResize(features, 0); reason = ""; //--- Timeline fields retain the explicit domain supplied by ClockBinding. if((time_domain != "UTC_SECONDS_V1" && time_domain != "SERVER_CALENDAR_SECONDS_V1") || (time_domain == "SERVER_CALENDAR_SECONDS_V1" && (open_time != server_open || close_time != server_close))) {reason = "account13.history.time_domain"; return(false);} if(bar_ordinal < 0 || server_open <= 0 || server_close <= server_open || open_time <= 0 || close_time <= open_time || (snapshot_server != server_open && snapshot_server != server_close)) { reason = "account13.history.exact_snapshot_time_required"; return(false); } if(ArraySize(previous_state) != 33 || ArraySize(current_state) != 33 || !DPCCSupportedHistoricalCalculationMode(calculation_mode)) { reason = "account13.history.native_calculation_mode_required"; return(false); } for(int i = 0; i < 33; i++) if(!MathIsValidNumber((double)previous_state[i]) || !MathIsValidNumber((double)current_state[i])) { reason = "account13.history.nonfinite_snapshot"; return(false); } if(previous_state[24] != 0 || current_state[24] != 0 || previous_state[32] != 0 || current_state[32] != 0 || previous_state[0] <= 0 || previous_state[1] <= 0 || previous_state[6] < 0 || current_state[6] < 0) { reason = "account13.history.isolated_valid_snapshot_required"; return(false); } const long snapshot_time = (snapshot_server == server_open ? open_time : close_time); if(previous_position.initialized) { if(previous_position.time_domain != time_domain) {reason = "account13.history.time_domain_changed"; return(false);} const bool next_open = (previous_position.snapshot_bar < LONG_MAX && bar_ordinal == previous_position.snapshot_bar + 1 && previous_position.snapshot_server == previous_position.bar_server_close && previous_position.snapshot_time == previous_position.bar_close_time && snapshot_server == server_open && open_time >= previous_position.bar_close_time); const bool same_bar_close = (bar_ordinal == previous_position.snapshot_bar && previous_position.snapshot_server == server_open && previous_position.snapshot_time == open_time && snapshot_server == server_close && previous_position.bar_server_open == server_open && previous_position.bar_server_close == server_close && previous_position.bar_open_time == open_time && previous_position.bar_close_time == close_time); if((!next_open && !same_bar_close) || previous_position.signed_lots != previous_state[2]) { reason = "account13.history.snapshot_phase_join"; return(false); } } else if(previous_state[2] != 0 || current_state[2] != 0 || snapshot_server != server_open) { reason = "account13.history.initial_flat_open_required"; return(false); } //--- Preserve first exposure across additions; flat and reversal end the old age. SDPCCSimulatedPositionMetadata candidate; ZeroMemory(candidate); candidate.initialized = true; candidate.time_domain = time_domain; candidate.snapshot_bar = bar_ordinal; candidate.snapshot_server = snapshot_server; candidate.snapshot_time = snapshot_time; candidate.bar_server_open = server_open; candidate.bar_server_close = server_close; candidate.bar_open_time = open_time; candidate.bar_close_time = close_time; candidate.signed_lots = current_state[2]; long age_seconds = 0; if(current_state[2] != 0) { const bool same_side = (previous_state[2] > 0 && current_state[2] > 0) || (previous_state[2] < 0 && current_state[2] < 0); if(same_side) { if(!previous_position.initialized || previous_position.first_open_bar < 0 || previous_position.first_open_bar > previous_position.snapshot_bar || previous_position.first_open_server <= 0 || previous_position.first_open_time <= 0 || previous_position.first_open_time > previous_position.snapshot_time) { reason = "account13.history.unsourced_position_age"; return(false); } candidate.first_open_bar = previous_position.first_open_bar; candidate.first_open_server = previous_position.first_open_server; candidate.first_open_time = previous_position.first_open_time; } else { candidate.first_open_bar = bar_ordinal; candidate.first_open_server = server_open; candidate.first_open_time = open_time; } if(candidate.first_open_time > snapshot_time) { reason = "account13.history.negative_position_age"; return(false); } age_seconds = snapshot_time - candidate.first_open_time; } //--- For zero anchor and no collateral, DPCCMark gives equity=balance+profit-cost. const double profit = (double)current_state[1] - (double)current_state[0] + (double)current_state[6]; if(current_state[2] == 0 && (current_state[6] != 0 || profit != 0)) { reason = "account13.history.flat_snapshot_money"; return(false); } const double buy_lots = MathMax((double)current_state[2], 0.0); const double sell_lots = MathMax(-(double)current_state[2], 0.0); const double buy_profit = (current_state[2] > 0 ? profit : 0.0); const double sell_profit = (current_state[2] < 0 ? profit : 0.0); const double discount = (double)age_seconds / 36000.0 * MathAbs(profit); if(!DPCCBuildAccount13Features(current_state[0], previous_state[0], current_state[1], previous_state[1], buy_lots, sell_lots, buy_profit, sell_profit, discount, snapshot_server, features, reason)) return(false); position = candidate; return(true); } //+------------------------------------------------------------------+ //| Function DPCCMaterializeRolloutRewards | //+------------------------------------------------------------------+ bool DPCCMaterializeRolloutRewards(CNeuronScenarioRolloutOCL &rollout, const SDPCCConfig &config, const uint slot_index, const uint scenario_index, const float tolerance, float &rewards[], string &reason) { ArrayResize(rewards, 0); reason = ""; if(rollout.LastStatus() != DPCC_OK) { reason = "ROLLOUT_LAST_STATUS_NOT_OK"; return(false); } const uint slot_count = config.shape.M; const uint scenario_count = config.shape.K; const uint horizon = config.shape.H; const float gamma = config.gamma; if(slot_count == 0 || scenario_count == 0 || horizon == 0 || slot_index >= slot_count || scenario_index >= scenario_count || !MathIsValidNumber(gamma) || gamma <= 0 || gamma > 1 || !MathIsValidNumber(tolerance) || tolerance < 0 || slot_count > 2147483647) { reason = "INVALID_CONFIG_OR_PAIR"; return(false); } const ulong max_buffer = (ulong)2147483647; if((ulong)slot_count > max_buffer / (ulong)scenario_count) { reason = "NATIVE_BUFFER_SHAPE_OVERFLOW"; return(false); } const ulong pair_count = (ulong)slot_count * (ulong)scenario_count; if(pair_count > max_buffer / 8 || pair_count > max_buffer / 4 / (ulong)horizon || pair_count > max_buffer / 24 / ((ulong)horizon + 1)) { reason = "NATIVE_BUFFER_SHAPE_OVERFLOW"; return(false); } CLayerDescription *description = rollout.GetLayerInfo(); if(!description || ArraySize(description.windows) != 9 || ArraySize(description.units) != 5 || description.type != defNeuronScenarioRolloutOCL || description.windows[1] != scenario_count || description.windows[3] != horizon || description.windows[5] != slot_count || rollout.ContractSignature() != DPCCFixedSignature(config, defNeuronScenarioRolloutOCL)) { if(description) delete description; reason = "CONFIG_LAYER_CONTRACT_MISMATCH"; return(false); } delete description; CBufferFloat *native_returns = rollout.getOutput(); CBufferFloat *native_states = rollout.GetStates(); CBufferFloat *native_validity = rollout.GetValidity(); CBufferFloat *native_violations = rollout.GetViolations(); CBufferFloat *native_diagnostics = rollout.GetDiagnostics(); const ulong pair = (ulong)slot_index * scenario_count + scenario_index; if(!native_returns || !native_states || !native_validity || !native_violations || !native_diagnostics || native_returns.Total() != (int)pair_count || native_states.Total() != (int)(pair_count * ((ulong)horizon + 1) * 24) || native_validity.Total() != (int)slot_count || native_violations.Total() != (int)(pair_count * 8) || native_diagnostics.Total() != (int)(pair_count * horizon * 4)) { reason = "NATIVE_BUFFER_SHAPE_MISMATCH"; return(false); } if(!native_returns.BufferRead() || !native_states.BufferRead() || !native_validity.BufferRead() || !native_violations.BufferRead() || !native_diagnostics.BufferRead()) { reason = "NATIVE_BUFFER_READ_FAILED"; return(false); } if(native_validity[(int)slot_index] != 1.0f) { reason = "ROLLOUT_SLOT_INVALID"; return(false); } const ulong state_base = pair * ((ulong)horizon + 1) * 24; const ulong violation_base = pair * 8; const ulong diagnostic_base = pair * horizon * 4; for(uint i = 0; i < 8; i++) if(!MathIsValidNumber(native_violations[(int)(violation_base + i)])) { reason = "NONFINITE_NATIVE_VIOLATION"; return(false); } if(native_violations[(int)(violation_base)] != -1.0f || native_violations[(int)(violation_base + 2)] != 0.0f || native_violations[(int)(violation_base + 3)] != 0.0f || native_violations[(int)(violation_base + 4)] != 0.0f || native_violations[(int)(violation_base + 5)] != 0.0f) { reason = "ROLLOUT_VIOLATION_NUMERIC_OR_TERMINAL"; return(false); } float staged[]; if(ArrayResize(staged, (int)horizon) != (int)horizon) { reason = "REWARD_ALLOCATION_FAILED"; return(false); } double discounted_sum = 0.0; double discount = 1.0; for(uint h = 0; h < horizon; h++) { const ulong before_offset = state_base + (ulong)h * 24; const ulong after_offset = before_offset + 24; for(uint field = 0; field < 24; field++) if(!MathIsValidNumber(native_states[(int)(before_offset + field)]) || !MathIsValidNumber(native_states[(int)(after_offset + field)])) { reason = "NONFINITE_NATIVE_STATE"; return(false); } if(native_states[(int)(after_offset + 19)] != 0.0f) { reason = "TERMINAL_STEP_UNSUPPORTED"; return(false); } const ulong diagnostic_offset = diagnostic_base + (ulong)h * 4; for(uint field = 0; field < 4; field++) if(!MathIsValidNumber(native_diagnostics[(int)(diagnostic_offset + field)])) { reason = "NONFINITE_BRANCH_DIAGNOSTIC"; return(false); } const float chosen_branch = native_diagnostics[(int)diagnostic_offset]; if(chosen_branch != 0.0f && chosen_branch != 1.0f) { reason = "UNACCEPTED_OR_INVALID_NATIVE_STEP"; return(false); } const float reward = native_states[(int)(after_offset + 1)] - native_states[(int)(before_offset + 1)]; if(!MathIsValidNumber(reward)) { reason = "NONFINITE_EQUITY_DELTA"; return(false); } staged[h] = reward; discounted_sum += discount * (double)reward; discount *= (double)gamma; } const float native_return= native_returns[(int)pair]; if(!MathIsValidNumber(native_return) || !MathIsValidNumber(discounted_sum) || MathAbs(discounted_sum - (double)native_return) > (double)tolerance) { reason = "DISCOUNTED_REWARD_RETURN_MISMATCH"; return(false); } if(ArrayResize(rewards, (int)horizon) != (int)horizon) { ArrayResize(rewards, 0); reason = "REWARD_OUTPUT_ALLOCATION_FAILED"; return(false); } if(ArrayCopy(rewards, staged, 0, 0, (int)horizon) != (int)horizon) { ArrayResize(rewards, 0); reason = "REWARD_COPY_FAILED"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCRunHistoricalRollout evaluates an explicit physical H6 plan | //+------------------------------------------------------------------+ bool DPCCRunHistoricalRollout(CNet &net, const SDPCCShape &source_shape, const float &context_without_noise[], const float &target_native_ohlc[], const long decision_server_time, const long decision_ordinal, const long &target_open_server[], const long &target_close_server[], const long &target_ordinals[], const float &physical_plan[], const SDPCCConfig &config, const float tolerance, float &native_return, float &rewards[], string &reason) { native_return= 0.0f; ArrayResize(rewards, 0); reason = ""; //--- Validate the caller-owned plan and derive the one-path context. if(source_shape.H == 0 || source_shape.H > 2147483647 / 6 || ArraySize(physical_plan) != (int)(source_shape.H * 6)) { reason = "historical_rollout.plan_shape"; return(false); } for(int i = 0; i < ArraySize(physical_plan); i++) if(!MathIsValidNumber((double)physical_plan[i])) { reason = "historical_rollout.plan_numeric"; return(false); } SDPCCShape rollout_shape; float rollout_values[]; if(!DPCCBuildHistoricalRolloutContext(source_shape, context_without_noise, target_native_ohlc, decision_server_time, decision_ordinal, target_open_server, target_close_server, target_ordinals, rollout_shape, rollout_values, reason)) return(false); SDPCCLayout rollout_layout; if(!DPCCBuildRolloutLayout(rollout_shape, rollout_layout) || ArraySize(rollout_values) != (int)rollout_layout.total) { ArrayResize(rollout_values, 0); reason = "historical_rollout.context_shape"; return(false); } //--- Bind exact inputs to this CNet's OpenCL context and execute its rollout. SDPCCConfig native_config = config; native_config.shape = rollout_shape; CBufferFloat plan_buffer, context_buffer; if(!plan_buffer.BufferInit((uint)ArraySize(physical_plan), 0.0f) || !context_buffer.BufferInit((uint)ArraySize(rollout_values), 0.0f)) { reason = "historical_rollout.input_allocation"; return(false); } for(int i = 0; i < ArraySize(physical_plan); i++) if(!plan_buffer.Update(i, physical_plan[i])) { reason = "historical_rollout.plan_copy"; return(false); } for(int i = 0; i < ArraySize(rollout_values); i++) if(!context_buffer.Update(i, rollout_values[i])) { reason = "historical_rollout.context_copy"; return(false); } if(!plan_buffer.BufferCreate(net.GetOpenCL()) || !context_buffer.BufferCreate(net.GetOpenCL())) { reason = "historical_rollout.device_bind"; return(false); } if(!net.feedForward(GetPointer(plan_buffer), 1, false, GetPointer(context_buffer))) { reason = "historical_rollout.forward_failed"; return(false); } CNeuronScenarioRolloutOCL *rollout = net.Layer(1); if(!rollout || rollout.Type() != defNeuronScenarioRolloutOCL || rollout.LastStatus() != DPCC_OK) { reason = "historical_rollout.native_status"; return(false); } //--- Publish only a reconciled native return and its equity-delta rewards. float staged_rewards[]; if(!DPCCMaterializeRolloutRewards(rollout, native_config, 0, 0, tolerance, staged_rewards, reason)) return(false); CBufferFloat *returns = rollout.getOutput(); if(!returns || !returns.BufferRead() || returns.Total() != 1 || !MathIsValidNumber((double)returns[0])) { ArrayResize(staged_rewards, 0); reason = "historical_rollout.return_read"; return(false); } native_return= returns[0]; if(ArrayResize(rewards, ArraySize(staged_rewards)) != ArraySize(staged_rewards) || ArrayCopy(rewards, staged_rewards, 0, 0, ArraySize(staged_rewards)) != ArraySize(staged_rewards)) { native_return= 0.0f; ArrayResize(rewards, 0); reason = "historical_rollout.reward_publish"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Direct sigmoid lots preserve each Action6 before execution | //+------------------------------------------------------------------+ const string DPCC_Z_ACTION_CONTRACT = "SIGMOID_DIRECT_LOTS_BUY_MINUS_SELL_V1"; const string DPCC_Z_GRADIENT_MODE = "RAW_SIGMOID_FIRST_ACTION6_V1"; bool DPCCBindPolicyActionProfile(const string gradient_mode, string &wire, string &reason) { reason = ""; if(gradient_mode != DPCC_Z_GRADIENT_MODE && gradient_mode != "FROZEN_NO_GRADIENT") {reason = "BLOCKED.policy.action_profile_contract"; return(false);} const string fields[] = {DPCC_Z_ACTION_CONTRACT, "BROKER_VOLUME_GRID_AND_ORDER_CHECK_V1", gradient_mode}; for(int i = 0; i < ArraySize(fields); i++) DPCCOfflineField(wire, fields[i]); return(true); } //+------------------------------------------------------------------+ //| Hash shared model settings while excluding the attached symbol | //+------------------------------------------------------------------+ bool DPCCBuildChartNeutralPolicyProfileSHA(const string chart_symbol, const string schema, const string gradient_mode, const string &fields[], string &sha, string &reason) { sha = ""; reason = ""; if(StringLen(chart_symbol) == 0 || StringLen(schema) == 0 || StringFind(schema, "\n") >= 0) {reason = "BLOCKED.policy.profile_identity_input"; return(false);} string wire = schema + "\n"; for(int i = 0; i < ArraySize(fields); i++) DPCCOfflineField(wire, fields[i]); if(!DPCCBindPolicyActionProfile(gradient_mode, wire, reason) || !DPCCCheckpointTextSHA(wire, sha, reason)) { sha = ""; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCPreparePolicyActions | //+------------------------------------------------------------------+ bool DPCCPreparePolicyActions(const float &actions[], float &prepared[], string &reason) { ArrayResize(prepared, 0); reason = ""; const int count = ArraySize(actions); if(count < 6 || count % 6 != 0) {reason = "policy.Action6_shape"; return(false);} for(int i = 0; i < count; i++) if(!MathIsValidNumber(actions[i]) || actions[i] < 0 || actions[i] > 1) {reason = "policy.sigmoid_action_range"; return(false);} if(ArrayCopy(prepared, actions) != count) {ArrayResize(prepared, 0); reason = "policy.action_capacity"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCPrepareHistoricalPolicyActions | //+------------------------------------------------------------------+ bool DPCCPrepareHistoricalPolicyActions(const SDPCCShape &shape, const float &context[], const float &actions[], float &prepared[], string &reason) { ArrayResize(prepared, 0); SDPCCLayout layout, unused; uint tokens = 0; if(!DPCCCompositeLayouts(shape, layout, unused, tokens) || ArraySize(context) != (int)layout.offsets[10] || layout.lengths[6] != 24) {reason = "policy.historical_context_shape"; return(false);} //--- Reject old capped profiles before scoring or committing a direct-lots plan. const int limits = (int)layout.offsets[6]; if(context[limits] != 0 || context[limits + 20] != 8 || context[limits + 3] != 50) {reason = "BLOCKED.policy.direct_lots_financial_profile_required"; return(false);} return(DPCCPreparePolicyActions(actions, prepared, reason)); } //+------------------------------------------------------------------+ //| DPCCGenerateOracleCandidatePlans adds one clipped neighbor per | //| coordinate, alternating preferred direction with fallback | //+------------------------------------------------------------------+ bool DPCCGenerateOracleCandidatePlans(const float &seed_plan[], const uint horizon, const float &coordinate_deltas[], float &candidate_plans[], int &candidate_count, string &reason) { ArrayResize(candidate_plans, 0); candidate_count = 0; reason = ""; //--- Validate seed geometry and bounded candidate-count arithmetic. if(horizon == 0 || horizon > (2147483647 - 2) / 6 || ArraySize(coordinate_deltas) != 6 || horizon > 2147483647 / 6 || ArraySize(seed_plan) != (int)(horizon * 6)) { reason = "oracle_candidates.input_shape"; return(false); } for(int i = 0; i < ArraySize(seed_plan); i++) if(!MathIsValidNumber((double)seed_plan[i]) || seed_plan[i] < 0.0f || seed_plan[i] > 1.0f) { reason = "oracle_candidates.seed_action"; return(false); } for(int a = 0; a < ArraySize(coordinate_deltas); a++) if(!MathIsValidNumber((double)coordinate_deltas[a]) || coordinate_deltas[a] < 0.0f) { reason = "oracle_candidates.coordinate_delta"; return(false); } const int plan_size = (int)(horizon * 6); int neighbor_count = 0; for(uint h = 0; h < horizon; h++) for(int a = 0; a < 6; a++) { const int coordinate = (int)(h * 6) + (int)a; const float seed = seed_plan[coordinate]; const float plus = (float)MathMin(1.0, (double)seed + (double)coordinate_deltas[a]); const float minus = (float)MathMax(0.0, (double)seed - (double)coordinate_deltas[a]); const bool prefer_plus = ((h + (uint)a) % 2) == 0; const float preferred = (prefer_plus ? plus : minus); const float fallback = (prefer_plus ? minus : plus); if(preferred != seed || fallback != seed) neighbor_count++; } int actual_count = neighbor_count + 1; const int maximum_count = 2 + plan_size; if(actual_count > maximum_count || actual_count > 2147483647 / plan_size) { reason = "oracle_candidates.capacity"; return(false); } int total_elements = actual_count * plan_size; float staged[]; if(ArrayResize(staged, total_elements) != total_elements) { reason = "oracle_candidates.allocation"; return(false); } if(ArrayCopy(staged, seed_plan, 0, 0, plan_size) != plan_size) { reason = "oracle_candidates.seed_copy"; return(false); } //--- Append seed neighbors in horizon, coordinate order. Alternate the //--- preferred direction; use the opposite only when clipping removes it. int output_index = 1; for(uint h = 0; h < horizon; h++) for(int a = 0; a < 6; a++) { const int coordinate = (int)(h * 6) + (int)a; const float seed = seed_plan[coordinate]; const float plus = (float)MathMin(1.0, (double)seed + (double)coordinate_deltas[a]); const float minus = (float)MathMax(0.0, (double)seed - (double)coordinate_deltas[a]); const bool prefer_plus = ((h + (uint)a) % 2) == 0; const float preferred = (prefer_plus ? plus : minus); const float fallback = (prefer_plus ? minus : plus); const float neighbor = (preferred != seed ? preferred : fallback); if(neighbor == seed) continue; const int destination = output_index * plan_size; if(ArrayCopy(staged, seed_plan, destination, 0, plan_size) != plan_size) { reason = "oracle_candidates.plan_copy"; return(false); } staged[destination + coordinate] = neighbor; output_index++; } //--- Keep one deterministic flat plan available as the no-position fallback. bool flat_present = false; for(int candidate = 0; candidate < output_index && !flat_present; candidate++) { flat_present = true; const int offset = candidate * plan_size; for(int i = 0; i < plan_size; i++) if(staged[offset + i] != 0.0f) { flat_present = false; break; } } if(!flat_present) { actual_count++; if(actual_count > maximum_count || actual_count > 2147483647 / plan_size) { reason = "oracle_candidates.capacity"; return(false); } total_elements = actual_count * plan_size; if(ArrayResize(staged, total_elements) != total_elements) { reason = "oracle_candidates.flat_allocation"; return(false); } const int offset = (actual_count - 1) * plan_size; for(int i = 0; i < plan_size; i++) staged[offset + i] = 0.0f; output_index++; } if(output_index != actual_count || ArrayResize(candidate_plans, total_elements) != total_elements || ArrayCopy(candidate_plans, staged, 0, 0, total_elements) != total_elements) { ArrayResize(candidate_plans, 0); reason = "oracle_candidates.publish"; return(false); } candidate_count = actual_count; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Check the raw flat seed against native first-bar initialization | //+------------------------------------------------------------------+ bool DPCCValidateHistoricalOracleBootstrap(const SDPCCShape &shape, const float &context[], const float &state[], const uint currency_digits, string &reason) { reason = ""; SDPCCLayout layout; if(shape.K != 1 || shape.M != 1 || !DPCCBuildRolloutLayout(shape, layout) || ArraySize(context) != (int)layout.total || ArraySize(state) != DPCC_CONTINUATION_FLOATS || currency_digits > 8) { reason = "historical_oracle.bootstrap_shape"; return(false); } const int account = (int)layout.offsets[1]; const int instrument = (int)layout.offsets[2]; const float balance = context[account]; const float bid = context[account + 15]; const float ask = context[account + 16]; const float calculation = context[instrument + 11]; if(!MathIsValidNumber((double)balance) || balance <= 0.0f || !MathIsValidNumber((double)bid) || !MathIsValidNumber((double)ask) || bid <= 0.0f || ask < bid || !DPCCSupportedHistoricalCalculationModeValue(calculation)) { reason = "historical_oracle.bootstrap_flat_profile"; return(false); } //--- Every Account24 field must describe the fresh flat seed and match its context. for(int field = 0; field < 24; field++) { float expected = 0.0f; if(field == 0 || field == 1 || field == 7 || field == 8 || field == 9 || field == 12) expected = balance; else if(field == 15) expected = bid; else if(field == 16) expected = ask; else if(field == 20) expected = 1.0f; if(!MathIsValidNumber((double)state[field]) || state[field] != expected || context[account + field] != state[field]) { reason = "historical_oracle.bootstrap_account." + (string)field; return(false); } } //--- The seed carries the native initialization values before any open/calendar event. const float transient[] = {0.0f, 0.0f, bid, context[instrument + 23], 0.0f, (float)currency_digits, -1.0f, 0.0f, 0.0f }; for(int field = 24; field < DPCC_CONTINUATION_FLOATS; field++) if(!MathIsValidNumber((double)state[field]) || !MathIsValidNumber((double)transient[field - 24]) || state[field] != transient[field - 24]) { reason = "historical_oracle.bootstrap_transient." + (string)field; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCSelectHistoricalOracleCandidateFromContinuation scores plans | //+------------------------------------------------------------------+ bool DPCCSelectHistoricalOracleCandidateFromContinuation( CNet &net, const SDPCCShape &source_shape, const float &context_without_noise[], const float &target_native_ohlc[], const long decision_server_time, const long decision_ordinal, const long &target_open_server[], const long &target_close_server[], const long &target_ordinals[], const float &candidate_plans[], const int candidate_count, const SDPCCConfig &config, const float tolerance, CBufferFloat *continuation_in, const uint episode_start_step, const uint episode_horizon, float &selected_plan[], double &selected_score, int &selected_source, uchar &valid_horizons[], string &reason, const bool operation_only = false) { ArrayResize(selected_plan, 0); ArrayResize(valid_horizons, 0); selected_score = 0.0; selected_source = -1; reason = ""; //--- Validate the bounded flattened input shape before indexing candidates. const bool has_continuation = (continuation_in != NULL); const ulong absolute_end = (ulong)episode_start_step + (ulong)source_shape.H; if(candidate_count <= 0 || (operation_only && candidate_count != 1) || source_shape.H == 0 || source_shape.H > (2147483647 - 1) / 12 || config.shape.H != source_shape.H || !MathIsValidNumber(config.gamma) || config.gamma <= 0.0f || config.gamma > 1.0f || !MathIsValidNumber((double)tolerance) || tolerance < 0.0f || episode_horizon == 0 || episode_horizon > 16777216 || absolute_end > (ulong)episode_horizon || (!has_continuation && episode_start_step != 0) || (has_continuation && episode_start_step == 0 && operation_only)) { reason = "historical_oracle.input_shape_or_profile"; return(false); } COpenCLMy *native_context = net.GetOpenCL(); if(CheckPointer(native_context) == POINTER_INVALID) { reason = "historical_oracle.native_context_unavailable"; return(false); } if(has_continuation) { if(CheckPointer(continuation_in) == POINTER_INVALID || continuation_in.Total() != DPCC_CONTINUATION_FLOATS || continuation_in.GetIndex() < 0 || continuation_in.GetOpenCL() != native_context || !continuation_in.BufferRead()) { reason = "historical_oracle.continuation_shape_or_context"; return(false); } for(int field = 0; field < DPCC_CONTINUATION_FLOATS; field++) if(!MathIsValidNumber((double)continuation_in[field])) { reason = "historical_oracle.continuation_nonfinite"; return(false); } } const int plan_size = (int)(source_shape.H * 6); const int max_candidates = 1 + 2 * plan_size; if(candidate_count > max_candidates || candidate_count > 2147483647 / plan_size || ArraySize(candidate_plans) != candidate_count * plan_size) { reason = "historical_oracle.candidate_capacity"; return(false); } float prepared_candidates[]; if(!DPCCPrepareHistoricalPolicyActions(source_shape, context_without_noise, candidate_plans, prepared_candidates, reason)) return(false); float best_plan[]; uchar best_mask[]; if(ArrayResize(best_plan, plan_size) != plan_size || ArrayResize(best_mask, (int)source_shape.H) != (int)source_shape.H) { reason = "historical_oracle.output_allocation"; return(false); } double best_score = 0.0; float candidate_plan[]; //--- Score carried full plans in caller order using the same native ledger. if(has_continuation && !operation_only) { if(DPCCTrainingFloorReached((double)continuation_in[DPCC_CONTINUATION_ACCOUNT_OFFSET], (double)continuation_in[DPCC_CONTINUATION_ACCOUNT_OFFSET + 1])) { reason = "historical_oracle.initial_account_below_floor"; return(false); } SDPCCShape rollout_shape; float rollout_values[]; if(!DPCCBuildHistoricalRolloutContext(source_shape, context_without_noise, target_native_ohlc, decision_server_time, decision_ordinal, target_open_server, target_close_server, target_ordinals, rollout_shape, rollout_values, reason)) { reason = "historical_oracle.context." + reason; return(false); } if(rollout_shape.H != source_shape.H || rollout_shape.K != 1 || rollout_shape.M != 1) { reason = "historical_oracle.context_shape"; return(false); } if(episode_start_step == 0) { float raw_seed[]; if(ArrayResize(raw_seed, DPCC_CONTINUATION_FLOATS) != DPCC_CONTINUATION_FLOATS) { reason = "historical_oracle.bootstrap_capacity"; return(false); } for(int field = 0; field < DPCC_CONTINUATION_FLOATS; field++) raw_seed[field] = continuation_in[field]; if(!DPCCValidateHistoricalOracleBootstrap(rollout_shape, rollout_values, raw_seed, config.account_currency_digits, reason)) return(false); } CNeuronScenarioRolloutOCL *rollout = net.Layer(1); if(!rollout || rollout.Type() != defNeuronScenarioRolloutOCL || rollout.LastStatus() != DPCC_OK) { reason = "historical_oracle.native_rollout_unavailable"; return(false); } CBufferFloat full_context; if(!full_context.BufferInit((uint)ArraySize(rollout_values), 0.0f)) { reason = "historical_oracle.batch_context_allocation"; return(false); } for(int i = 0; i < ArraySize(rollout_values); i++) if(!full_context.Update(i, rollout_values[i])) { reason = "historical_oracle.batch_context_copy"; return(false); } if(!full_context.BufferCreate(native_context)) { reason = "historical_oracle.batch_context_bind"; return(false); } //--- Score the complete Oracle candidate set in one native batch. const int batch_capacity = candidate_count; for(int first = 0; first < candidate_count; first += batch_capacity) { const int slots = MathMin(batch_capacity, candidate_count - first); uchar candidate_infeasible[]; if(ArrayResize(candidate_infeasible, slots) != slots) { reason = "historical_oracle.batch_rejection_allocation"; return(false); } ArrayInitialize(candidate_infeasible, (uchar)0); const ulong state_count = (ulong)slots * ((ulong)source_shape.H + 1) * 24; const ulong branch_count = (ulong)slots * source_shape.H * 4; if(state_count > (ulong)INT_MAX || branch_count > (ulong)INT_MAX) { reason = "historical_oracle.batch_capacity"; return(false); } CBufferFloat plans, active, carried, returns, states, violations, branches, rewards, executed; CBufferFloat *buffers[] = {GetPointer(plans), GetPointer(active), GetPointer(carried), GetPointer(returns), GetPointer(states), GetPointer(violations), GetPointer(branches), GetPointer(rewards), GetPointer(executed) }; const uint counts[] = {(uint)(slots * plan_size), (uint)slots, (uint)(slots * DPCC_CONTINUATION_FLOATS), (uint)slots, (uint)state_count, (uint)(slots * 8), (uint)branch_count, (uint)((ulong)slots * source_shape.H), (uint)((ulong)slots * source_shape.H) }; for(int buffer = 0; buffer < ArraySize(buffers); buffer++) if(!buffers[buffer].BufferInit(counts[buffer], 0.0f)) { reason = "historical_oracle.batch_buffer_allocation." + (string)buffer; return(false); } for(int slot = 0; slot < slots; slot++) { if(!active.Update(slot, 1.0f)) { reason = "historical_oracle.batch_active_copy"; return(false); } for(int field = 0; field < DPCC_CONTINUATION_FLOATS; field++) if(!carried.Update(slot * DPCC_CONTINUATION_FLOATS + field, continuation_in[field])) { reason = "historical_oracle.batch_continuation_copy"; return(false); } for(int i = 0; i < plan_size; i++) { const float action = prepared_candidates[(first + slot) * plan_size + i]; if(!MathIsValidNumber((double)action) || action < 0.0f || action > 1.0f || !plans.Update(slot * plan_size + i, action)) { reason = "historical_oracle.batch_action_copy_or_range"; return(false); } } } for(int buffer = 0; buffer < ArraySize(buffers); buffer++) if(!buffers[buffer].BufferCreate(native_context)) { reason = "historical_oracle.batch_buffer_bind." + (string)buffer; return(false); } bool batch_success = rollout.EvaluateContinuedPlanBatch(GetPointer(plans), 0, (uint)slots, GetPointer(active), GetPointer(full_context), GetPointer(carried), episode_start_step, episode_horizon, 50.0f, GetPointer(returns), GetPointer(states), GetPointer( violations), GetPointer(branches), GetPointer(rewards), GetPointer(executed)); int native_status = (int)rollout.LastStatus(); if(!batch_success && native_status == (int)DPCC_INVALID_INPUT) { //--- Recover only candidates whose native risk provenance is complete. bool readback_ok = true; for(int buffer = 3; buffer < ArraySize(buffers); buffer++) { if(!buffers[buffer].BufferRead()) { readback_ok = false; break; } for(int i = 0; i < buffers[buffer].Total(); i++) if(!MathIsValidNumber((double)buffers[buffer].At(i))) { readback_ok = false; break; } if(!readback_ok) break; } bool provenance_ok = readback_ok; bool any_infeasible = false; const int horizon = (int)source_shape.H; for(int slot = 0; slot < slots && provenance_ok; slot++) { const int offset = slot * 8; const float status = violations[offset + 5]; if(status == 0.0f) continue; bool valid_risk = (status == (float)DPCC_INVALID_INPUT); for(int field = 0; field < 8; field++) if(!MathIsValidNumber((double)violations[offset + field])) valid_risk = false; const float first_h = violations[offset]; const float constraint = violations[offset + 1]; const float phase = violations[offset + 6]; const float source_action = violations[offset + 7]; valid_risk = valid_risk && first_h >= 0.0f && first_h < (float)horizon && MathFloor(first_h) == first_h && constraint >= 0.0f && constraint <= 15.0f && MathFloor(constraint) == constraint && violations[offset + 2] >= 0.0f && violations[offset + 3] >= 0.0f && phase >= 0.0f && phase <= 3.0f && MathFloor(phase) == phase && source_action >= -1.0f && source_action < (float)horizon && MathFloor(source_action) == source_action; if(!valid_risk) { provenance_ok = false; break; } candidate_infeasible[slot] = 1; any_infeasible = true; } if(provenance_ok && any_infeasible) { for(int slot = 0; slot < slots; slot++) if(candidate_infeasible[slot] != 0 && !active.Update(slot, 0.0f)) { reason = "historical_oracle.batch_deactivate"; return(false); } if(!active.BufferWrite()) { reason = "historical_oracle.batch_deactivate_write"; return(false); } batch_success = rollout.EvaluateContinuedPlanBatch(GetPointer(plans), 0, (uint)slots, GetPointer(active), GetPointer(full_context), GetPointer(carried), episode_start_step, episode_horizon, 50.0f, GetPointer(returns), GetPointer(states), GetPointer(violations), GetPointer(branches), GetPointer(rewards), GetPointer(executed)); native_status = (int)rollout.LastStatus(); } } if(!batch_success || native_status != (int)DPCC_OK) { //--- Read failed outputs without losing the original native verdict. bool output_read[6]; for(int buffer = 3; buffer < ArraySize(buffers); buffer++) output_read[buffer - 3] = buffers[buffer].BufferRead(); reason = StringFormat("historical_oracle.native_batch_failed:success=%d;native_status=%d;", (int)batch_success, native_status); reason += StringFormat("first=%d;slots=%d;start=%u;horizon=%u", first, slots, episode_start_step, episode_horizon); PrintFormat("DPCC_ORACLE_BATCH_FAIL %s output_read=%d,%d,%d,%d,%d,%d", reason, (int)output_read[0], (int)output_read[1], (int)output_read[2], (int)output_read[3], (int)output_read[4], (int)output_read[5]); const bool slot_status_available = output_read[2] && (native_status == (int)DPCC_INVALID_INPUT || native_status == ( int)DPCC_NUMERIC_ERROR); const int horizon = (int)source_shape.H; for(int slot = 0; slot < slots; slot++) { if(slot_status_available && violations[slot * 8 + 5] == 0.0f) continue; string fields = ""; if(output_read[2]) for(int field = 0; field < 8; field++) fields += StringFormat(" v%d=%.9g", field, (double)violations[slot * 8 + field]); int problem_h = -1; for(int h = 0; h < horizon; h++) { const int branch = (slot * horizon + h) * 4; const int mask = slot * horizon + h; if(output_read[3] && output_read[5] && branches[branch] >= 0.0f && executed[mask] == 0.0f) {problem_h = h; break;} } if(problem_h < 0 && output_read[2] && violations[slot * 8] >= 0.0f && violations[slot * 8] < (float)horizon && MathFloor(violations[slot * 8]) == violations[slot * 8]) problem_h = (int)violations[slot * 8]; PrintFormat("DPCC_ORACLE_BATCH_SLOT candidate=%d problem_h=%d status_available=%d%s", first + slot, problem_h, (int)slot_status_available, fields); for(int h = 0; h < horizon; h++) { const int branch = (slot * horizon + h) * 4; const string mask = (output_read[5] ? StringFormat("%.9g", (double)executed[slot * horizon + h]) : "UNREAD"); string values = "UNREAD"; if(output_read[3]) values = StringFormat("%.9g,%.9g,%.9g,%.9g", (double)branches[branch], (double)branches[branch + 1], (double)branches[branch + 2], ( double)branches[branch + 3]); PrintFormat("DPCC_ORACLE_BATCH_STEP candidate=%d h=%d executed=%s branches=%s", first + slot, h, mask, values); } int action_h = problem_h; if(action_h < 0 && output_read[2] && violations[slot * 8 + 5] != 0.0f) action_h = 0; if(action_h >= 0) { fields = ""; for(int action = 0; action < 6; action++) fields += StringFormat(" a%d=%.9g", action, (double)prepared_candidates[(first + slot) * plan_size + action_h * 6 + action]); PrintFormat("DPCC_ORACLE_BATCH_ACTION candidate=%d h=%d%s", first + slot, action_h, fields); } if(output_read[1] && output_read[2] && violations[slot * 8 + 5] != 0.0f) for(int snapshot = 0; snapshot < 2; snapshot++) for(int start = 0; start < 24; start += 12) { fields = ""; const int offset = (slot * (horizon + 1) + snapshot) * 24; for(int field = start; field < start + 12; field++) fields += StringFormat(" %d=%.9g", field, (double)states[offset + field]); PrintFormat("DPCC_ORACLE_BATCH_ACCOUNT candidate=%d snapshot=%s%s", first + slot, (snapshot == 0 ? "INITIAL" : "AFTER_H0"), fields); } } return(false); } for(int buffer = 3; buffer < ArraySize(buffers); buffer++) { if(!buffers[buffer].BufferRead()) { reason = "historical_oracle.batch_readback." + (string)buffer; return(false); } for(int i = 0; i < buffers[buffer].Total(); i++) if(!MathIsValidNumber((double)buffers[buffer].At(i))) { reason = "historical_oracle.nonfinite_batch_output." + (string)buffer; return(false); } } //--- Ordinary risk diagnostics do not reject a numerically valid plan. for(int slot = 0; slot < slots; slot++) { if(candidate_infeasible[slot] != 0) continue; if(violations[slot * 8 + 5] != 0.0f) { reason = "historical_oracle.native_batch_status"; return(false); } const int state_base = (int)((ulong)slot * ((ulong)source_shape.H + 1) * 24); const int step_base = (int)((ulong)slot * source_shape.H); for(int field = 0; field < 24; field++) if(states[state_base + field] != continuation_in[field]) { reason = "historical_oracle.batch_initial_state_mismatch"; return(false); } uchar candidate_mask[]; if(ArrayResize(candidate_mask, (int)source_shape.H) != (int)source_shape.H) { reason = "historical_oracle.mask_allocation"; return(false); } ArrayInitialize(candidate_mask, (uchar)0); bool stopped = false; double candidate_score = 0.0; double discount = 1.0; for(uint h = 0; h < source_shape.H; h++) { const int step = step_base + (int)h; const int branch = step * 4; const int state = state_base + ((int)h + 1) * 24; if(executed[step] != (stopped ? 0.0f : 1.0f)) { reason = "historical_oracle.batch_execution_mask"; return(false); } if(stopped) { if(rewards[step] != 0.0f || branches[branch] != -1.0f) { reason = "historical_oracle.batch_stopped_tail"; return(false); } for(int field = 0; field < 24; field++) if(states[state + field] != states[state - 24 + field]) { reason = "historical_oracle.batch_tail_state_changed"; return(false); } continue; } if(branches[branch] != 0.0f && branches[branch] != 1.0f) { reason = "historical_oracle.native_branch_invalid"; return(false); } candidate_mask[(int)h] = 1; candidate_score += discount * (double)rewards[step]; if(!MathIsValidNumber(candidate_score)) { reason = "historical_oracle.nonfinite_discounted_score"; return(false); } stopped = DPCCTrainingFloorReached((double)states[state], (double)states[state + 1]); if(!stopped) discount *= ((double)config.gamma); } bool candidate_is_flat = true; for(int i = 0; i < plan_size; i++) if(prepared_candidates[(first + slot) * plan_size + i] != 0.0f) { candidate_is_flat = false; break; } bool best_is_flat = false; if(selected_source >= 0) { best_is_flat = true; for(int i = 0; i < plan_size; i++) if(best_plan[i] != 0.0f) { best_is_flat = false; break; } } //--- Prefer the no-position plan only when exact scores tie. if(selected_source < 0 || candidate_score > best_score || (candidate_score == best_score && candidate_is_flat && !best_is_flat)) { best_score = candidate_score; for(int i = 0; i < plan_size; i++) best_plan[i] = prepared_candidates[(first + slot) * plan_size + i]; ArrayCopy(best_mask, candidate_mask, 0, 0, (int)source_shape.H); selected_source = first + slot; } } } } else //--- Evaluate each complete candidate with the canonical native rollout. for(int candidate = 0; candidate < candidate_count; candidate++) { if(ArrayResize(candidate_plan, plan_size) != plan_size) { reason = "historical_oracle.plan_allocation"; return(false); } const int source_offset = candidate * plan_size; for(int i = 0; i < plan_size; i++) { const float action = prepared_candidates[source_offset + i]; if(!MathIsValidNumber((double)action) || action < 0.0f || action > 1.0f) { reason = "historical_oracle.action_range"; return(false); } candidate_plan[i] = action; } SDPCCShape rollout_shape; float rollout_values[]; if(!DPCCBuildHistoricalRolloutContext(source_shape, context_without_noise, target_native_ohlc, decision_server_time, decision_ordinal, target_open_server, target_close_server, target_ordinals, rollout_shape, rollout_values, reason)) { reason = "historical_oracle.context." + reason; return(false); } if(rollout_shape.H != source_shape.H || rollout_shape.K != 1 || rollout_shape.M != 1) { reason = "historical_oracle.context_shape"; return(false); } CNeuronScenarioRolloutOCL *rollout = net.Layer(1); if(!rollout || rollout.Type() != defNeuronScenarioRolloutOCL || rollout.LastStatus() != DPCC_OK) { reason = "historical_oracle.native_rollout_unavailable"; return(false); } CBufferFloat plan_buffer, state_first, state_second, reward_buffer, status_buffer, branch_buffer; if(!plan_buffer.BufferInit(plan_size, 0.0f) || !state_first.BufferInit(DPCC_CONTINUATION_FLOATS, 0.0f) || !state_second.BufferInit(DPCC_CONTINUATION_FLOATS, 0.0f) || !reward_buffer.BufferInit(1, 0.0f) || !status_buffer.BufferInit(1, 0.0f) || !branch_buffer.BufferInit(4, 0.0f)) { reason = "historical_oracle.native_buffer_allocation"; return(false); } for(int i = 0; i < plan_size; i++) if(!plan_buffer.Update(i, candidate_plan[i])) { reason = "historical_oracle.native_plan_copy"; return(false); } if(!plan_buffer.BufferCreate(net.GetOpenCL()) || !state_first.BufferCreate(net.GetOpenCL()) || !state_second.BufferCreate(net.GetOpenCL()) || !reward_buffer.BufferCreate(net.GetOpenCL()) || !status_buffer.BufferCreate(net.GetOpenCL()) || !branch_buffer.BufferCreate(net.GetOpenCL())) { reason = "historical_oracle.native_buffer_bind"; return(false); } uchar candidate_mask[]; if(ArrayResize(candidate_mask, (int)source_shape.H) != (int)source_shape.H) { reason = "historical_oracle.mask_allocation"; return(false); } ArrayInitialize(candidate_mask, (uchar)0); double candidate_score = 0.0; double discount = 1.0; for(uint h = 0; h < source_shape.H; h++) { SDPCCShape step_shape; float step_values[]; SDPCCLayout step_layout; if(!DPCCBuildHistoricalRolloutStepContext(rollout_shape, rollout_values, h, step_shape, step_values, reason) || step_shape.H != 1 || step_shape.K != 1 || step_shape.M != 1 || !DPCCBuildRolloutLayout(step_shape, step_layout) || ArraySize(step_values) != (int)step_layout.total) { reason = "historical_oracle.step_context." + reason; return(false); } CBufferFloat step_context; if(ArraySize(step_values) == 0 || !step_context.BufferInit((uint)ArraySize(step_values), 0.0f)) { reason = "historical_oracle.step_context_allocation"; return(false); } for(int i = 0; i < ArraySize(step_values); i++) if(!step_context.Update(i, step_values[i])) { reason = "historical_oracle.step_context_copy"; return(false); } if(!step_context.BufferCreate(net.GetOpenCL())) { reason = "historical_oracle.step_context_bind"; return(false); } CBufferFloat *input_state = (h == 0 ? continuation_in : (h % 2 == 1 ? GetPointer(state_first) : GetPointer(state_second))); CBufferFloat *output_state = (h % 2 == 0 ? GetPointer(state_first) : GetPointer(state_second)); const uint absolute_step = episode_start_step + h; const bool advanced = (operation_only && h > 0 ? rollout.EvaluateOneBarWithoutAction(GetPointer(step_context), absolute_step, episode_horizon, input_state, output_state, GetPointer(reward_buffer), GetPointer( status_buffer), GetPointer(branch_buffer)) : rollout.EvaluateOneBar(GetPointer(plan_buffer), h * 6, GetPointer(step_context), absolute_step, episode_horizon, (!has_continuation && h == 0), input_state, output_state, GetPointer(reward_buffer), GetPointer(status_buffer), GetPointer( branch_buffer))); if(!advanced) { reason = "historical_oracle.native_step_failed"; return(false); } if(!output_state.BufferRead() || !reward_buffer.BufferRead() || !status_buffer.BufferRead() || !branch_buffer.BufferRead() || status_buffer[0] != 0.0f) { reason = "historical_oracle.native_step_read_or_status"; return(false); } for(int field = 0; field < DPCC_CONTINUATION_FLOATS; field++) if(!MathIsValidNumber((double)output_state[field])) { reason = "historical_oracle.nonfinite_native_state"; return(false); } for(int field = 0; field < 4; field++) if(!MathIsValidNumber((double)branch_buffer[field])) { reason = "historical_oracle.nonfinite_native_diagnostic"; return(false); } if(branch_buffer[0] != 0.0f && branch_buffer[0] != 1.0f) { reason = "historical_oracle.native_branch_invalid"; return(false); } const double balance = (double)output_state[DPCC_CONTINUATION_ACCOUNT_OFFSET]; const double equity = (double)output_state[DPCC_CONTINUATION_ACCOUNT_OFFSET + 1]; const double reward = (double)reward_buffer[0]; if(!MathIsValidNumber(balance) || !MathIsValidNumber(equity) || !MathIsValidNumber(reward)) { reason = "historical_oracle.nonfinite_native_value"; return(false); } if(h == 0) { double initial_balance = 0.0; double initial_equity = 0.0; if(has_continuation) { initial_balance = (double)continuation_in[DPCC_CONTINUATION_ACCOUNT_OFFSET]; initial_equity = (double)continuation_in[DPCC_CONTINUATION_ACCOUNT_OFFSET + 1]; } else { const int account_offset = (int)step_layout.offsets[1]; initial_balance = (double)step_values[account_offset]; initial_equity = (double)step_values[account_offset + 1]; } if(!MathIsValidNumber(initial_balance) || !MathIsValidNumber(initial_equity) || DPCCTrainingFloorReached(initial_balance, initial_equity)) { reason = "historical_oracle.initial_account_below_floor"; return(false); } } candidate_mask[(int)h] = 1; candidate_score += discount * reward; if(!MathIsValidNumber(candidate_score)) { reason = "historical_oracle.nonfinite_discounted_score"; return(false); } if(DPCCTrainingFloorReached(balance, equity)) break; discount *= ((double)config.gamma); } //--- Strict comparison preserves the first caller-ordered exact tie. bool candidate_is_flat = true; for(int i = 0; i < plan_size; i++) if(candidate_plan[i] != 0.0f) { candidate_is_flat = false; break; } bool best_is_flat = false; if(selected_source >= 0) { best_is_flat = true; for(int i = 0; i < plan_size; i++) if(best_plan[i] != 0.0f) { best_is_flat = false; break; } } if(selected_source < 0 || candidate_score > best_score || (candidate_score == best_score && candidate_is_flat && !best_is_flat)) { best_score = candidate_score; ArrayCopy(best_plan, candidate_plan, 0, 0, plan_size); ArrayCopy(best_mask, candidate_mask, 0, 0, (int)source_shape.H); selected_source = candidate; } } //--- Publish the winner only after every supplied candidate passed validation. if(selected_source < 0 || ArrayResize(selected_plan, plan_size) != plan_size || ArrayResize(valid_horizons, (int)source_shape.H) != (int)source_shape.H) { ArrayResize(selected_plan, 0); ArrayResize(valid_horizons, 0); selected_score = 0.0; selected_source = -1; reason = "historical_oracle.output_publish"; return(false); } if(ArrayCopy(selected_plan, best_plan, 0, 0, plan_size) != plan_size || ArrayCopy(valid_horizons, best_mask, 0, 0, (int)source_shape.H) != (int)source_shape.H) { ArrayResize(selected_plan, 0); ArrayResize(valid_horizons, 0); selected_score = 0.0; selected_source = -1; reason = "historical_oracle.output_copy"; return(false); } selected_score = best_score; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Legacy selector retains initial-account behavior through wrapper | //+------------------------------------------------------------------+ bool DPCCSelectHistoricalOracleCandidate(CNet &net, const SDPCCShape &source_shape, const float &context_without_noise[], const float &target_native_ohlc[], const long decision_server_time, const long decision_ordinal, const long &target_open_server[], const long &target_close_server[], const long &target_ordinals[], const float &candidate_plans[], const int candidate_count, const SDPCCConfig &config, const float tolerance, float &selected_plan[], double &selected_score, int &selected_source, uchar &valid_horizons[], string &reason) { return(DPCCSelectHistoricalOracleCandidateFromContinuation( net, source_shape, context_without_noise, target_native_ohlc, decision_server_time, decision_ordinal, target_open_server, target_close_server, target_ordinals, candidate_plans, candidate_count, config, tolerance, NULL, 0, source_shape.H, selected_plan, selected_score, selected_source, valid_horizons, reason)); } //+------------------------------------------------------------------+ //| DPCCUpdateHistoricalOracleActor applies one factual-horizon | //| update | //+------------------------------------------------------------------+ bool DPCCUpdateHistoricalOracleActor( CNet &actor, CNet &rollout, const SDPCCShape &shape, CBufferFloat *scenarios, const float &account13[], const float &context_without_noise[], const float &target_native_ohlc[], const long decision_server_time, const long decision_ordinal, const long &target_open_server[], const long &target_close_server[], const long &target_ordinals[], const float &coordinate_deltas[], const SDPCCConfig &config, const float tolerance, CBufferFloat *continuation_in, const uint episode_start_step, const uint episode_horizon, float &pre_update_plan[], float &executed_action[], float &teacher_plan[], uchar &valid_horizons[], double &teacher_score, int &teacher_source, string &reason) { //--- Public outputs stay empty until the complete update is ready. ArrayResize(pre_update_plan, 0); ArrayResize(executed_action, 0); ArrayResize(teacher_plan, 0); ArrayResize(valid_horizons, 0); teacher_score = 0.0; teacher_source = -1; reason = ""; if(!DPCCValidatePolicyNet(actor, shape, false, reason)) return(false); COpenCLMy *context = actor.GetOpenCL(); if(!actor.WeightsUpdateEnabled() || GetPointer(actor) == GetPointer(rollout) || rollout.GetOpenCL() != context || CheckPointer(scenarios) == POINTER_INVALID || (scenarios.GetIndex() >= 0 && scenarios.GetOpenCL() != context)) { reason = "policy.oracle_update.context_or_update_state"; return(false); } for(int layer = 0; layer < 3; layer++) { CBufferFloat *output = actor.Layer(layer).getOutput(); CBufferFloat *gradient = actor.Layer(layer).getGradient(); if(CheckPointer(output) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || output.GetOpenCL() != context || gradient.GetOpenCL() != context || output.GetIndex() < 0 || gradient.GetIndex() < 0) { reason = StringFormat("policy.oracle_update.layer_buffers.%d", layer); return(false); } if(layer > 0) { CNeuronConvOCL *convolution = dynamic_cast(actor.Layer(layer)); CBufferFloat *weights = convolution.GetWeightsConv(); if(CheckPointer(weights) == POINTER_INVALID || weights.GetOpenCL() != context || weights.GetIndex() < 0) { reason = StringFormat("policy.oracle_update.weight_buffer.%d", layer); return(false); } } } //--- Preserve the emitted plan before any hindsight target or weight update. CBufferFloat actor_input; if(!DPCCPackPolicyInput(shape, scenarios, account13, NULL, GetPointer(actor_input), reason)) return(false); if(!actor.feedForward(GetPointer(actor_input), 1, false, (CBufferFloat*)NULL)) { reason = "policy.oracle_update.actor_forward"; return(false); } CBufferFloat *values = actor.Layer(2).getOutput(); float seed[], candidates[], selected[], step_mask[]; uchar selected_mask[]; const int plan_size = (int)(shape.H * shape.NActions); if(!values.BufferRead() || ArrayResize(seed, plan_size) != plan_size) { reason = "policy.oracle_update.seed_read_or_capacity"; return(false); } for(int i = 0; i < plan_size; i++) seed[i] = values.At(i); int candidate_count = 0, selected_source = -1; double selected_score = 0.0; if(!DPCCGenerateOracleCandidatePlans(seed, shape.H, coordinate_deltas, candidates, candidate_count, reason)) return(false); if(!DPCCSelectHistoricalOracleCandidateFromContinuation( rollout, shape, context_without_noise, target_native_ohlc, decision_server_time, decision_ordinal, target_open_server, target_close_server, target_ordinals, candidates, candidate_count, config, tolerance, continuation_in, episode_start_step, episode_horizon, selected, selected_score, selected_source, selected_mask, reason)) return(false); if(ArrayResize(step_mask, (int)shape.H) != (int)shape.H) { reason = "policy.oracle_update.mask_capacity"; return(false); } for(uint h = 0; h < shape.H; h++) step_mask[h] = (float)selected_mask[h]; if(!DPCCPrepareOracleActorGradient(actor, shape, selected, step_mask, config.gamma, reason)) return(false); float prepared_seed[]; if(!DPCCPrepareHistoricalPolicyActions(shape, context_without_noise, seed, prepared_seed, reason)) return(false); //--- Allocate publication before the fallible native update; no financial state is committed here. if(ArrayCopy(pre_update_plan, seed) != plan_size || ArrayResize(executed_action, 6) != 6 || ArrayCopy(teacher_plan, selected) != plan_size || ArrayCopy(valid_horizons, selected_mask) != (int)shape.H) { ArrayResize(pre_update_plan, 0); ArrayResize(executed_action, 0); ArrayResize(teacher_plan, 0); ArrayResize(valid_horizons, 0); reason = "policy.oracle_update.output_capacity"; return(false); } for(int i = 0; i < 6; i++) executed_action[i] = prepared_seed[i]; if(!actor.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, -1, true)) { ArrayResize(pre_update_plan, 0); ArrayResize(executed_action, 0); ArrayResize(teacher_plan, 0); ArrayResize(valid_horizons, 0); reason = "policy.oracle_update.actor_backward"; return(false); } teacher_score = selected_score; teacher_source = selected_source; reason = ""; return(true); } struct SDPCCExecutionProfileSnapshot { string raw_history_sha256, history_symbol, terminal_company, terminal_name, terminal_server; string account_currency; datetime snapshot_time_server; ENUM_TIMEFRAMES history_timeframe; uint history_rows; long terminal_build, account_currency_digits; long symbol_digits, spread_is_floating, trade_mode, trade_execution_mode, calculation_mode; long order_mode, filling_mode, expiration_mode, stops_level_points, freeze_level_points; long swap_mode, swap_rollover_3days, margin_hedged_use_leg; double point_size, trade_tick_size, trade_tick_value, trade_tick_value_profit, trade_tick_value_loss; double contract_size, volume_min, volume_max, volume_step, volume_limit; double swap_long, swap_short, margin_initial, margin_maintenance, margin_hedged; double bond_face_value, bond_accrued_interest, session_price_settlement; string currency_base, currency_profit, currency_margin, description, path; MqlCommission commission_rules[]; }; //--- Dated modeled inputs; native rollout alone applies accounting transitions. struct SDPCCHistoricalFinancialProfile { float instrument[48], limits[24]; double swap_cash_buy, swap_cash_sell, swap_factors[7]; long account_currency_digits; bool server_midnight; string time_domain, account_currency; string payload, sha256; string fx_profit_symbol, fx_margin_symbol; bool fx_profit_inverse, fx_margin_inverse, swap_profit_currency; double fx_profit_point, fx_margin_point; long fx_source_open[]; float fx_frozen[]; }; //+------------------------------------------------------------------+ //| DPCCExecutionAppend adds one length-delimited canonical field | //+------------------------------------------------------------------+ bool DPCCExecutionAppend(string &payload, const string key, const string value) { const string record = key + "|" + (string)StringLen(value) + "|" + value + "\n"; if(StringLen(payload) > 1048576 - StringLen(record)) return(false); payload += record; return(true); } //--- Bounded host-side execution intent/reconciliation contract. #define DPCC_EXECUTION_RECEIPT_CAPACITY 256 #define DPCC_EXECUTION_LOT_EPSILON 1.7763568394002505e-15 enum ENUM_DPCC_EXECUTION_RECEIPT_KIND { DPCC_EXECUTION_ORDER_STATE = 0, DPCC_EXECUTION_DEAL_FILL = 1 }; //--- Raw absolute OHLC: five available closed bars, oldest to newest; double //--- two-pass population variance/N, published float32, gain=1/bias=0, no floor. //--- This DPCC baseline is distinct from native PeriodNorm's float/epsilon policy. const string DPCC_STUDY_DECODER_NORMALIZATION_ID = "DPCC_INPUT5_OHLC_DOUBLE_POPULATION_V1"; //+------------------------------------------------------------------+ //| Shared StudyForecast Decoder architecture and normalization | //+------------------------------------------------------------------+ bool DPCCStudyDecoderContract(SDPCCConfig &config, SDPCCDecoderComposition &composition, string &normalization_digest, string &reason) { ZeroMemory(config); config.shape.K = 21; config.shape.V = 9; config.shape.H = 12; config.shape.D = 16; config.shape.M = 1; config.shape.J = 1; config.shape.E = 1; config.shape.PlanBatch = 1; config.shape.NActions = 6; config.shape.StateSize = 13; config.shape.Embedding = 64; config.shape.Heads = 4; config.shape.KeyDimension = 16; config.shape.Depth = 0; config.contract_version = 2; config.binary_layer_version = 3; config.normalization_mode = DPCC_NORMALIZATION_CAUSAL_AFFINE_V2; config.loss_coordinates = DPCC_LOSS_NORMALIZED; composition.embedding = 64; composition.architecture = "SHARED_VD_E4"; composition.activation_profile = "GELU"; composition.backward_profile = "DPCC_EXACT_CHAIN_V1"; composition.optimizer_profile = "NN_NATIVE_CONV_ADAM_C6A8_V1"; const string graph = "DPCC_STUDY_DECODER_GRAPH_V1\nSHARED_VD_E4\nGELU\nDPCC_EXACT_CHAIN_V1\n" + "NN_NATIVE_CONV_ADAM_C6A8_V1\n21:9:12:16:64\n"; if(!DPCCCheckpointTextSHA(graph, composition.graph_digest, reason)) return(false); return(DPCCCheckpointTextSHA(DPCC_STUDY_DECODER_NORMALIZATION_ID + "\n5_CLOSED_OLDEST_FIRST_DOUBLE_TWO_PASS_POPULATION_N_FLOAT32_GAIN1_BIAS0_NO_FLOOR\n", normalization_digest, reason)); } //+------------------------------------------------------------------+ //| Encoder receipt binds source, latent axes and the frozen teacher | //+------------------------------------------------------------------+ struct SDPCCStudyEncoderReceipt { string encoder_sha256, teacher_sha256, profile_sha256; uint fit_updates, holdout_examples; double latent_coordinate_mae; }; //+------------------------------------------------------------------+ //| Function DPCCStudyEncoderProfileSHA | //+------------------------------------------------------------------+ bool DPCCStudyEncoderProfileSHA(string &sha, string &reason) { const string profile = "DPCC_ENCODER_TEACHER_PROFILE_V1\n" "input=causal_five_closed_bars_feature9_variable_major_45\n" "encoder=Base45;BatchNorm45;Dropout0.1;CogData9_window5_units504,504,8;" "Rank16_window85_variables9_units504,4;OMPB9_window16_layers5_units256,64_BYPASS;" "Forecast21_TopK5_D16_H12_V9_RecoveryAge256\n" "teacher=Base108;PeriodNorm12x9;Conv3_to16_step1_GELU;" "Conv16_to32_step16_GELU;Conv32_to16_step32_None;PeriodNorm16x9\n" "teacher_input=future12_feature9_variable_major_108\n" "Z=21,9,12,16;t=((k*9+v)*12+h);width=16;teacher_label=9,12,16\n" "fit=2025;holdout=2026.01.01..2026.02.01;teacher=fresh_frozen_untrained\n" "objective=native_forecast_latent_responsibilities;timezone_resolution=NOT_CLAIMED\n"; return(DPCCCheckpointTextSHA(profile, sha, reason)); } //+------------------------------------------------------------------+ //| Decode a new Encoder receipt independently of legacy OHLC heads | //+------------------------------------------------------------------+ bool DPCCDecodeStudyEncoderReceipt(const string wire, const string expected_raw_sha, const uint fit_count, const uint holdout_count, SDPCCStudyEncoderReceipt &receipt, string &reason, const bool common_files = false) { reason = "study_encoder.receipt.schema"; string lines[], values[]; if(!DPCCArchitectureSHA256Encoding(expected_raw_sha) || fit_count == 0 || holdout_count == 0 || StringLen(wire) > 16384 || StringFind(wire, "\r") >= 0 || StringSplit(wire, '\n', lines) != 13 || lines[0] != "DPCC_ENCODER_TEACHER_STUDY_V1" || lines[12] != "") return(false); const string keys[] = {"raw_sha256", "encoder_sha256", "teacher_frozen_untrained_sha256", "profile_sha256", "fit_updates", "holdout_examples", "latent_coordinate_mae", "holdout_server_time", "teacher", "timezone_resolution", "research_acceptance" }; if(ArrayResize(values, 11) != 11) return(false); for(int i = 0; i < 11; i++) { const string prefix = keys[i] + "="; if(StringFind(lines[i + 1], prefix) != 0) return(false); values[i] = StringSubstr(lines[i + 1], StringLen(prefix)); } string profile = ""; if(!DPCCStudyEncoderProfileSHA(profile, reason)) return(false); reason = "study_encoder.receipt.binding"; if(values[0] != expected_raw_sha || !DPCCArchitectureSHA256Encoding(values[1]) || !DPCCArchitectureSHA256Encoding(values[2]) || values[3] != profile || values[7] != "2026.01.01..2026.02.01" || values[8] != "fresh_frozen_untrained" || values[9] != "NOT_CLAIMED" || values[10] != "NOT_CLAIMED") return(false); ulong fit = 0, holdout = 0; if(!DPCCCheckpointReadUnsigned(values[4], UINT_MAX, fit) || fit != fit_count || !DPCCCheckpointReadUnsigned(values[5], UINT_MAX, holdout) || holdout != holdout_count) return(false); CDPCCJsonDocument scalar; double loss = 0; if(!scalar.Parse(values[6], reason) || !scalar.TryNumber(scalar.Root(), loss) || !MathIsValidNumber(loss) || loss < 0 || DoubleToString(loss, 17) != values[6]) { reason = "study_encoder.receipt.loss"; return(false); } SDPCCStudyEncoderReceipt candidate; candidate.encoder_sha256 = values[1]; candidate.teacher_sha256 = values[2]; candidate.profile_sha256 = values[3]; candidate.fit_updates = (uint)fit; candidate.holdout_examples = (uint)holdout; candidate.latent_coordinate_mae = loss; receipt = candidate; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCReadStudyEncoderReceipt | //+------------------------------------------------------------------+ bool DPCCReadStudyEncoderReceipt(const string root, const string trusted_receipt_sha, const string expected_raw_sha, const uint fit_count, const uint holdout_count, SDPCCStudyEncoderReceipt &receipt, string &reason, const bool common_files = false) { string canonical = "", actual = "", wire = ""; if(!DPCCArchitectureSHA256Encoding(trusted_receipt_sha)) { reason = "study_encoder.restore.trusted_receipt_sha"; return(false); } if(!DPCCCheckpointCanonicalPath(root, canonical, reason) || canonical != root) return(false); uchar bytes[]; if(!DPCCReadArtifactBytes(root + "/EncoderTeacher.study.txt", common_files, 16384, bytes, reason) || !DPCCSHA256Lower(bytes, actual, reason) || actual != trusted_receipt_sha) { reason = "study_encoder.restore.receipt_sha"; return(false); } if(!DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 16384, wire, reason)) return(false); return(DPCCDecodeStudyEncoderReceipt(wire, expected_raw_sha, fit_count, holdout_count, receipt, reason)); } struct SDPCCStudyForecastReceipt { string market_sha256, target_sha256, decoder_sha256; uint forecast_fit_updates, decoder_fit_updates, holdout_examples; double decoder_normalized_rho_mse; }; //+------------------------------------------------------------------+ //| Decode the exact completed receipt using shared scalar readers | //+------------------------------------------------------------------+ bool DPCCDecodeStudyForecastReceipt(const string wire, const string expected_raw_sha, const uint fit_count, const uint holdout_count, SDPCCStudyForecastReceipt &receipt, string &reason) { reason = "study_decoder.receipt.schema"; string lines[], values[]; if(!DPCCArchitectureSHA256Encoding(expected_raw_sha) || fit_count == 0 || holdout_count == 0 || StringLen(wire) > 16384 || StringFind(wire, "\r") >= 0 || StringSplit(wire, '\n', lines) != 22 || lines[0] != "DPCC_FORECAST_DECODER_STUDY_V1" || lines[21] != "") return(false); const string keys[] = {"raw_sha256", "normalization_id", "normalization_policy_sha256", "decoder_graph_sha256", "market_sha256", "target_frozen_untrained_sha256", "decoder_sha256", "K", "V", "TopK", "H", "D", "decoder_width", "forecast_fit_updates", "decoder_fit_updates", "holdout_examples", "decoder_normalized_rho_mse", "holdout_server_time", "teacher", "research_acceptance" }; if(ArrayResize(values, 20) != 20) return(false); for(int i = 0; i < 20; i++) { const string prefix = keys[i] + "="; if(StringFind(lines[i + 1], prefix) != 0) return(false); values[i] = StringSubstr(lines[i + 1], StringLen(prefix)); } SDPCCConfig config; SDPCCDecoderComposition composition; string normalization = ""; if(!DPCCStudyDecoderContract(config, composition, normalization, reason)) return(false); reason = "study_decoder.receipt.binding"; if(values[0] != expected_raw_sha || values[1] != DPCC_STUDY_DECODER_NORMALIZATION_ID || values[2] != normalization || values[3] != composition.graph_digest || values[7] != "21" || values[8] != "9" || values[9] != "5" || values[10] != "12" || values[11] != "16" || values[12] != "64" || values[17] != "2026.01.01..2026.02.01" || values[18] != "fresh_frozen_untrained" || values[19] != "NOT_CLAIMED") return(false); for(int i = 4; i < 7; i++) if(!DPCCArchitectureSHA256Encoding(values[i])) return(false); ulong counters[3]; for(int i = 0; i < 3; i++) if(!DPCCCheckpointReadUnsigned(values[13 + i], UINT_MAX, counters[i])) return(false); if(counters[0] != fit_count || counters[1] != fit_count || counters[2] != holdout_count) return(false); CDPCCJsonDocument scalar; double loss = 0; if(!scalar.Parse(values[16], reason) || !scalar.TryNumber(scalar.Root(), loss) || !MathIsValidNumber(loss) || loss < 0 || DoubleToString(loss, 17) != values[16]) { reason = "study_decoder.receipt.loss"; return(false); } SDPCCStudyForecastReceipt candidate; candidate.market_sha256 = values[4]; candidate.target_sha256 = values[5]; candidate.decoder_sha256 = values[6]; candidate.forecast_fit_updates = (uint)counters[0]; candidate.decoder_fit_updates = (uint)counters[1]; candidate.holdout_examples = (uint)counters[2]; candidate.decoder_normalized_rho_mse = loss; receipt = candidate; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Verify the caller's independent receipt pin before decoding | //+------------------------------------------------------------------+ bool DPCCReadStudyForecastReceipt(const string root, const string trusted_receipt_sha, const string expected_raw_sha, const uint fit_count, const uint holdout_count, SDPCCStudyForecastReceipt &receipt, string &reason) { string canonical = "", actual = "", wire = ""; if(!DPCCArchitectureSHA256Encoding(trusted_receipt_sha)) { reason = "study_decoder.restore.trusted_receipt_sha"; return(false); } if(!DPCCCheckpointCanonicalPath(root, canonical, reason) || canonical != root) return(false); uchar bytes[]; if(!DPCCReadArtifactBytes(root + "/ForecastDecoder.study.txt", false, 16384, bytes, reason) || !DPCCSHA256Lower(bytes, actual, reason) || actual != trusted_receipt_sha) { reason = "study_decoder.restore.receipt_sha"; return(false); } if(!DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 16384, wire, reason)) return(false); return(DPCCDecodeStudyForecastReceipt(wire, expected_raw_sha, fit_count, holdout_count, receipt, reason)); } //+------------------------------------------------------------------+ //| Function DPCCPrepareStudyDecoderNormalization | //+------------------------------------------------------------------+ bool DPCCPrepareStudyDecoderNormalization(const SDPCCStudyForecastBar &bars[], const int decision, float &price_norm[], string &reason) { ArrayResize(price_norm, 0); reason = ""; if(decision < 5 || decision > ArraySize(bars)) {reason = "study_decoder.normalization.history"; return(false);} double means[4]; ZeroMemory(means); for(int b = decision - 5; b < decision; b++) { if(bars[b].server_open <= 0 || (b > decision - 5 && bars[b].server_open <= bars[b - 1].server_open) || (decision < ArraySize(bars) && bars[b].server_open >= bars[decision].server_open)) {reason = "study_decoder.normalization.causal_time"; return(false);} for(int c = 0; c < 4; c++) { const double value = bars[b].ohlc[c]; if(!MathIsValidNumber(value) || value <= 0) {reason = "study_decoder.normalization.ohlc"; return(false);} means[c] += value; } if(bars[b].ohlc[1] < MathMax(bars[b].ohlc[0], bars[b].ohlc[3]) || bars[b].ohlc[2] > MathMin(bars[b].ohlc[0], bars[b].ohlc[3])) {reason = "study_decoder.normalization.geometry"; return(false);} } float candidate[]; if(ArrayResize(candidate, 16) != 16) {reason = "study_decoder.normalization.capacity"; return(false);} for(int c = 0; c < 4; c++) { means[c] /= 5.0; double squares = 0; for(int b = decision - 5; b < decision; b++) {const double delta = bars[b].ohlc[c] - means[c]; squares += delta * delta;} const double variance = squares / 5.0; const double std = MathSqrt(variance); const float fm = (float)means[c], fs = (float)std, fv = fs * fs; if(!MathIsValidNumber(means[c]) || !MathIsValidNumber(variance) || variance <= 0 || !MathIsValidNumber(std) || std <= 0 || !MathIsValidNumber(fm) || !MathIsValidNumber(fs) || fs <= 0 || !MathIsValidNumber(fv) || fv <= 0) {reason = "study_decoder.normalization.positive_representable_variance"; return(false);} candidate[c * 4] = fm; candidate[c * 4 + 1] = fs; candidate[c * 4 + 2] = 1; candidate[c * 4 + 3] = 0; } if(ArrayCopy(price_norm, candidate) != 16) {ArrayResize(price_norm, 0); reason = "study_decoder.normalization.publish"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Prepare one linked closed bar using the shared causal transforms | //+------------------------------------------------------------------+ bool DPCCPrepareStudyLiveInput(const SDPCCStudyForecastBar &prior[], const SDPCCClosedHistoryRow &row, const long expected_open, const long expected_ordinal, float &state[], float &price_norm[], string &reason) { //--- Admit the observation and its observed successor before preparing any output. ArrayResize(state, 0); ArrayResize(price_norm, 0); reason = ""; const int count = ArraySize(prior); if(count < 4 || expected_open <= 0 || expected_ordinal < 0 || row.ordinal != expected_ordinal || row.bar.server_open != expected_open || row.bar.server_open <= prior[count - 1].server_open || row.successor_open_server <= row.bar.server_open || row.spread < 0 || row.spread > INT_MAX || row.real_volume < 0 || row.bar.spread != row.spread) { reason = "study_decoder.live.observation_link"; return(false); } SDPCCStudyForecastBar context[]; if(ArrayResize(context, 5) != 5) { reason = "study_decoder.live.context_capacity"; return(false); } for(int i = 0; i < 4; i++) context[i] = prior[count - 4 + i]; context[4] = row.bar; for(int i = 0; i < 5; i++) { float features[]; if(context[i].server_open <= 0 || (i > 0 && context[i].server_open <= context[i - 1].server_open) || context[i].spread < 0 || context[i].spread > INT_MAX || !DPCCStudyForecastFeature(context[i], features)) { reason = "study_decoder.live.context_observation"; return(false); } } //--- Include the newly closed bar exactly once as the newest causal input. float candidate_state[], candidate_norm[]; if(!DPCCPackStudyForecastInput(context, 5, candidate_state) || ArraySize(candidate_state) != 45 || !DPCCPrepareStudyDecoderNormalization(context, 5, candidate_norm, reason) || ArraySize(candidate_norm) != 16) { reason = "study_decoder.live.causal_transform:" + reason; return(false); } if(ArrayCopy(state, candidate_state) != 45 || ArrayCopy(price_norm, candidate_norm) != 16) { ArrayResize(state, 0); ArrayResize(price_norm, 0); reason = "study_decoder.live.input_publish"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Rebase one locally parsed capture onto the restored live stream | //+------------------------------------------------------------------+ bool DPCCRebaseStudyLiveRow(const SDPCCClosedHistoryRow &captured, const long expected_open, const long expected_ordinal, SDPCCClosedHistoryRow &stream_row, string &reason) { ZeroMemory(stream_row); reason = ""; if(captured.ordinal != 0 || expected_open <= 0 || expected_ordinal < 0 || captured.bar.server_open != expected_open || captured.successor_open_server <= expected_open) { reason = "study_decoder.live.capture_link"; return(false); } SDPCCClosedHistoryRow candidate; candidate = captured; candidate.ordinal = expected_ordinal; stream_row = candidate; return(true); } //--- Detached labels only; call after the causal Forecast forward has finished. bool DPCCPackStudyDecoderNativeTarget(const SDPCCStudyForecastBar &bars[], const int decision, const uint horizon, float &target[], string &reason) { ArrayResize(target, 0); reason = ""; if(decision < 5 || horizon == 0 || horizon > uint(INT_MAX / 4) || decision > ArraySize(bars) || (ulong)decision + horizon > (ulong)ArraySize(bars)) {reason = "study_decoder.target.shape"; return(false);} float candidate[]; if(ArrayResize(candidate, (int)horizon * 4) != (int)horizon * 4) {reason = "study_decoder.target.capacity"; return(false);} for(uint h = 0; h < horizon; h++) { const int row = decision + (int)h; if(bars[row].server_open <= bars[row - 1].server_open || bars[row].ohlc[1] < MathMax(bars[row].ohlc[0], bars[row].ohlc[3]) || bars[row].ohlc[2] > MathMin(bars[row].ohlc[0], bars[row].ohlc[3])) {reason = "study_decoder.target.geometry_or_order"; return(false);} for(uint c = 0; c < 4; c++) { const float value = (float)bars[row].ohlc[c]; if(!MathIsValidNumber(bars[row].ohlc[c]) || !MathIsValidNumber(value) || value <= 0) {reason = "study_decoder.target.ohlc"; return(false);} candidate[h * 4 + c] = value; } } if(ArrayCopy(target, candidate) != (int)horizon * 4) {ArrayResize(target, 0); reason = "study_decoder.target.publish"; return(false);} return(true); } //--- One already-forwarded native Forecast decision plus its factual, detached H1 label. //--- This is an in-memory witness contract; bank encoding/admission remains downstream. struct SDPCCStudyForecastWitness { bool valid; string raw_sha256, teacher; int decision_index; long decision_server_open; ulong market_fingerprint, target_fingerprint; float z[], u[], pi[], rho[], scenarios[], price_norm[], target_ohlc[]; long horizon_server_opens[]; }; //+------------------------------------------------------------------+ //| Function DPCCClearStudyForecastWitness | //+------------------------------------------------------------------+ void DPCCClearStudyForecastWitness(SDPCCStudyForecastWitness &witness) { ZeroMemory(witness); witness.raw_sha256 = ""; witness.teacher = ""; ArrayResize(witness.z, 0); ArrayResize(witness.u, 0); ArrayResize(witness.pi, 0); ArrayResize(witness.rho, 0); ArrayResize(witness.scenarios, 0); ArrayResize(witness.price_norm, 0); ArrayResize(witness.target_ohlc, 0); ArrayResize(witness.horizon_server_opens, 0); } //+------------------------------------------------------------------+ //| Function DPCCValidateStudyForecastWitness | //+------------------------------------------------------------------+ bool DPCCValidateStudyForecastWitness(const SDPCCStudyForecastBar &bars[], const SDPCCStudyForecastWitness &witness, string &reason) { reason = "study_forecast.witness.contract"; const int horizon = 12; if(!witness.valid || witness.teacher != "fresh_frozen_untrained_random_target" || !DPCCArchitectureSHA256Encoding(witness.raw_sha256) || witness.decision_index < 5 || witness.decision_index >= ArraySize(bars) || witness.decision_server_open <= 0 || bars[witness.decision_index].server_open != witness.decision_server_open || ArraySize(witness.z) != 21 * 9 * 12 * 16 || ArraySize(witness.u) != 21 * 9 * 12 || ArraySize(witness.pi) != 21 || ArraySize(witness.rho) != 21 || ArraySize(witness.scenarios) != 21 * 12 * 4 || ArraySize(witness.price_norm) != 16 || ArraySize(witness.target_ohlc) != horizon * 4 || ArraySize(witness.horizon_server_opens) != horizon) return(false); double pi_sum = 0, rho_sum = 0; for(int i = 0; i < ArraySize(witness.z); i++) if(!MathIsValidNumber((double)witness.z[i])) return(false); for(int i = 0; i < ArraySize(witness.u); i++) if(!MathIsValidNumber((double)witness.u[i]) || witness.u[i] < 0) return(false); for(int k = 0; k < 21; k++) { if(!MathIsValidNumber((double)witness.pi[k]) || witness.pi[k] < 0 || witness.pi[k] > 1 || !MathIsValidNumber((double)witness.rho[k]) || witness.rho[k] < 0 || witness.rho[k] > 1) return(false); pi_sum += witness.pi[k]; rho_sum += witness.rho[k]; } if(MathAbs(pi_sum - 1.0) > 1e-5 || MathAbs(rho_sum - 1.0) > 1e-5) return(false); for(int i = 0; i < ArraySize(witness.scenarios); i++) if(!MathIsValidNumber((double)witness.scenarios[i])) return(false); for(int c = 0; c < 4; c++) { const float mean = witness.price_norm[c * 4], std = witness.price_norm[c * 4 + 1]; if(!MathIsValidNumber((double)mean) || !MathIsValidNumber((double)std) || mean <= 0 || std <= 0 || witness.price_norm[c * 4 + 2] != 1 || witness.price_norm[c * 4 + 3] != 0 || !MathIsValidNumber((double)(std * std)) || std * std <= 0) return(false); } for(int h = 0; h < horizon; h++) if(witness.horizon_server_opens[h] <= 0 || (h > 0 && witness.horizon_server_opens[h] <= witness.horizon_server_opens[h - 1])) return(false); if(!DPCCStudyForecastMatchNativeHorizon(bars, witness.decision_index, witness.target_ohlc, witness.horizon_server_opens, reason)) return(false); reason = ""; return(true); } //--- Export only buffers from the current Forecast pass; this routine never forwards or updates a net. bool DPCCCaptureStudyForecastWitness(CNeuronScenarioForecast *forecast, const SDPCCStudyForecastBar &bars[], const int decision, const string raw_sha256, const ulong market_fingerprint, const ulong target_fingerprint, const float &scenarios[], const float &price_norm[], SDPCCStudyForecastWitness &witness, string &reason) { DPCCClearStudyForecastWitness(witness); reason = "study_forecast.witness.native_state"; if(CheckPointer(forecast) == POINTER_INVALID || forecast.Type() != defNeuronScenarioForecast || forecast.CodebookUpdate() || !forecast.LastTargetAccepted() || !forecast.LastBatchControlValid() || forecast.Scenarios() != 21 || forecast.Variables() != 9 || forecast.Horizon() != 12 || forecast.Dimension() != 16 || decision < 5 || decision >= ArraySize(bars) || !DPCCArchitectureSHA256Encoding(raw_sha256) || ArraySize(scenarios) != 21 * 12 * 4 || ArraySize(price_norm) != 16) return(false); CBufferFloat *z = forecast.GetZ(), *u = forecast.GetU(), *pi = forecast.GetPi(), *rho = forecast.GetResponsibilities( ); if(CheckPointer(z) == POINTER_INVALID || CheckPointer(u) == POINTER_INVALID || CheckPointer(pi) == POINTER_INVALID || CheckPointer(rho) == POINTER_INVALID || z.Total() != 21 * 9 * 12 * 16 || u.Total() != 21 * 9 * 12 || pi.Total() != 21 || rho.Total() != 21 || !z.BufferRead() || !u.BufferRead() || !pi.BufferRead() || !rho.BufferRead()) return(false); SDPCCStudyForecastWitness candidate; DPCCClearStudyForecastWitness(candidate); candidate.raw_sha256 = raw_sha256; candidate.teacher = "fresh_frozen_untrained_random_target"; candidate.decision_index = decision; candidate.decision_server_open = bars[decision].server_open; candidate.market_fingerprint = market_fingerprint; candidate.target_fingerprint = target_fingerprint; float target[]; if(z.GetData(candidate.z) != 21 * 9 * 12 * 16 || u.GetData(candidate.u) != 21 * 9 * 12 || pi.GetData(candidate.pi) != 21 || rho.GetData(candidate.rho) != 21 || ArrayCopy(candidate.scenarios, scenarios) != 21 * 12 * 4 || ArrayCopy(candidate.price_norm, price_norm) != 16 || !DPCCPackStudyDecoderNativeTarget(bars, decision, 12, target, reason) || ArrayCopy(candidate.target_ohlc, target) != 48 || ArrayResize(candidate.horizon_server_opens, 12) != 12) return(false); for(int h = 0; h < 12; h++) candidate.horizon_server_opens[h] = bars[decision + h].server_open; candidate.valid = true; if(!DPCCValidateStudyForecastWitness(bars, candidate, reason)) return(false); witness.valid = candidate.valid; witness.raw_sha256 = candidate.raw_sha256; witness.teacher = candidate.teacher; witness.decision_index = candidate.decision_index; witness.decision_server_open = candidate.decision_server_open; witness.market_fingerprint = candidate.market_fingerprint; witness.target_fingerprint = candidate.target_fingerprint; if(ArrayCopy(witness.z, candidate.z) != ArraySize(candidate.z) || ArrayCopy(witness.u, candidate.u) != ArraySize(candidate.u) || ArrayCopy(witness.pi, candidate.pi) != ArraySize(candidate.pi) || ArrayCopy(witness.rho, candidate.rho) != ArraySize(candidate.rho) || ArrayCopy(witness.scenarios, candidate.scenarios) != ArraySize(candidate.scenarios) || ArrayCopy(witness.price_norm, candidate.price_norm) != ArraySize(candidate.price_norm) || ArrayCopy(witness.target_ohlc, candidate.target_ohlc) != ArraySize(candidate.target_ohlc) || ArrayCopy(witness.horizon_server_opens, candidate.horizon_server_opens) != ArraySize( candidate.horizon_server_opens)) { DPCCClearStudyForecastWitness(witness); reason = "study_forecast.witness.publish"; return(false); } reason = ""; return(true); } enum ENUM_DPCC_EXECUTION_ORDER_STATE { DPCC_ORDER_PENDING = 0, DPCC_ORDER_AMBIGUOUS = 1, DPCC_ORDER_PARTIAL = 2, DPCC_ORDER_FILLED = 3, DPCC_ORDER_REJECTED = 4, DPCC_ORDER_CANCELED = 5 }; struct SDPCCExecutionIntent { string terminal_binding_wire, terminal_result_wire; ulong intent_id, decision_identity, order_ticket; long magic; string deployment_id, symbol, policy_sha256, checkpoint_sha256, account_snapshot_sha256, intent_sha256; double current_signed_lots, requested_signed_delta_lots, target_signed_lots, actions[6], filled_signed_lots; ENUM_DPCC_EXECUTION_ORDER_STATE order_state; long last_order_state_time_msc; uint receipt_count; string receipt_keys[DPCC_EXECUTION_RECEIPT_CAPACITY]; string receipt_digests[DPCC_EXECUTION_RECEIPT_CAPACITY]; }; struct SDPCCExecutionReceipt { ulong intent_id, order_ticket, deal_id; long magic, time_msc; string deployment_id, symbol, intent_sha256; ENUM_DPCC_EXECUTION_RECEIPT_KIND kind; ENUM_DPCC_EXECUTION_ORDER_STATE status; double signed_fill_lots, fill_price; string canonical_digest; }; //+------------------------------------------------------------------+ //| Function DPCCExecutionReceiptCanonical | //+------------------------------------------------------------------+ bool DPCCExecutionReceiptCanonical(const SDPCCExecutionReceipt &receipt, string &key, string &digest, string &reason) { key = ""; digest = ""; reason = ""; if(receipt.intent_id == 0 || receipt.deployment_id == "" || receipt.magic == 0 || receipt.symbol == "" || receipt.time_msc <= 0) {reason = "execution_receipt.identity"; return(false);} if(!DPCCArchitectureSHA256Encoding(receipt.intent_sha256) || (receipt.order_ticket == 0 && !(receipt.kind == DPCC_EXECUTION_ORDER_STATE && receipt.status == DPCC_ORDER_AMBIGUOUS))) {reason = "execution_receipt.intent_binding_or_ticket"; return(false);} string payload = ""; if(receipt.kind == DPCC_EXECUTION_ORDER_STATE) { if(receipt.deal_id != 0 || receipt.signed_fill_lots != 0 || receipt.fill_price != 0 || (receipt.status != DPCC_ORDER_PENDING && receipt.status != DPCC_ORDER_AMBIGUOUS && receipt.status != DPCC_ORDER_REJECTED && receipt.status != DPCC_ORDER_CANCELED)) {reason = "execution_receipt.order_state_fields"; return(false);} key = "O:" + (string)receipt.order_ticket + ":" + (string)receipt.time_msc; } else if(receipt.kind == DPCC_EXECUTION_DEAL_FILL) { if(receipt.deal_id == 0 || !MathIsValidNumber(receipt.signed_fill_lots) || receipt.signed_fill_lots == 0 || !MathIsValidNumber(receipt.fill_price) || receipt.fill_price <= 0 || (receipt.status != DPCC_ORDER_PARTIAL && receipt.status != DPCC_ORDER_FILLED)) {reason = "execution_receipt.deal_fields"; return(false);} key = "D:" + (string)receipt.deal_id; } else {reason = "execution_receipt.kind"; return(false);} if(!DPCCExecutionAppend(payload, "schema", "DPCC_EXECUTION_RECEIPT_V1") || !DPCCExecutionAppend(payload, "kind", (string)receipt.kind) || !DPCCExecutionAppend(payload, "intent_id", (string)receipt.intent_id) || !DPCCExecutionAppend(payload, "intent_sha256", receipt.intent_sha256) || !DPCCExecutionAppend(payload, "deployment_id", receipt.deployment_id) || !DPCCExecutionAppend(payload, "symbol", receipt.symbol) || !DPCCExecutionAppend(payload, "magic", (string)receipt.magic) || !DPCCExecutionAppend(payload, "order_ticket", (string)receipt.order_ticket) || !DPCCExecutionAppend(payload, "deal_id", (string)receipt.deal_id) || !DPCCExecutionAppend(payload, "status", (string)receipt.status) || !DPCCExecutionAppend(payload, "signed_fill_lots", StringFormat("%.17g", receipt.signed_fill_lots)) || !DPCCExecutionAppend(payload, "fill_price", StringFormat("%.17g", receipt.fill_price)) || !DPCCExecutionAppend(payload, "time_msc", (string)receipt.time_msc) || !DPCCCheckpointTextSHA(payload, digest, reason)) {if(reason == "")reason = "execution_receipt.canonical_payload"; return(false);} if(receipt.canonical_digest != "" && (!DPCCArchitectureSHA256Encoding(receipt.canonical_digest) || digest != receipt.canonical_digest)) {reason = "execution_receipt.digest_mismatch"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCExecutionIntentSHA | //+------------------------------------------------------------------+ bool DPCCExecutionIntentSHA(const SDPCCExecutionIntent &intent, string &digest, string &reason) { digest = ""; reason = ""; string payload = ""; if(!DPCCExecutionAppend(payload, "schema", "DPCC_EXECUTION_INTENT_V1") || !DPCCExecutionAppend(payload, "intent_id", (string)intent.intent_id) || !DPCCExecutionAppend(payload, "decision_identity", (string)intent.decision_identity) || !DPCCExecutionAppend(payload, "deployment_id", intent.deployment_id) || !DPCCExecutionAppend(payload, "magic", (string)intent.magic) || !DPCCExecutionAppend(payload, "symbol", intent.symbol) || !DPCCExecutionAppend(payload, "policy_sha256", intent.policy_sha256) || !DPCCExecutionAppend(payload, "checkpoint_sha256", intent.checkpoint_sha256) || !DPCCExecutionAppend(payload, "account_snapshot_sha256", intent.account_snapshot_sha256) || !DPCCExecutionAppend(payload, "current_signed_lots", StringFormat("%.17g", intent.current_signed_lots)) || !DPCCExecutionAppend(payload, "requested_signed_delta_lots", StringFormat("%.17g", intent.requested_signed_delta_lots)) || !DPCCExecutionAppend(payload, "target_signed_lots", StringFormat("%.17g", intent.target_signed_lots))) {reason = "execution_intent.digest_payload"; return(false);} for(int i = 0; i < 6; i++) if(!DPCCExecutionAppend(payload, StringFormat("action_%d", i), StringFormat("%.17g", intent.actions[i]))) {reason = "execution_intent.digest_actions"; return(false);} return(DPCCCheckpointTextSHA(payload, digest, reason)); } //+------------------------------------------------------------------+ //| Function DPCCPrepareExecutionIntent | //+------------------------------------------------------------------+ bool DPCCPrepareExecutionIntent(const ulong intent_id, const ulong decision_identity, const string deployment_id, const long magic, const string symbol, const string policy_sha256, const string checkpoint_sha256, const string account_snapshot_sha256, const double current_signed_lots, const double requested_signed_delta_lots, const double target_signed_lots, const double &actions[], SDPCCExecutionIntent &intent, string &reason) { reason = ""; if(intent_id == 0 || decision_identity == 0 || deployment_id == "" || magic == 0 || symbol == "" || !DPCCArchitectureSHA256Encoding(policy_sha256) || !DPCCArchitectureSHA256Encoding(checkpoint_sha256) || !DPCCArchitectureSHA256Encoding(account_snapshot_sha256) || !MathIsValidNumber(current_signed_lots) || !MathIsValidNumber(requested_signed_delta_lots) || !MathIsValidNumber(target_signed_lots) || ArraySize(actions) != 6) {reason = "execution_intent.identity_or_shape"; return(false);} const double exposure_tolerance = DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, MathMax(MathAbs(target_signed_lots), MathAbs(current_signed_lots))); if(MathAbs(requested_signed_delta_lots - (target_signed_lots - current_signed_lots)) > exposure_tolerance) {reason = "execution_intent.delta_target_current_mismatch"; return(false);} for(int i = 0; i < 6; i++) if(!MathIsValidNumber(actions[i]) || actions[i] < 0 || actions[i] > 1) {reason = StringFormat("execution_intent.action.%d", i); return(false);} SDPCCExecutionIntent candidate; candidate.terminal_binding_wire = ""; candidate.terminal_result_wire = ""; candidate.intent_id = intent_id; candidate.decision_identity = decision_identity; candidate.deployment_id = deployment_id; candidate.magic = magic; candidate.symbol = symbol; candidate.policy_sha256 = policy_sha256; candidate.checkpoint_sha256 = checkpoint_sha256; candidate.account_snapshot_sha256 = account_snapshot_sha256; candidate.requested_signed_delta_lots = requested_signed_delta_lots; candidate.current_signed_lots = current_signed_lots; candidate.target_signed_lots = target_signed_lots; candidate.order_ticket = 0; candidate.filled_signed_lots = 0; candidate.receipt_count = 0; candidate.order_state = DPCC_ORDER_PENDING; candidate.last_order_state_time_msc = 0; for(int i = 0; i < DPCC_EXECUTION_RECEIPT_CAPACITY; i++) {candidate.receipt_keys[i] = ""; candidate.receipt_digests[i] = "";} for(int i = 0; i < 6; i++) candidate.actions[i] = actions[i]; string candidate_intent_sha = ""; if(!DPCCExecutionIntentSHA(candidate, candidate_intent_sha, reason)) return(false); candidate.intent_sha256 = candidate_intent_sha; //--- Publish only after the full candidate is valid and initialized. intent.intent_id = candidate.intent_id; intent.decision_identity = candidate.decision_identity; intent.terminal_binding_wire = ""; intent.terminal_result_wire = ""; intent.deployment_id = candidate.deployment_id; intent.magic = candidate.magic; intent.symbol = candidate.symbol; intent.policy_sha256 = candidate.policy_sha256; intent.checkpoint_sha256 = candidate.checkpoint_sha256; intent.account_snapshot_sha256 = candidate.account_snapshot_sha256; intent.intent_sha256 = candidate.intent_sha256; intent.current_signed_lots = candidate.current_signed_lots; intent.requested_signed_delta_lots = candidate.requested_signed_delta_lots; intent.target_signed_lots = candidate.target_signed_lots; intent.order_ticket = candidate.order_ticket; intent.filled_signed_lots = candidate.filled_signed_lots; intent.order_state = candidate.order_state; intent.last_order_state_time_msc = candidate.last_order_state_time_msc; intent.receipt_count = candidate.receipt_count; for(int i = 0; i < DPCC_EXECUTION_RECEIPT_CAPACITY; i++) {intent.receipt_keys[i] = candidate.receipt_keys[i]; intent.receipt_digests[i] = candidate.receipt_digests[i];} for(int i = 0; i < 6; i++) intent.actions[i] = candidate.actions[i]; return(true); } //| Stage all validation first; mutate the caller only after every gate passes. bool DPCCApplyExecutionReceipt(SDPCCExecutionIntent &intent, const SDPCCExecutionReceipt &receipt, string &reason) { reason = ""; if(intent.intent_id == 0 || intent.decision_identity == 0 || intent.deployment_id == "" || intent.magic == 0 || intent.symbol == "" || !DPCCArchitectureSHA256Encoding(intent.policy_sha256) || !DPCCArchitectureSHA256Encoding(intent.checkpoint_sha256) || !DPCCArchitectureSHA256Encoding(intent.account_snapshot_sha256) || !MathIsValidNumber(intent.requested_signed_delta_lots) || !MathIsValidNumber(intent.current_signed_lots) || !MathIsValidNumber(intent.target_signed_lots) || !MathIsValidNumber(intent.filled_signed_lots) || MathAbs(intent.requested_signed_delta_lots - (intent.target_signed_lots - intent.current_signed_lots)) > DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, MathMax(MathAbs(intent.target_signed_lots), MathAbs(intent.current_signed_lots))) || MathAbs(intent.filled_signed_lots) > MathAbs(intent.requested_signed_delta_lots) || (intent.filled_signed_lots != 0 && ((intent.filled_signed_lots > 0) != (intent.requested_signed_delta_lots > 0))) || intent.order_state < DPCC_ORDER_PENDING || intent.order_state > DPCC_ORDER_CANCELED) {reason = "execution_receipt.intent_state_invalid"; return(false);} for(int i = 0; i < 6; i++) if(!MathIsValidNumber(intent.actions[i]) || intent.actions[i] < 0 || intent.actions[i] > 1) {reason = "execution_receipt.intent_action_invalid"; return(false);} string key, digest; string observed_intent_sha = ""; if(!DPCCExecutionIntentSHA(intent, observed_intent_sha, reason) || !DPCCArchitectureSHA256Encoding(intent.intent_sha256) || observed_intent_sha != intent.intent_sha256) {reason = "execution_receipt.intent_digest_mismatch"; return(false);} if(!DPCCExecutionReceiptCanonical(receipt, key, digest, reason)) return(false); if(!DPCCArchitectureSHA256Encoding(receipt.canonical_digest) || digest != receipt.canonical_digest) {reason = "execution_receipt.digest_required"; return(false);} if(receipt.intent_id != intent.intent_id || receipt.intent_sha256 != intent.intent_sha256 || receipt.deployment_id != intent.deployment_id || receipt.magic != intent.magic || receipt.symbol != intent.symbol) {reason = "execution_receipt.foreign_binding"; return(false);} if(intent.order_ticket != 0 && intent.order_ticket != receipt.order_ticket) {reason = "execution_receipt.conflicting_order_ticket"; return(false);} if(intent.receipt_count > DPCC_EXECUTION_RECEIPT_CAPACITY) {reason = "execution_receipt.corrupt_count"; return(false);} for(uint i = 0; i < intent.receipt_count; i++) if(intent.receipt_keys[i] == key) { if(intent.receipt_digests[i] == digest) return(true); reason = "execution_receipt.conflicting_duplicate"; return(false); } if(intent.receipt_count >= DPCC_EXECUTION_RECEIPT_CAPACITY) {reason = "execution_receipt.capacity"; return(false);} if(intent.order_state == DPCC_ORDER_AMBIGUOUS || intent.order_state == DPCC_ORDER_REJECTED || intent.order_state == DPCC_ORDER_CANCELED || intent.order_state == DPCC_ORDER_FILLED) {reason = "execution_receipt.closed_or_ambiguous_intent"; return(false);} double staged_filled = intent.filled_signed_lots; ENUM_DPCC_EXECUTION_ORDER_STATE staged_state = receipt.status; long staged_order_state_time = intent.last_order_state_time_msc; if(receipt.kind == DPCC_EXECUTION_DEAL_FILL) { const double lot_scale = MathMax(1.0, MathMax(MathAbs(intent.requested_signed_delta_lots), MathMax(MathAbs(intent.current_signed_lots), MathMax(MathAbs(intent.target_signed_lots), MathMax(MathAbs(staged_filled), MathAbs( receipt.signed_fill_lots)))))); const double lot_tolerance = DPCC_EXECUTION_LOT_EPSILON * lot_scale; if(!MathIsValidNumber(intent.requested_signed_delta_lots) || !MathIsValidNumber(staged_filled) || (receipt.signed_fill_lots > 0) != (intent.requested_signed_delta_lots > 0) || MathAbs(receipt.signed_fill_lots) > MathAbs(intent.requested_signed_delta_lots) - MathAbs(staged_filled) + lot_tolerance) {reason = "execution_receipt.overfill_or_direction"; return(false);} staged_filled += receipt.signed_fill_lots; if(!MathIsValidNumber(staged_filled)) {reason = "execution_receipt.cumulative_numeric"; return(false);} const double cumulative_error = MathAbs(MathAbs(staged_filled) - MathAbs(intent.requested_signed_delta_lots)); if(cumulative_error <= lot_tolerance) staged_filled = intent.requested_signed_delta_lots; const bool complete_fill = (staged_filled == intent.requested_signed_delta_lots); if((receipt.status == DPCC_ORDER_FILLED) != complete_fill) {reason = "execution_receipt.fill_status_mismatch"; return(false);} if(complete_fill) staged_state = DPCC_ORDER_FILLED; else staged_state = DPCC_ORDER_PARTIAL; } else { if(receipt.time_msc <= intent.last_order_state_time_msc || (intent.order_state == DPCC_ORDER_PARTIAL && receipt.status == DPCC_ORDER_PENDING)) {reason = "execution_receipt.order_state_regression"; return(false);} if(receipt.status == DPCC_ORDER_PENDING && intent.order_state != DPCC_ORDER_PENDING) {reason = "execution_receipt.order_state_regression"; return(false);} staged_order_state_time = receipt.time_msc; } //--- Publish staged values only after validation and bounded storage capacity are known. intent.order_ticket = receipt.order_ticket; intent.filled_signed_lots = staged_filled; intent.order_state = staged_state; intent.last_order_state_time_msc = staged_order_state_time; intent.receipt_keys[intent.receipt_count] = key; intent.receipt_digests[intent.receipt_count] = digest; intent.receipt_count++; return(true); } //+------------------------------------------------------------------+ //| Canonical full terminal request, without a trading sender | //+------------------------------------------------------------------+ string DPCCExecutionTerminalComment(const SDPCCExecutionIntent &intent) { return("DPCC:" + StringSubstr(intent.intent_sha256, 0, 24)); } //+------------------------------------------------------------------+ //| Function DPCCExecutionTerminalText | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalText(const string value) { return(DPCCExecutionTranscriptText(value) && StringFind(value, "^") < 0); } //+------------------------------------------------------------------+ //| Function DPCCExecutionTerminalRequestWire | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalRequestWire(const SDPCCExecutionIntent &intent, const MqlTradeRequest &request, string &wire, string &reason) { reason = ""; wire = ""; string observed = ""; if(!DPCCExecutionIntentSHA(intent, observed, reason) || observed != intent.intent_sha256 || intent.magic <= 0 || request.magic != (ulong)intent.magic || request.symbol != intent.symbol || request.comment != DPCCExecutionTerminalComment(intent) || request.action != TRADE_ACTION_DEAL || request.order != 0 || request.position_by != 0 || request.stoplimit != 0 || request.type != (intent.requested_signed_delta_lots > 0 ? ORDER_TYPE_BUY : ORDER_TYPE_SELL) || request.type_filling < ORDER_FILLING_FOK || request.type_filling > ORDER_FILLING_RETURN || request.type_time != ORDER_TIME_GTC || request.expiration != 0 || !DPCCExecutionTerminalText(request.symbol) || !DPCCExecutionTerminalText(request.comment)) {reason = "terminal.request.identity_or_kind"; return(false);} const double numbers[] = {request.volume, request.price, request.stoplimit, request.sl, request.tp}; for(int i = 0; i < ArraySize(numbers); i++) if(!MathIsValidNumber(numbers[i]) || numbers[i] < 0) {reason = "terminal.request.numeric"; return(false);} if(request.volume <= 0 || MathAbs(request.volume - MathAbs(intent.requested_signed_delta_lots)) > DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume)) {reason = "terminal.request.volume"; return(false);} wire = (string)request.action + "^" + (string)request.magic + "^" + (string)request.order + "^" + request.symbol + "^"; for(int i = 0; i < 5; i++) wire += DPCCPhysicalDoubleWire(numbers[i]) + "^"; wire += (string)request.deviation + "^" + (string)request.type + "^" + (string)request.type_filling + "^"; wire += (string)request.type_time + "^" + (string)request.expiration + "^" + request.comment + "^"; wire += (string)request.position + "^" + (string)request.position_by; return(true); } //+------------------------------------------------------------------+ //| Bind the request to the predecessor before a future dispatch | //+------------------------------------------------------------------+ bool DPCCBindTerminalRequest(SDPCCExecutionIntent &intent, const MqlTradeRequest &request, const long login, const string server, const string terminal_path, const long quote_time_msc, const long dispatch_time_msc, string &reason) { string request_wire = "", request_sha = ""; if(intent.terminal_binding_wire != "" || intent.receipt_count != 0 || intent.order_ticket != 0 || login <= 0 || server == "" || terminal_path == "" || quote_time_msc <= 0 || dispatch_time_msc < quote_time_msc || !DPCCExecutionTerminalText(server) || !DPCCExecutionTerminalText(terminal_path) || !DPCCExecutionTerminalRequestWire(intent, request, request_wire, reason) || !DPCCCheckpointTextSHA(request_wire, request_sha, reason)) {if(reason == "")reason = "terminal.binding.boundary"; return(false);} intent.terminal_binding_wire = "DPCC_TERMINAL_REQUEST_V1^" + intent.intent_sha256 + "^" + intent.checkpoint_sha256 + "^"; intent.terminal_binding_wire += (string)login + "^" + server + "^" + terminal_path + "^" + (string)quote_time_msc + "^"; intent.terminal_binding_wire += (string)dispatch_time_msc + "^0^" + request_sha + "^" + request_wire + "^END_TERMINAL_REQUEST"; return(true); } //+------------------------------------------------------------------+ //| Restore every request field and reject noncanonical evidence | //+------------------------------------------------------------------+ bool DPCCReadTerminalRequest(const SDPCCExecutionIntent &intent, MqlTradeRequest &request, long &login, string &server, string &terminal_path, long "e_time_msc, long &dispatch_time_msc, uint &request_id, string &reason) { string f[]; if(StringLen(intent.terminal_binding_wire) > 8192 || StringSplit(intent.terminal_binding_wire, '^', f) != 28 || f[0] != "DPCC_TERMINAL_REQUEST_V1" || f[1] != intent.intent_sha256 || f[2] != intent.checkpoint_sha256 || f[27] != "END_TERMINAL_REQUEST") {reason = "terminal.binding.schema_or_intent"; return(false);} ulong values[12]; const int indices[] = {3, 6, 7, 8, 10, 11, 12, 19, 20, 21, 22, 23}; for(int i = 0; i < 12; i++) if(!DPCCCheckpointReadUnsigned(f[indices[i]], ULONG_MAX, values[i])) {reason = "terminal.binding.integer"; return(false);} if(values[0] == 0 || values[0] > (ulong)LONG_MAX || values[1] == 0 || values[2] < values[1] || values[2] > (ulong)LONG_MAX || values[3] > UINT_MAX || values[11] > (ulong)LONG_MAX) {reason = "terminal.binding.range"; return(false);} MqlTradeRequest decoded; ZeroMemory(decoded); decoded.action = (ENUM_TRADE_REQUEST_ACTIONS)values[4]; decoded.magic = values[5]; decoded.order = values[6]; decoded.symbol = f[13]; double numbers[5]; for(int i = 0; i < 5; i++) if(!DPCCPhysicalReadDouble(f[14 + i], numbers[i])) {reason = "terminal.binding.double"; return(false);} decoded.volume = numbers[0]; decoded.price = numbers[1]; decoded.stoplimit = numbers[2]; decoded.sl = numbers[3]; decoded.tp = numbers[4]; decoded.deviation = values[7]; decoded.type = (ENUM_ORDER_TYPE)values[8]; decoded.type_filling = (ENUM_ORDER_TYPE_FILLING)values[9]; decoded.type_time = (ENUM_ORDER_TYPE_TIME)values[10]; decoded.expiration = (datetime)values[11]; decoded.comment = f[24]; if(!DPCCCheckpointReadUnsigned(f[25], ULONG_MAX, decoded.position) || !DPCCCheckpointReadUnsigned(f[26], ULONG_MAX, decoded.position_by)) return(false); string request_wire = "", sha = ""; if(f[4] == "" || f[5] == "" || !DPCCExecutionTerminalText(f[4]) || !DPCCExecutionTerminalText(f[5]) || !DPCCExecutionTerminalRequestWire(intent, decoded, request_wire, reason) || !DPCCCheckpointTextSHA(request_wire, sha, reason) || sha != f[9]) return(false); string canonical = "DPCC_TERMINAL_REQUEST_V1^" + intent.intent_sha256 + "^" + intent.checkpoint_sha256 + "^"; canonical += (string)values[0] + "^" + f[4] + "^" + f[5] + "^" + (string)values[1] + "^" + (string)values[2] + "^"; canonical += (string)values[3] + "^" + sha + "^" + request_wire + "^END_TERMINAL_REQUEST"; if(canonical != intent.terminal_binding_wire) {reason = "terminal.binding.noncanonical"; return(false);} request = decoded; login = (long)values[0]; server = f[4]; terminal_path = f[5]; quote_time_msc = (long)values[1]; dispatch_time_msc = (long)values[2]; request_id = (uint)values[3]; return(true); } //+------------------------------------------------------------------+ //| A live read must match account, broker and terminal installation | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalLiveBinding(const SDPCCExecutionIntent &intent, MqlTradeRequest &request, string &reason) { long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint id = 0; if(!DPCCReadTerminalRequest(intent, request, login, server, path, quote, dispatch, id, reason)) return(false); if(login != AccountInfoInteger(ACCOUNT_LOGIN) || server != AccountInfoString(ACCOUNT_SERVER) || path != TerminalInfoString(TERMINAL_DATA_PATH)) {reason = "terminal.binding.foreign_terminal_account"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Seal a terminal observation using the existing immutable reducer | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalSeal(const SDPCCExecutionIntent &intent, SDPCCExecutionReceipt &receipt, string &reason) { receipt.intent_id = intent.intent_id; receipt.intent_sha256 = intent.intent_sha256; receipt.deployment_id = intent.deployment_id; receipt.magic = intent.magic; receipt.symbol = intent.symbol; receipt.canonical_digest = ""; string key = "", digest = ""; if(!DPCCExecutionReceiptCanonical(receipt, key, digest, reason)) return(false); receipt.canonical_digest = digest; return(true); } //+------------------------------------------------------------------+ //| Preserve every MqlTradeResult field with exact floating bits | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalResultWire(const MqlTradeResult &result, string &wire, string &reason) { const double values[] = {result.volume, result.price, result.bid, result.ask}; for(int i = 0; i < 4; i++) if(!MathIsValidNumber(values[i]) || values[i] < 0) {reason = "terminal.result.numeric"; return(false);} if(!DPCCExecutionTerminalText(result.comment)) {reason = "terminal.result.comment"; return(false);} wire = (string)result.retcode + "^" + (string)result.deal + "^" + (string)result.order + "^"; for(int i = 0; i < 4; i++) wire += DPCCPhysicalDoubleWire(values[i]) + "^"; wire += result.comment + "^" + (string)result.request_id + "^" + (string)result.retcode_external; return(true); } //+------------------------------------------------------------------+ //| Function DPCCReadTerminalResult | //+------------------------------------------------------------------+ bool DPCCReadTerminalResult(const SDPCCExecutionIntent &intent, MqlTradeResult &result, string &reason) { string fields[]; if(StringSplit(intent.terminal_result_wire, '^', fields) != 10) {reason = "terminal.result.schema"; return(false);} MqlTradeResult candidate; ZeroMemory(candidate); ulong retcode = 0, id = 0; if(!DPCCCheckpointReadUnsigned(fields[0], UINT_MAX, retcode) || !DPCCCheckpointReadUnsigned(fields[1], ULONG_MAX, candidate.deal) || !DPCCCheckpointReadUnsigned(fields[2], ULONG_MAX, candidate.order) || !DPCCCheckpointReadUnsigned(fields[8], UINT_MAX, id)) return(false); candidate.retcode = (uint)retcode; candidate.request_id = (uint)id; candidate.retcode_external = (int)StringToInteger(fields[9]); if((string)candidate.retcode_external != fields[9] || !DPCCPhysicalReadDouble(fields[3], candidate.volume) || !DPCCPhysicalReadDouble(fields[4], candidate.price) || !DPCCPhysicalReadDouble(fields[5], candidate.bid) || !DPCCPhysicalReadDouble(fields[6], candidate.ask)) return(false); candidate.comment = fields[7]; string canonical = ""; if(!DPCCExecutionTerminalResultWire(candidate, canonical, reason) || canonical != intent.terminal_result_wire) {reason = "terminal.result.noncanonical"; return(false);} MqlTradeRequest request; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint request_id = 0; if(!DPCCReadTerminalRequest(intent, request, login, server, path, quote, dispatch, request_id, reason) || request_id != candidate.request_id || (intent.order_ticket != 0 && candidate.order != intent.order_ticket) || candidate.volume > request.volume + DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume)) {reason = "terminal.result.binding"; return(false);} result = candidate; return(true); } //+------------------------------------------------------------------+ //| Result acknowledgement binds a request ID; history proves fills | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalResult(SDPCCExecutionIntent &intent, const MqlTradeRequest &request, const MqlTradeResult &result, const long observed_time_msc, SDPCCExecutionReceipt &receipt, string &reason) { MqlTradeRequest stored; long login = 0, quote = 0, dispatch = 0; string server = "", path = "", wire = "", stored_wire = ""; uint id = 0; if(!DPCCReadTerminalRequest(intent, stored, login, server, path, quote, dispatch, id, reason) || !DPCCExecutionTerminalRequestWire(intent, request, wire, reason) || !DPCCExecutionTerminalRequestWire(intent, stored, stored_wire, reason) || wire != stored_wire || observed_time_msc < dispatch || (id != 0 && id != result.request_id) || (intent.order_ticket != 0 && result.order != intent.order_ticket)) {if(reason == "")reason = "terminal.result.request_identity"; return(false);} SDPCCExecutionReceipt candidate; ZeroMemory(candidate); candidate.kind = DPCC_EXECUTION_ORDER_STATE; candidate.order_ticket = result.order; candidate.time_msc = observed_time_msc; const bool accepted = result.retcode == TRADE_RETCODE_PLACED || result.retcode == TRADE_RETCODE_DONE || result.retcode == TRADE_RETCODE_DONE_PARTIAL; candidate.status = (accepted && result.order != 0 && result.request_id != 0 ? DPCC_ORDER_PENDING : DPCC_ORDER_AMBIGUOUS); if(candidate.status == DPCC_ORDER_PENDING && intent.receipt_count > 0) { //--- A late REQUEST callback cannot regress a deal-confirmed partial or complete state. for(uint i = 0; i < intent.receipt_count && i < DPCC_EXECUTION_RECEIPT_CAPACITY; i++) { string parts[]; if(StringSplit(intent.receipt_keys[i], ':', parts) != 3 || parts[0] != "O" || parts[1] != (string)result.order) continue; ulong time = 0; if(!DPCCCheckpointReadUnsigned(parts[2], (ulong)LONG_MAX, time)) return(false); candidate.time_msc = (long)time; if(!DPCCExecutionTerminalSeal(intent, candidate, reason)) return(false); if(candidate.canonical_digest == intent.receipt_digests[i]) break; candidate.time_msc = observed_time_msc; } } if(!DPCCExecutionTerminalSeal(intent, candidate, reason)) return(false); string fields[]; if(StringSplit(intent.terminal_binding_wire, '^', fields) != 28) return(false); fields[8] = (string)result.request_id; string staged = ""; for(int i = 0; i < 28; i++) staged += (i > 0 ? "^" : "") + fields[i]; SDPCCExecutionIntent validated = intent; validated.terminal_binding_wire = staged; string result_wire = ""; if(!DPCCExecutionTerminalResultWire(result, result_wire, reason) || result.volume > stored.volume + DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, stored.volume)) return(false); validated.terminal_result_wire = result_wire; if(!DPCCApplyExecutionReceipt(validated, candidate, reason)) return(false); intent.terminal_binding_wire = staged; intent.terminal_result_wire = result_wire; receipt = candidate; return(true); } //+------------------------------------------------------------------+ //| Deal identity and side, including closing and reversing trades | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalDeal(const SDPCCExecutionIntent &intent, const ulong order, const ulong deal, const long magic, const string symbol, const ENUM_DEAL_TYPE type, const ENUM_DEAL_ENTRY entry, const long time_msc, const double volume, const double price, SDPCCExecutionReceipt &receipt, string &reason) { MqlTradeRequest request; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint request_id = 0; if(!DPCCReadTerminalRequest(intent, request, login, server, path, quote, dispatch, request_id, reason) || order == 0 || order != intent.order_ticket || deal == 0 || magic != intent.magic || symbol != intent.symbol || time_msc < dispatch || (type != DEAL_TYPE_BUY && type != DEAL_TYPE_SELL) || (entry != DEAL_ENTRY_IN && entry != DEAL_ENTRY_OUT && entry != DEAL_ENTRY_INOUT) || !MathIsValidNumber(volume) || volume <= 0 || !MathIsValidNumber(price) || price <= 0 || (type == DEAL_TYPE_BUY) != (intent.requested_signed_delta_lots > 0)) {if(reason == "")reason = "terminal.deal.identity_or_direction"; return(false);} SDPCCExecutionReceipt candidate; ZeroMemory(candidate); candidate.kind = DPCC_EXECUTION_DEAL_FILL; candidate.order_ticket = order; candidate.deal_id = deal; candidate.time_msc = time_msc; candidate.signed_fill_lots = (type == DEAL_TYPE_BUY ? volume : -volume); candidate.fill_price = price; const string key = "D:" + (string)deal; //--- A duplicate keeps its original status even after subsequent partial fills. for(uint i = 0; i < intent.receipt_count && i < DPCC_EXECUTION_RECEIPT_CAPACITY; i++) if(intent.receipt_keys[i] == key) { for(int status = DPCC_ORDER_PARTIAL; status <= DPCC_ORDER_FILLED; status++) { candidate.status = (ENUM_DPCC_EXECUTION_ORDER_STATE)status; if(!DPCCExecutionTerminalSeal(intent, candidate, reason)) return(false); if(candidate.canonical_digest == intent.receipt_digests[i]) {receipt = candidate; return(true);} } reason = "terminal.deal.conflicting_duplicate"; return(false); } const double cumulative = intent.filled_signed_lots + candidate.signed_fill_lots; const double tolerance = DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume); if(MathAbs(cumulative) > request.volume + tolerance) {reason = "terminal.deal.overfill"; return(false);} candidate.status = (MathAbs(MathAbs(cumulative) - request.volume) <= tolerance ? DPCC_ORDER_FILLED : DPCC_ORDER_PARTIAL); if(!DPCCExecutionTerminalSeal(intent, candidate, reason)) return(false); receipt = candidate; return(true); } //+------------------------------------------------------------------+ //| Check native order identity before reducing its terminal state | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalOrder(const SDPCCExecutionIntent &intent, const ulong ticket, const long magic, const string symbol, const string comment, const ENUM_ORDER_TYPE type, const double initial_volume, const ENUM_ORDER_STATE state, const long time_msc, SDPCCExecutionReceipt &receipt, string &reason) { MqlTradeRequest request; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint id = 0; if(!DPCCReadTerminalRequest(intent, request, login, server, path, quote, dispatch, id, reason) || ticket == 0 || ticket != intent.order_ticket || magic != intent.magic || symbol != intent.symbol || comment != request.comment || type != request.type || time_msc < dispatch || !MathIsValidNumber(initial_volume) || MathAbs(initial_volume - request.volume) > DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume)) {if(reason == "")reason = "terminal.order.identity"; return(false);} SDPCCExecutionReceipt candidate; ZeroMemory(candidate); candidate.order_ticket = ticket; candidate.kind = DPCC_EXECUTION_ORDER_STATE; candidate.time_msc = time_msc; if(state == ORDER_STATE_CANCELED || state == ORDER_STATE_EXPIRED) candidate.status = DPCC_ORDER_CANCELED; else if(state == ORDER_STATE_REJECTED) candidate.status = DPCC_ORDER_REJECTED; else if(state == ORDER_STATE_FILLED) candidate.status = DPCC_ORDER_AMBIGUOUS; else if(state == ORDER_STATE_STARTED || state == ORDER_STATE_PLACED || state == ORDER_STATE_PARTIAL || state == ORDER_STATE_REQUEST_ADD || state == ORDER_STATE_REQUEST_MODIFY || state == ORDER_STATE_REQUEST_CANCEL) candidate.status = DPCC_ORDER_PENDING; else {reason = "terminal.order.unknown_state"; return(false);} if(!DPCCExecutionTerminalSeal(intent, candidate, reason)) return(false); receipt = candidate; return(true); } //+------------------------------------------------------------------+ //| Read one deal using native history properties, without sending | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalHistoryDeal(const SDPCCExecutionIntent &intent, const ulong deal, SDPCCExecutionReceipt &receipt, string &reason) { MqlTradeRequest request; if(!DPCCExecutionTerminalLiveBinding(intent, request, reason)) return(false); ResetLastError(); if(!HistoryDealSelect(deal)) {reason = "terminal.history.deal_select." + (string)GetLastError(); return(false);} long order = 0, magic = 0, type = 0, entry = 0, time = 0; string symbol = "", comment = ""; double volume = 0, price = 0; if(!HistoryDealGetInteger(deal, DEAL_ORDER, order) || !HistoryDealGetInteger(deal, DEAL_MAGIC, magic) || !HistoryDealGetInteger(deal, DEAL_TYPE, type) || !HistoryDealGetInteger(deal, DEAL_ENTRY, entry) || !HistoryDealGetInteger(deal, DEAL_TIME_MSC, time) || !HistoryDealGetString(deal, DEAL_SYMBOL, symbol) || !HistoryDealGetString(deal, DEAL_COMMENT, comment) || !HistoryDealGetDouble(deal, DEAL_VOLUME, volume) || !HistoryDealGetDouble(deal, DEAL_PRICE, price)) {reason = "terminal.history.deal_properties." + (string)GetLastError(); return(false);} if(comment != request.comment) {if(reason == "")reason = "terminal.history.deal_comment"; return(false);} return(DPCCExecutionTerminalDeal(intent, (ulong)order, deal, magic, symbol, (ENUM_DEAL_TYPE)type, (ENUM_DEAL_ENTRY)entry, time, volume, price, receipt, reason)); } //+------------------------------------------------------------------+ //| Bind a durable deal-only result after exact history evidence | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalResolveDealOrder(const SDPCCExecutionIntent &intent, const bool available, const ulong deal, const ulong order, const long magic, const string symbol, const string comment, const ENUM_DEAL_TYPE type, const ENUM_DEAL_ENTRY entry, const long time_msc, const double volume, const double price, SDPCCExecutionIntent &resolved, string &reason) { reason = ""; if(!available) {reason = "terminal.history.result_deal_unavailable"; return(false);} MqlTradeRequest request; MqlTradeResult result; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint request_id = 0; if(!DPCCReadTerminalRequest(intent, request, login, server, path, quote, dispatch, request_id, reason) || !DPCCReadTerminalResult(intent, result, reason) || result.deal == 0 || result.deal != deal || order == 0 || (result.order != 0 && result.order != order) || (intent.order_ticket != 0 && intent.order_ticket != order) || magic != intent.magic || symbol != intent.symbol || comment != request.comment || type != (request.type == ORDER_TYPE_BUY ? DEAL_TYPE_BUY : DEAL_TYPE_SELL) || (entry != DEAL_ENTRY_IN && entry != DEAL_ENTRY_OUT && entry != DEAL_ENTRY_INOUT) || time_msc < dispatch || !MathIsValidNumber(volume) || volume <= 0 || !MathIsValidNumber(price) || price <= 0 || volume > request.volume + DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume) || volume > result.volume + DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, request.volume)) {if(reason == "")reason = "terminal.history.result_deal_identity"; return(false);} resolved = intent; resolved.order_ticket = order; return(true); } //+------------------------------------------------------------------+ //| Stage all history fills before a canceled or finished order | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalReconcile(const SDPCCExecutionIntent &intent, SDPCCExecutionReceipt &receipts[], string &reason) { reason = ""; MqlTradeRequest request; if(!DPCCExecutionTerminalLiveBinding(intent, request, reason)) return(false); SDPCCExecutionIntent staged = intent; if(staged.order_ticket == 0) { MqlTradeResult result; if(!DPCCReadTerminalResult(staged, result, reason) || result.deal == 0) {if(reason == "")reason = "terminal.history.no_durable_order_or_deal"; return(false);} ResetLastError(); if(!HistoryDealSelect(result.deal)) {reason = "terminal.history.result_deal_unavailable." + (string)GetLastError(); return(false);} long deal_order = 0, magic = 0, type = 0, entry = 0, time = 0; string symbol = "", comment = ""; double volume = 0, price = 0; if(!HistoryDealGetInteger(result.deal, DEAL_ORDER, deal_order) || !HistoryDealGetInteger(result.deal, DEAL_MAGIC, magic) || !HistoryDealGetInteger(result.deal, DEAL_TYPE, type) || !HistoryDealGetInteger(result.deal, DEAL_ENTRY, entry) || !HistoryDealGetInteger(result.deal, DEAL_TIME_MSC, time) || !HistoryDealGetString(result.deal, DEAL_SYMBOL, symbol) || !HistoryDealGetString(result.deal, DEAL_COMMENT, comment) || !HistoryDealGetDouble(result.deal, DEAL_VOLUME, volume) || !HistoryDealGetDouble(result.deal, DEAL_PRICE, price)) {reason = "terminal.history.result_deal_properties." + (string)GetLastError(); return(false);} if(!DPCCExecutionTerminalResolveDealOrder(staged, true, result.deal, (ulong)deal_order, magic, symbol, comment, (ENUM_DEAL_TYPE)type, (ENUM_DEAL_ENTRY)entry, time, volume, price, staged, reason)) return(false); } const ulong order = staged.order_ticket; ResetLastError(); const bool history = HistoryOrderSelect(order); if(!history && !OrderSelect(order)) {reason = "terminal.history.order_unavailable." + (string)GetLastError(); return(false);} long magic = 0, type = 0, state = 0, time = 0; string symbol = "", comment = ""; double volume = 0; bool properties = false; if(history) properties = HistoryOrderGetInteger(order, ORDER_MAGIC, magic) && HistoryOrderGetInteger(order, ORDER_TYPE, type) && HistoryOrderGetInteger(order, ORDER_STATE, state) && HistoryOrderGetInteger(order, ORDER_TIME_DONE_MSC, time) && HistoryOrderGetString(order, ORDER_SYMBOL, symbol) && HistoryOrderGetString(order, ORDER_COMMENT, comment) && HistoryOrderGetDouble(order, ORDER_VOLUME_INITIAL, volume); else properties = OrderGetInteger(ORDER_MAGIC, magic) && OrderGetInteger(ORDER_TYPE, type) && OrderGetInteger(ORDER_STATE, state) && OrderGetInteger(ORDER_TIME_SETUP_MSC, time) && OrderGetString(ORDER_SYMBOL, symbol) && OrderGetString(ORDER_COMMENT, comment) && OrderGetDouble(ORDER_VOLUME_INITIAL, volume); if(!properties) {reason = "terminal.history.order_properties." + (string)GetLastError(); return(false);} SDPCCExecutionReceipt order_receipt; if(!DPCCExecutionTerminalOrder(intent, order, magic, symbol, comment, (ENUM_ORDER_TYPE)type, volume, (ENUM_ORDER_STATE)state, time, order_receipt, reason)) return(false); MqlTradeRequest restored; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint id = 0; if(!DPCCReadTerminalRequest(intent, restored, login, server, path, quote, dispatch, id, reason)) return(false); if(!HistorySelect((datetime)(quote / 1000), TimeCurrent())) {reason = "terminal.history.window." + (string)GetLastError(); return(false);} ulong tickets[]; long times[]; for(int i = 0; i < HistoryDealsTotal(); i++) { const ulong deal = HistoryDealGetTicket(i); long deal_order = 0, deal_time = 0; if(deal == 0 || !HistoryDealGetInteger(deal, DEAL_ORDER, deal_order)) {reason = "terminal.history.deal_enumeration"; return(false);} if((ulong)deal_order != order) continue; if(!HistoryDealGetInteger(deal, DEAL_TIME_MSC, deal_time)) return(false); const int size = ArraySize(tickets); if(size >= DPCC_EXECUTION_RECEIPT_CAPACITY || ArrayResize(tickets, size + 1) != size + 1 || ArrayResize(times, size + 1) != size + 1) {reason = "terminal.history.deal_capacity"; return(false);} int at = size; //--- Deterministic local replay order; this is not a broker FIFO priority rule. while(at > 0 && (times[at - 1] > deal_time || (times[at - 1] == deal_time && tickets[at - 1] > deal))) {times[at] = times[at - 1]; tickets[at] = tickets[at - 1]; at--;} times[at] = deal_time; tickets[at] = deal; } SDPCCExecutionReceipt batch[]; for(int i = 0; i < ArraySize(tickets); i++) { SDPCCExecutionReceipt fill; if(!DPCCExecutionTerminalHistoryDeal(staged, tickets[i], fill, reason)) return(false); const uint before = staged.receipt_count; if(!DPCCApplyExecutionReceipt(staged, fill, reason)) return(false); if(staged.receipt_count == before) continue; const int n = ArraySize(batch); if(ArrayResize(batch, n + 1) != n + 1) return(false); batch[n] = fill; } if(staged.order_state != DPCC_ORDER_FILLED && (order_receipt.status != DPCC_ORDER_PENDING || staged.order_state == DPCC_ORDER_PENDING)) { if((ENUM_ORDER_STATE)state == ORDER_STATE_FILLED) {reason = "terminal.history.incomplete_deal_visibility"; return(false);} if(order_receipt.time_msc > staged.last_order_state_time_msc) { if(!DPCCApplyExecutionReceipt(staged, order_receipt, reason)) return(false); const int n = ArraySize(batch); if(ArrayResize(batch, n + 1) != n + 1) return(false); batch[n] = order_receipt; } else if(order_receipt.status != DPCC_ORDER_PENDING && staged.order_state != order_receipt.status) {reason = "terminal.history.order_time_conflict"; return(false);} } if(ArrayResize(receipts, ArraySize(batch)) != ArraySize(batch)) return(false); for(int i = 0; i < ArraySize(batch); i++) receipts[i] = batch[i]; return(true); } //+------------------------------------------------------------------+ //| Only REQUEST carries request/result; other events use history | //+------------------------------------------------------------------+ bool DPCCExecutionTerminalTransaction(SDPCCExecutionIntent &intent, const MqlTradeTransaction &transaction, const MqlTradeRequest &request, const MqlTradeResult &result, const long observed_time_msc, SDPCCExecutionReceipt &receipts[], string &reason) { MqlTradeRequest stored; if(!DPCCExecutionTerminalLiveBinding(intent, stored, reason)) return(false); if(transaction.type == TRADE_TRANSACTION_REQUEST) { SDPCCExecutionIntent staged = intent; SDPCCExecutionReceipt receipt; if(!DPCCExecutionTerminalResult(staged, request, result, observed_time_msc, receipt, reason)) return(false); if(ArrayResize(receipts, 1) != 1) return(false); receipts[0] = receipt; intent.terminal_binding_wire = staged.terminal_binding_wire; intent.terminal_result_wire = staged.terminal_result_wire; return(true); } if(transaction.order == 0 || transaction.symbol != intent.symbol || (intent.order_ticket != 0 && transaction.order != intent.order_ticket)) {reason = "terminal.transaction.foreign_or_unbound_order"; return(false);} if(transaction.type == TRADE_TRANSACTION_DEAL_ADD) { if(intent.order_ticket == 0) { MqlTradeResult exact_result; if(!DPCCReadTerminalResult(intent, exact_result, reason) || exact_result.deal == 0 || exact_result.deal != transaction.deal) {if(reason == "")reason = "terminal.transaction.unbound_result_deal"; return(false);} ResetLastError(); if(!HistoryDealSelect(transaction.deal)) {reason = "terminal.transaction.result_deal_unavailable." + (string)GetLastError(); return(false);} long history_order = 0; if(!HistoryDealGetInteger(transaction.deal, DEAL_ORDER, history_order) || (ulong)history_order != transaction.order) {reason = "terminal.transaction.result_deal_order_mismatch"; return(false);} return(DPCCExecutionTerminalReconcile(intent, receipts, reason)); } SDPCCExecutionReceipt receipt; if(!DPCCExecutionTerminalHistoryDeal(intent, transaction.deal, receipt, reason) || transaction.deal_type != (receipt.signed_fill_lots > 0 ? DEAL_TYPE_BUY : DEAL_TYPE_SELL) || transaction.volume != MathAbs(receipt.signed_fill_lots) || transaction.price != receipt.fill_price) {if(reason == "")reason = "terminal.transaction.deal_history_mismatch"; return(false);} if(ArrayResize(receipts, 1) != 1) return(false); receipts[0] = receipt; return(true); } if(transaction.type == TRADE_TRANSACTION_ORDER_ADD || transaction.type == TRADE_TRANSACTION_ORDER_UPDATE || transaction.type == TRADE_TRANSACTION_ORDER_DELETE || transaction.type == TRADE_TRANSACTION_HISTORY_ADD || transaction.type == TRADE_TRANSACTION_HISTORY_UPDATE) return(DPCCExecutionTerminalReconcile(intent, receipts, reason)); reason = "terminal.transaction.unsupported_or_mutated_history"; return(false); } //+------------------------------------------------------------------+ //| DPCCCaptureExecutionProfile binds native symbol rules to history | //+------------------------------------------------------------------+ bool DPCCCaptureExecutionProfile(const string symbol, const ENUM_TIMEFRAMES timeframe, const string raw_history_sha256, const uint history_rows, SDPCCExecutionProfileSnapshot &snapshot, string &payload, string &digest, string &reason, const bool current_terminal_snapshot = false) { payload = ""; digest = ""; reason = ""; snapshot.raw_history_sha256 = ""; snapshot.history_symbol = ""; snapshot.terminal_company = ""; snapshot.terminal_name = ""; snapshot.terminal_server = ""; snapshot.account_currency = ""; snapshot.account_currency_digits = 0; snapshot.bond_face_value = 0; snapshot.bond_accrued_interest = 0; snapshot.session_price_settlement = 0; snapshot.snapshot_time_server = 0; ArrayResize(snapshot.commission_rules, 0); if(symbol == "" || timeframe <= PERIOD_CURRENT || (!current_terminal_snapshot && (history_rows == 0 || !DPCCArchitectureSHA256Encoding(raw_history_sha256))) || (current_terminal_snapshot && (history_rows != 0 || raw_history_sha256 != ""))) { reason = (current_terminal_snapshot ? "execution_profile.current_snapshot_scope" : "execution_profile.source_binding"); return(false); } snapshot.raw_history_sha256 = (current_terminal_snapshot ? "" : raw_history_sha256); snapshot.history_symbol = symbol; snapshot.history_timeframe = timeframe; snapshot.history_rows = history_rows; snapshot.terminal_company = TerminalInfoString(TERMINAL_COMPANY); snapshot.terminal_name = TerminalInfoString(TERMINAL_NAME); snapshot.terminal_server = AccountInfoString(ACCOUNT_SERVER); snapshot.account_currency = AccountInfoString(ACCOUNT_CURRENCY); snapshot.account_currency_digits = AccountInfoInteger(ACCOUNT_CURRENCY_DIGITS); snapshot.terminal_build = TerminalInfoInteger(TERMINAL_BUILD); //--- Keep a terminal/server timestamp for provenance; it is not an admission control. snapshot.snapshot_time_server = TimeCurrent(); if(snapshot.terminal_server == "" || snapshot.account_currency == "" || snapshot.account_currency_digits < 0 || snapshot.account_currency_digits > 8) { reason = "execution_profile.terminal_context"; return(false); } //--- A live chart snapshot has no fabricated raw-history identity. if(!DPCCExecutionAppend(payload, "schema", "DPCC_EXECUTION_PROFILE_V1") || !DPCCExecutionAppend(payload, "digest_algorithm", "SHA256") || !DPCCExecutionAppend(payload, "payload_encoding", "CP_UTF8") || !DPCCExecutionAppend(payload, "field_format", "key|mql_string_length|value|LF") || !DPCCExecutionAppend(payload, "snapshot_scope", current_terminal_snapshot ? "CURRENT_TERMINAL_PROPERTIES_V1" : "HISTORICAL_SOURCE_BOUND_V1")) { reason = "execution_profile.payload_capacity"; return(false); } if((current_terminal_snapshot && (!DPCCExecutionAppend(payload, "snapshot_symbol", symbol) || !DPCCExecutionAppend(payload, "snapshot_timeframe", EnumToString(timeframe)))) || (!current_terminal_snapshot && (!DPCCExecutionAppend(payload, "raw_history_sha256", raw_history_sha256) || !DPCCExecutionAppend(payload, "history_symbol", symbol) || !DPCCExecutionAppend(payload, "history_timeframe", EnumToString(timeframe)) || !DPCCExecutionAppend(payload, "history_rows", (string)history_rows))) || !DPCCExecutionAppend(payload, "terminal_company", TerminalInfoString(TERMINAL_COMPANY)) || !DPCCExecutionAppend(payload, "terminal_name", TerminalInfoString(TERMINAL_NAME)) || !DPCCExecutionAppend(payload, "terminal_server", snapshot.terminal_server) || !DPCCExecutionAppend(payload, "account_currency", snapshot.account_currency) || !DPCCExecutionAppend(payload, "account_currency_digits", (string)snapshot.account_currency_digits) || !DPCCExecutionAppend(payload, "terminal_build", (string)TerminalInfoInteger(TERMINAL_BUILD)) || !DPCCExecutionAppend(payload, "snapshot_time_server", TimeToString(snapshot.snapshot_time_server, TIME_DATE | TIME_SECONDS)) || !DPCCExecutionAppend(payload, "slippage_observation", "UNKNOWN_UNOBSERVED") || !DPCCExecutionAppend(payload, "slippage_model_policy", "ZERO_ASSUMED_BY_USER_INITIAL_STAGE")) { reason = "execution_profile.payload_capacity"; return(false); } //--- Capture stable execution and contract properties without reading a quote. ENUM_SYMBOL_INFO_INTEGER integer_ids[] = {SYMBOL_DIGITS, SYMBOL_SPREAD_FLOAT, SYMBOL_TRADE_MODE, SYMBOL_TRADE_EXEMODE, SYMBOL_TRADE_CALC_MODE, SYMBOL_ORDER_MODE, SYMBOL_FILLING_MODE, SYMBOL_EXPIRATION_MODE, SYMBOL_TRADE_STOPS_LEVEL, SYMBOL_TRADE_FREEZE_LEVEL, SYMBOL_SWAP_MODE, SYMBOL_SWAP_ROLLOVER3DAYS, SYMBOL_MARGIN_HEDGED_USE_LEG }; string integer_names[] = {"symbol_digits", "spread_is_floating", "trade_mode", "trade_execution_mode", "calculation_mode", "order_mode", "filling_mode", "expiration_mode", "stops_level_points", "freeze_level_points", "swap_mode", "swap_rollover_3days", "margin_hedged_use_leg" }; for(int i = 0; i < ArraySize(integer_ids); i++) { long value = 0; ResetLastError(); if(!SymbolInfoInteger(symbol, integer_ids[i], value)) { reason = StringFormat("execution_profile.integer.%s.error_%d", integer_names[i], GetLastError()); return(false); } if(!DPCCExecutionAppend(payload, integer_names[i], (string)value)) { reason = "execution_profile.payload_capacity"; return(false); } switch(i) { case 0: snapshot.symbol_digits = value; break; case 1: snapshot.spread_is_floating = value; break; case 2: snapshot.trade_mode = value; break; case 3: snapshot.trade_execution_mode = value; break; case 4: snapshot.calculation_mode = value; break; case 5: snapshot.order_mode = value; break; case 6: snapshot.filling_mode = value; break; case 7: snapshot.expiration_mode = value; break; case 8: snapshot.stops_level_points = value; break; case 9: snapshot.freeze_level_points = value; break; case 10: snapshot.swap_mode = value; break; case 11: snapshot.swap_rollover_3days = value; break; case 12: snapshot.margin_hedged_use_leg = value; break; } } ENUM_SYMBOL_INFO_DOUBLE double_ids[] = {SYMBOL_POINT, SYMBOL_TRADE_TICK_SIZE, SYMBOL_TRADE_TICK_VALUE, SYMBOL_TRADE_TICK_VALUE_PROFIT, SYMBOL_TRADE_TICK_VALUE_LOSS, SYMBOL_TRADE_CONTRACT_SIZE, SYMBOL_VOLUME_MIN, SYMBOL_VOLUME_MAX, SYMBOL_VOLUME_STEP, SYMBOL_VOLUME_LIMIT, SYMBOL_SWAP_LONG, SYMBOL_SWAP_SHORT, SYMBOL_MARGIN_INITIAL, SYMBOL_MARGIN_MAINTENANCE, SYMBOL_MARGIN_HEDGED }; string double_names[] = {"point_size", "trade_tick_size", "trade_tick_value", "trade_tick_value_profit", "trade_tick_value_loss", "contract_size", "volume_min", "volume_max", "volume_step", "volume_limit", "swap_long", "swap_short", "margin_initial", "margin_maintenance", "margin_hedged" }; for(int i = 0; i < ArraySize(double_ids); i++) { double value = 0; ResetLastError(); if(!SymbolInfoDouble(symbol, double_ids[i], value) || !MathIsValidNumber(value)) { reason = StringFormat("execution_profile.double.%s.error_%d", double_names[i], GetLastError()); return(false); } if(!DPCCExecutionAppend(payload, double_names[i], StringFormat("%.17g", value))) { reason = "execution_profile.payload_capacity"; return(false); } switch(i) { case 0: snapshot.point_size = value; break; case 1: snapshot.trade_tick_size = value; break; case 2: snapshot.trade_tick_value = value; break; case 3: snapshot.trade_tick_value_profit = value; break; case 4: snapshot.trade_tick_value_loss = value; break; case 5: snapshot.contract_size = value; break; case 6: snapshot.volume_min = value; break; case 7: snapshot.volume_max = value; break; case 8: snapshot.volume_step = value; break; case 9: snapshot.volume_limit = value; break; case 10: snapshot.swap_long = value; break; case 11: snapshot.swap_short = value; break; case 12: snapshot.margin_initial = value; break; case 13: snapshot.margin_maintenance = value; break; case 14: snapshot.margin_hedged = value; break; } } if(snapshot.calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS || snapshot.calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS_MOEX) { if(!SymbolInfoDouble(symbol, SYMBOL_TRADE_FACE_VALUE, snapshot.bond_face_value) || !SymbolInfoDouble(symbol, SYMBOL_TRADE_ACCRUED_INTEREST, snapshot.bond_accrued_interest) || !MathIsValidNumber(snapshot.bond_face_value) || snapshot.bond_face_value <= 0 || !MathIsValidNumber(snapshot.bond_accrued_interest)) {reason = "execution_profile.bond_properties"; return(false);} if(!DPCCExecutionAppend(payload, "bond_face_value", StringFormat("%.17g", snapshot.bond_face_value)) || !DPCCExecutionAppend(payload, "bond_accrued_interest", StringFormat("%.17g", snapshot.bond_accrued_interest)) || !DPCCExecutionAppend(payload, "bond_quote_price_factor", "0.01") || !DPCCExecutionAppend(payload, "bond_interest_policy", "CURRENT_SNAPSHOT_CONSTANT_OVER_HISTORY")) {reason = "execution_profile.bond_payload_capacity"; return(false);} } if(snapshot.calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES_FORTS) { if(!SymbolInfoDouble(symbol, SYMBOL_SESSION_PRICE_SETTLEMENT, snapshot.session_price_settlement) || !MathIsValidNumber(snapshot.session_price_settlement) || snapshot.session_price_settlement <= 0) {reason = "execution_profile.forts_settlement_property"; return(false);} if(!DPCCExecutionAppend(payload, "session_price_settlement", StringFormat("%.17g", snapshot.session_price_settlement)) || !DPCCExecutionAppend(payload, "financial_snapshot_policy", "CURRENT_TERMINAL_SNAPSHOT_REPEATED_FOR_HISTORICAL_BARS") || !DPCCExecutionAppend(payload, "forts_settlement_policy", "CURRENT_SESSION_PROPERTY_MODELED_AS_PREVIOUS_SESSION_CLEARING_OVER_HISTORY")) {reason = "execution_profile.forts_payload_capacity"; return(false);} } ENUM_SYMBOL_INFO_STRING string_ids[] = {SYMBOL_CURRENCY_BASE, SYMBOL_CURRENCY_PROFIT, SYMBOL_CURRENCY_MARGIN, SYMBOL_DESCRIPTION, SYMBOL_PATH }; string string_names[] = {"currency_base", "currency_profit", "currency_margin", "description", "path"}; for(int i = 0; i < ArraySize(string_ids); i++) { string value = ""; ResetLastError(); if(!SymbolInfoString(symbol, string_ids[i], value)) { reason = StringFormat("execution_profile.string.%s.error_%d", string_names[i], GetLastError()); return(false); } if(!DPCCExecutionAppend(payload, string_names[i], value)) { reason = "execution_profile.payload_capacity"; return(false); } switch(i) { case 0: snapshot.currency_base = value; break; case 1: snapshot.currency_profit = value; break; case 2: snapshot.currency_margin = value; break; case 3: snapshot.description = value; break; case 4: snapshot.path = value; break; } } //--- Preserve every native commission rule and tier, including a successful empty result. MqlCommission commissions[]; ResetLastError(); const int commission_count = SymbolInfoCommissions(symbol, commissions); if(commission_count < 0 || commission_count != ArraySize(commissions) || commission_count > 256) { reason = StringFormat("execution_profile.commissions.error_%d", GetLastError()); return(false); } if(ArrayResize(snapshot.commission_rules, commission_count) != commission_count) { reason = "execution_profile.commissions.copy_allocation"; return(false); } const string commission_state = (commission_count == 0 ? "OBSERVED_EMPTY" : "OBSERVED_RULES"); if(!DPCCExecutionAppend(payload, "commission_observation", commission_state) || !DPCCExecutionAppend(payload, "commission_rule_count", (string)commission_count)) { reason = "execution_profile.payload_capacity"; return(false); } for(int r = 0; r < commission_count; r++) { const string prefix = StringFormat("commission_rule_%03d_", r); if(!DPCCExecutionAppend(payload, prefix + "currency", commissions[r].currency) || !DPCCExecutionAppend(payload, prefix + "mode_range", (string)commissions[r].mode_range) || !DPCCExecutionAppend(payload, prefix + "mode_charge", (string)commissions[r].mode_charge) || !DPCCExecutionAppend(payload, prefix + "mode_entry", (string)commissions[r].mode_entry) || !DPCCExecutionAppend(payload, prefix + "mode_direction", (string)commissions[r].mode_direction) || !DPCCExecutionAppend(payload, prefix + "mode_profit", (string)commissions[r].mode_profit)) { reason = "execution_profile.payload_capacity"; return(false); } const int tier_count = ArraySize(commissions[r].tiers); if(tier_count > 256 || !DPCCExecutionAppend(payload, prefix + "tier_count", (string)tier_count)) { reason = "execution_profile.commissions.tier_count"; return(false); } snapshot.commission_rules[r].currency = commissions[r].currency; snapshot.commission_rules[r].mode_range = commissions[r].mode_range; snapshot.commission_rules[r].mode_charge = commissions[r].mode_charge; snapshot.commission_rules[r].mode_entry = commissions[r].mode_entry; snapshot.commission_rules[r].mode_direction = commissions[r].mode_direction; snapshot.commission_rules[r].mode_profit = commissions[r].mode_profit; if(ArrayResize(snapshot.commission_rules[r].tiers, tier_count) != tier_count) { reason = "execution_profile.commissions.tier_copy_allocation"; return(false); } for(int t = 0; t < tier_count; t++) { const string tier = prefix + StringFormat("tier_%03d_", t); MqlCommissionTier item = commissions[r].tiers[t]; snapshot.commission_rules[r].tiers[t] = item; if(!MathIsValidNumber(item.value) || !MathIsValidNumber(item.min_value) || !MathIsValidNumber(item.max_value) || !MathIsValidNumber(item.range_from) || !MathIsValidNumber(item.range_to) || !DPCCExecutionAppend(payload, tier + "mode", (string)item.mode) || !DPCCExecutionAppend(payload, tier + "volume_type", (string)item.volume_type) || !DPCCExecutionAppend(payload, tier + "value", StringFormat("%.17g", item.value)) || !DPCCExecutionAppend(payload, tier + "min_value", StringFormat("%.17g", item.min_value)) || !DPCCExecutionAppend(payload, tier + "max_value", StringFormat("%.17g", item.max_value)) || !DPCCExecutionAppend(payload, tier + "range_from", StringFormat("%.17g", item.range_from)) || !DPCCExecutionAppend(payload, tier + "range_to", StringFormat("%.17g", item.range_to)) || !DPCCExecutionAppend(payload, tier + "currency", item.currency)) { reason = "execution_profile.commissions.tier"; return(false); } } } //--- Hash the complete ordered receipt payload using shared UTF-8 SHA-256. if(!DPCCCheckpointTextSHA(payload, digest, reason)) { reason = "execution_profile.digest"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Admit only native profiles whose money fields are account | //| currency | //+------------------------------------------------------------------+ bool DPCCSupportedHistoricalNativeCurrencies(const ENUM_SYMBOL_CALC_MODE calculation_mode, const string base_currency, const string margin_currency, const string profit_currency, const string account_currency) { if(account_currency == "" || profit_currency != account_currency) return(false); if(calculation_mode == SYMBOL_CALC_MODE_FOREX || calculation_mode == SYMBOL_CALC_MODE_FOREX_NO_LEVERAGE) return(StringLen(base_currency) > 0 && margin_currency == base_currency); if(calculation_mode == SYMBOL_CALC_MODE_CFD || calculation_mode == SYMBOL_CALC_MODE_CFDLEVERAGE || calculation_mode == SYMBOL_CALC_MODE_CFDINDEX || calculation_mode == SYMBOL_CALC_MODE_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES_FORTS || calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS || calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS_MOEX) return(margin_currency == account_currency); return(false); } //+------------------------------------------------------------------+ //| Shared admission gate for native same-currency historical modes | //+------------------------------------------------------------------+ bool DPCCSupportedHistoricalNativeExecution(const SDPCCExecutionProfileSnapshot &execution, const uint margin_profile = DPCC_MARGIN_EXTERNAL_V2) { if(!DPCCSupportedHistoricalCalculationMode((ENUM_SYMBOL_CALC_MODE)execution.calculation_mode)) return(false); if(margin_profile == DPCC_MARGIN_HISTORICAL_CLOSE_V1) return(execution.currency_base != "" && execution.currency_margin != "" && execution.currency_profit != "" && execution.account_currency != ""); return(DPCCSupportedHistoricalNativeCurrencies((ENUM_SYMBOL_CALC_MODE)execution.calculation_mode, execution.currency_base, execution.currency_margin, execution.currency_profit, execution.account_currency)); } //+------------------------------------------------------------------+ //| Resolve one direct or inverse terminal currency instrument | //+------------------------------------------------------------------+ bool DPCCResolveHistoricalFxSymbol(const string source_currency, const string account_currency, string &symbol, bool &inverse, double &point, string &reason) { symbol = ""; inverse = false; point = 0; if(source_currency == "" || account_currency == "") {reason = "BLOCKED:history.fx.currency_missing"; return(false);} if(source_currency == account_currency) return(true); //--- Symbol names and Market Watch order do not decide the conversion. const int count = SymbolsTotal(false); if(count <= 0) {reason = "BLOCKED:history.fx.symbol_inventory"; return(false);} for(int i = 0; i < count; i++) { const string candidate = SymbolName(i, false); string base = "", profit = ""; long selected = 0, chart_mode = 0, calculation_mode = -1; double candidate_point = 0; if(candidate == "" || !SymbolInfoInteger(candidate, SYMBOL_SELECT, selected)) {reason = "BLOCKED:history.fx.symbol_selection_metadata"; return(false);} const bool temporary = (selected == 0); if(temporary && !SymbolSelect(candidate, true)) {reason = "BLOCKED:history.fx.symbol_select." + candidate; return(false);} const bool currencies = (SymbolInfoString(candidate, SYMBOL_CURRENCY_BASE, base) && SymbolInfoString(candidate, SYMBOL_CURRENCY_PROFIT, profit)); const bool direct = (currencies && base == source_currency && profit == account_currency); const bool reverse = (currencies && base == account_currency && profit == source_currency); bool calculation = true, quote = true; if(direct || reverse) calculation = SymbolInfoInteger(candidate, SYMBOL_TRADE_CALC_MODE, calculation_mode); const bool forex = (calculation_mode == SYMBOL_CALC_MODE_FOREX || calculation_mode == SYMBOL_CALC_MODE_FOREX_NO_LEVERAGE); if((direct || reverse) && calculation && forex) quote = (SymbolInfoDouble(candidate, SYMBOL_POINT, candidate_point) && SymbolInfoInteger(candidate, SYMBOL_CHART_MODE, chart_mode)); const bool restored = (!temporary || SymbolSelect(candidate, false)); if(!restored) {reason = "BLOCKED:history.fx.symbol_selection_restore." + candidate; return(false);} if(!currencies) {reason = "BLOCKED:history.fx.symbol_currency_metadata." + candidate; return(false);} if(!direct && !reverse) continue; if(!calculation) {reason = "BLOCKED:history.fx.calculation_mode_metadata." + candidate; return(false);} if(!forex) continue; if(!quote || !MathIsValidNumber(candidate_point) || candidate_point <= 0 || chart_mode != SYMBOL_CHART_MODE_BID) {reason = "BLOCKED:history.fx.bid_history_metadata." + candidate; return(false);} if(symbol != "") { reason = "BLOCKED:history.fx.ambiguous_pair." + source_currency + "." + account_currency; return(false); } symbol = candidate; inverse = reverse; point = candidate_point; } if(symbol == "") { reason = "BLOCKED:history.fx.pair_missing." + source_currency + "." + account_currency; return(false); } return(true); } //+------------------------------------------------------------------+ //| Read exact previous source H1 closes without current quote | //| fallback | //+------------------------------------------------------------------+ bool DPCCReadHistoricalFxCloses(const string symbol, const bool inverse, const double point, const SDPCCStudyForecastBar &bars[], const ENUM_TIMEFRAMES timeframe, float &gain[], float &loss[], string &reason) { ArrayResize(gain, 0); ArrayResize(loss, 0); const int count = ArraySize(bars); if(count < 2 || ArrayResize(gain, count) != count || ArrayResize(loss, count) != count) {reason = "BLOCKED:history.fx.history_capacity"; return(false);} ArrayInitialize(gain, 0.0f); ArrayInitialize(loss, 0.0f); const long period_seconds = PeriodSeconds(timeframe); if(period_seconds <= 0) {reason = "BLOCKED:history.fx.timeframe"; return(false);} if(symbol == "") { //--- Equal currencies are a true identity, never a missing-pair fallback. for(int i = 1; i < count; i++) { gain[i] = 1.0f; loss[i] = 1.0f; } return(true); } //--- Copy one terminal series, then join every required source timestamp. MqlRates rates[]; ArraySetAsSeries(rates, false); long selected = 0; if(!SymbolInfoInteger(symbol, SYMBOL_SELECT, selected)) {reason = "BLOCKED:history.fx.history_selection_metadata." + symbol; return(false);} const bool temporary = (selected == 0); if(temporary && !SymbolSelect(symbol, true)) {reason = "BLOCKED:history.fx.history_select." + symbol; return(false);} ResetLastError(); const int copied = CopyRates(symbol, timeframe, (datetime)bars[0].server_open, (datetime)bars[count - 2].server_open, rates); const int copy_error = GetLastError(); if(temporary && !SymbolSelect(symbol, false)) {reason = "BLOCKED:history.fx.history_selection_restore." + symbol; return(false);} if(copied <= 0 || copied != ArraySize(rates)) { reason = StringFormat("BLOCKED:history.fx.history_unavailable.%s.error_%d", symbol, copy_error); return(false); } for(int i = 0; i < copied; i++) if(rates[i].time <= 0 || (i > 0 && rates[i].time <= rates[i - 1].time)) {reason = "BLOCKED:history.fx.history_order." + symbol; return(false);} for(int i = 1; i < count; i++) { const long required = bars[i - 1].server_open; if(required <= 0 || required > LONG_MAX - period_seconds || required + period_seconds > bars[i].server_open) {reason = "BLOCKED:history.fx.source_bar_join"; return(false);} int lo = 0, hi = copied - 1, found = -1; while(lo <= hi) { const int middle = lo + (hi - lo) / 2; if((long)rates[middle].time == required) {found = middle; break;} if((long)rates[middle].time < required) lo = middle + 1; else hi = middle - 1; } if(found < 0) { reason = StringFormat("BLOCKED:history.fx.exact_previous_bar_missing.%s.%I64d", symbol, required); return(false); } const double bid = rates[found].close; const double spread = (double)rates[found].spread * point; const double ask = bid + spread; if(rates[found].spread < 0 || !MathIsValidNumber(bid) || bid <= 0 || !MathIsValidNumber(spread) || spread < 0 || !MathIsValidNumber(ask) || ask < bid) {reason = "BLOCKED:history.fx.previous_close_numeric." + symbol; return(false);} const double positive = (inverse ? 1.0 / ask : bid); const double negative = (inverse ? 1.0 / bid : ask); gain[i] = (float)positive; loss[i] = (float)negative; if(!MathIsValidNumber((double)gain[i]) || gain[i] <= 0 || !MathIsValidNumber((double)loss[i]) || loss[i] < gain[i]) {reason = "BLOCKED:history.fx.rate_float_domain." + symbol; return(false);} } return(true); } //+------------------------------------------------------------------+ //| Bind the frozen per-step conversion table to the financial | //| receipt | //+------------------------------------------------------------------+ bool DPCCPrepareHistoricalFxCloses(const SDPCCConfig &config, const SDPCCExecutionProfileSnapshot &execution, const SDPCCStudyForecastBar &bars[], const ENUM_TIMEFRAMES timeframe, SDPCCHistoricalFinancialProfile &profile, string &reason) { if(config.margin_fx_profile != DPCC_MARGIN_HISTORICAL_CLOSE_V1) return(true); if(!DPCCResolveHistoricalFxSymbol(execution.currency_profit, execution.account_currency, profile.fx_profit_symbol, profile.fx_profit_inverse, profile.fx_profit_point, reason)) return(false); if(!DPCCResolveHistoricalFxSymbol(execution.currency_margin, execution.account_currency, profile.fx_margin_symbol, profile.fx_margin_inverse, profile.fx_margin_point, reason)) return(false); float gain[], loss[], margin_gain[], margin_loss[]; if(!DPCCReadHistoricalFxCloses(profile.fx_profit_symbol, profile.fx_profit_inverse, profile.fx_profit_point, bars, timeframe, gain, loss, reason)) return(false); if(profile.fx_profit_symbol == profile.fx_margin_symbol && profile.fx_profit_inverse == profile.fx_margin_inverse) { if(ArrayCopy(margin_gain, gain) != ArraySize(gain) || ArrayCopy(margin_loss, loss) != ArraySize(loss)) {reason = "BLOCKED:history.fx.shared_rate_capacity"; return(false);} } else if(!DPCCReadHistoricalFxCloses(profile.fx_margin_symbol, profile.fx_margin_inverse, profile.fx_margin_point, bars, timeframe, margin_gain, margin_loss, reason)) return(false); const int count = ArraySize(bars); if(count > INT_MAX / 4 || ArrayResize(profile.fx_source_open, count) != count || ArrayResize(profile.fx_frozen, count * 4) != count * 4) {reason = "BLOCKED:history.fx.frozen_table_capacity"; return(false);} ArrayInitialize(profile.fx_source_open, 0); ArrayInitialize(profile.fx_frozen, 0.0f); string wire = "DPCC_HISTORICAL_PREVIOUS_SOURCE_BAR_CLOSE_FX_V1\n"; for(int i = 1; i < count; i++) { profile.fx_source_open[i] = bars[i - 1].server_open; profile.fx_frozen[i * 4] = gain[i]; profile.fx_frozen[i * 4 + 1] = loss[i]; //--- Direct Ask / inverse 1/Bid for BUY; direct Bid / inverse 1/Ask for SELL. profile.fx_frozen[i * 4 + 2] = margin_loss[i]; profile.fx_frozen[i * 4 + 3] = margin_gain[i]; wire += StringFormat("%I64d|%I64d|%.9g|%.9g|%.9g|%.9g\n", bars[i].server_open, profile.fx_source_open[i], gain[i], loss[i], margin_loss[i], margin_gain[i]); } string rates_sha = ""; if(!DPCCCheckpointTextSHA(wire, rates_sha, reason)) return(false); if(!DPCCExecutionAppend(profile.payload, "fx_policy", "PREVIOUS_ALIGNED_BAR_CLOSE_FROZEN_FOR_NEXT_STEP_V1") || !DPCCExecutionAppend(profile.payload, "fx_spread_policy", "REPORTED_BAR_SPREAD_AT_CLOSE") || !DPCCExecutionAppend(profile.payload, "fx_profit_symbol", profile.fx_profit_symbol) || !DPCCExecutionAppend(profile.payload, "fx_profit_inverse", (string)(int)profile.fx_profit_inverse) || !DPCCExecutionAppend(profile.payload, "fx_profit_point", StringFormat("%.17g", profile.fx_profit_point)) || !DPCCExecutionAppend(profile.payload, "fx_margin_symbol", profile.fx_margin_symbol) || !DPCCExecutionAppend(profile.payload, "fx_margin_inverse", (string)(int)profile.fx_margin_inverse) || !DPCCExecutionAppend(profile.payload, "fx_margin_point", StringFormat("%.17g", profile.fx_margin_point)) || !DPCCExecutionAppend(profile.payload, "fx_frozen_rows", (string)(count - 1)) || !DPCCExecutionAppend(profile.payload, "fx_frozen_sha256", rates_sha)) {reason = "BLOCKED:history.fx.receipt_capacity"; return(false);} return(DPCCCheckpointTextSHA(profile.payload, profile.sha256, reason)); } //+------------------------------------------------------------------+ //| Capture only formulas supported by the native external contract | //+------------------------------------------------------------------+ bool DPCCPrepareHistoricalFinancialProfile(const SDPCCConfig &config, const float &limits[], const SDPCCExecutionProfileSnapshot &execution, const string execution_payload, const bool server_midnight, const bool historical_clock_policy, SDPCCHistoricalFinancialProfile &out, string &reason, const bool server_calendar = false, const bool current_chart_snapshot = false) { ZeroMemory(out); reason = ""; out.account_currency = ""; out.account_currency_digits = 0; if(config.shape.E < DPCC_HISTORY_CALENDAR_MIN_EVENTS) {reason = "BLOCKED:history.calendar.E_below_global_bound"; return(false);} const ENUM_SYMBOL_CALC_MODE calculation_mode = (ENUM_SYMBOL_CALC_MODE)execution.calculation_mode; const bool spot_fx = (calculation_mode == SYMBOL_CALC_MODE_FOREX || calculation_mode == SYMBOL_CALC_MODE_FOREX_NO_LEVERAGE); const bool bond = (calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS || calculation_mode == SYMBOL_CALC_MODE_EXCH_BONDS_MOEX); const bool forts = (calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES_FORTS); const bool exchange_stock = (calculation_mode == SYMBOL_CALC_MODE_EXCH_STOCKS || calculation_mode == SYMBOL_CALC_MODE_EXCH_STOCKS_MOEX); const bool same_currency_native = (calculation_mode == SYMBOL_CALC_MODE_CFD || calculation_mode == SYMBOL_CALC_MODE_CFDLEVERAGE || calculation_mode == SYMBOL_CALC_MODE_CFDINDEX || calculation_mode == SYMBOL_CALC_MODE_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES || forts || (current_chart_snapshot && exchange_stock)); const bool frozen_fx = (config.margin_fx_profile == DPCC_MARGIN_HISTORICAL_CLOSE_V1); const uint expected_margin_profile = (frozen_fx ? DPCC_MARGIN_HISTORICAL_CLOSE_V1 : (spot_fx ? DPCC_MARGIN_FOREX_SPOT_V1 : DPCC_MARGIN_EXTERNAL_V2)); const bool current_native_currency = (execution.account_currency != "" && execution.currency_profit == execution.account_currency && (spot_fx ? (execution.currency_base != "" && execution.currency_margin == execution.currency_base) : execution.currency_margin == execution.account_currency)); const bool native_currency_supported = (current_chart_snapshot ? current_native_currency : DPCCSupportedHistoricalNativeExecution(execution, config.margin_fx_profile)); const bool quote_contract_supported = (current_chart_snapshot ? (config.price_basis <= 1 && config.quote_adapter == config.price_basis) : (config.price_basis == DPCC_PRICE_BID && config.quote_adapter == DPCC_QUOTE_BID_IDENTITY_V1)); if(ArraySize(limits) != 24 || !DPCCSupportedHistoricalCalculationMode(calculation_mode) || !native_currency_supported || (!spot_fx && !same_currency_native && !bond) || config.margin_fx_profile != expected_margin_profile || !quote_contract_supported || ((spot_fx || frozen_fx) && config.account_currency_digits != (uint)execution.account_currency_digits) || (!spot_fx && !frozen_fx && config.account_currency_digits != 0)) {reason = "BLOCKED:history.profile.native_currency_or_model_contract"; return(false);} //--- A rule table cannot be forced into the native fixed per-lot fee fields. if(ArraySize(execution.commission_rules) != 0) {reason = "BLOCKED:history.profile.commission_rules_require_exact_native_mapping"; return(false);} if(limits[13] != 0) {reason = "BLOCKED:history.profile.model_extra_slippage_must_be_zero"; return(false);} long chart_mode = 0; const long expected_chart_mode = (config.price_basis == 0 ? SYMBOL_CHART_MODE_BID : SYMBOL_CHART_MODE_LAST); if(!SymbolInfoInteger(execution.history_symbol, SYMBOL_CHART_MODE, chart_mode) || chart_mode != (current_chart_snapshot ? expected_chart_mode : SYMBOL_CHART_MODE_BID)) {reason = "BLOCKED:history.profile.bid_chart"; return(false);} const long leverage = AccountInfoInteger(ACCOUNT_LEVERAGE); const long margin_mode = AccountInfoInteger(ACCOUNT_MARGIN_MODE); double buy_initial = 0, buy_maintenance = 0, sell_initial = 0, sell_maintenance = 0; if(leverage <= 0 || margin_mode < 0 || margin_mode > 2 || !SymbolInfoMarginRate(execution.history_symbol, ORDER_TYPE_BUY, buy_initial, buy_maintenance) || !SymbolInfoMarginRate(execution.history_symbol, ORDER_TYPE_SELL, sell_initial, sell_maintenance)) {reason = "BLOCKED:history.profile.account_margin_rates"; return(false);} if(!MathIsValidNumber(buy_initial) || !MathIsValidNumber(buy_maintenance) || !MathIsValidNumber(sell_initial) || !MathIsValidNumber(sell_maintenance) || execution.margin_initial < 0 || execution.margin_maintenance < 0) {reason = "BLOCKED:history.profile.margin_input_domain"; return(false);} if(spot_fx && !frozen_fx && (execution.margin_initial != 0 || execution.margin_maintenance != 0)) {reason = "BLOCKED:history.profile.fixed_margin_override"; return(false);} if((calculation_mode == SYMBOL_CALC_MODE_CFDINDEX || calculation_mode == SYMBOL_CALC_MODE_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES || forts) && (execution.trade_tick_value_profit <= 0 || execution.trade_tick_value_loss <= 0)) {reason = "BLOCKED:history.profile.tick_value_required"; return(false);} if((calculation_mode == SYMBOL_CALC_MODE_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES || forts) && execution.margin_initial <= 0) {reason = "BLOCKED:history.profile.initial_margin_required"; return(false);} const int slots[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 18, 19, 20, 21, 29, 30, 31, 32, 34, 35, 38, 41, 42}; const double values[] = {execution.point_size, execution.trade_tick_size, execution.contract_size, execution.volume_min, execution.volume_max, execution.volume_step, MaxTP, MaxSL, (double)execution.stops_level_points, (double)execution.freeze_level_points, (double)execution.symbol_digits, (double)execution.calculation_mode, 1, 1, buy_initial, sell_initial, (double)leverage, execution.trade_tick_value_profit, execution.trade_tick_value_loss, (double)execution.trade_mode, (frozen_fx ? 4.0 : (spot_fx ? 3.0 : 2.0)), (current_chart_snapshot ? (double)config.price_basis : 0.0), (current_chart_snapshot ? (double)config.quote_adapter : 0.0), (double)execution.order_mode, (double)margin_mode, execution.volume_limit, buy_maintenance, sell_maintenance }; for(int i = 0; i < ArraySize(slots); i++) out.instrument[slots[i]] = (float)values[i]; if(bond) { out.instrument[22] = (float)execution.bond_face_value; out.instrument[23] = (float)execution.bond_accrued_interest; out.instrument[44] = 0.01f; if(!MathIsValidNumber((double)out.instrument[22]) || out.instrument[22] <= 0.0f || !MathIsValidNumber((double)out.instrument[23])) {reason = "BLOCKED:history.profile.bond_properties"; return(false);} } if(forts) { out.instrument[40] = (float)execution.session_price_settlement; if(!MathIsValidNumber((double)out.instrument[40]) || out.instrument[40] <= 0.0f) {reason = "BLOCKED:history.profile.forts_settlement_float_domain"; return(false);} } if(frozen_fx && (calculation_mode == SYMBOL_CALC_MODE_CFDINDEX || calculation_mode == SYMBOL_CALC_MODE_FUTURES || calculation_mode == SYMBOL_CALC_MODE_EXCH_FUTURES || forts) && execution.currency_profit != execution.account_currency) {reason = "BLOCKED:history.profile.account_tick_value_requires_native_tick_price"; return(false);} if(frozen_fx && execution.margin_initial == 0) { //--- A floating formula has a native denomination before its FX boundary. string native_margin_currency = ""; if(spot_fx) native_margin_currency = execution.currency_base; else if(calculation_mode == SYMBOL_CALC_MODE_CFD || calculation_mode == SYMBOL_CALC_MODE_CFDLEVERAGE || bond) native_margin_currency = execution.currency_profit; else if(calculation_mode == SYMBOL_CALC_MODE_CFDINDEX) native_margin_currency = execution.account_currency; if(native_margin_currency == "" || execution.currency_margin != native_margin_currency) {reason = "BLOCKED:history.profile.margin_formula_currency_mismatch"; return(false);} } if(!spot_fx || frozen_fx) { //--- Tick values supplied by the terminal are already in account currency. out.instrument[15] = (float)execution.margin_initial; out.instrument[16] = (float)execution.margin_maintenance; out.instrument[43] = (spot_fx ? 0.0f : 1.0f); out.instrument[46] = (execution.margin_initial > 0 ? 1.0f : 0.0f); out.instrument[47] = (execution.margin_maintenance > 0 ? 1.0f : 0.0f); } for(int i = 0; i < 24; i++) out.limits[i] = limits[i]; out.limits[15] = (float)MathMin((double)limits[15], execution.trade_tick_size * 0.25); //--- Currency conversion for event packing, not a second account simulator. if(execution.swap_mode == SYMBOL_SWAP_MODE_DISABLED) { out.swap_cash_buy = 0; out.swap_cash_sell = 0; } else if(execution.swap_mode == SYMBOL_SWAP_MODE_POINTS) { out.swap_cash_buy = execution.swap_long * execution.point_size * execution.contract_size; out.swap_cash_sell = execution.swap_short * execution.point_size * execution.contract_size; if(bond) { out.swap_cash_buy *= execution.bond_face_value * 0.01; out.swap_cash_sell *= execution.bond_face_value * 0.01; } out.swap_profit_currency = frozen_fx; } else if(execution.swap_mode == SYMBOL_SWAP_MODE_CURRENCY_DEPOSIT) { out.swap_cash_buy = execution.swap_long; out.swap_cash_sell = execution.swap_short; } else if(execution.swap_mode == SYMBOL_SWAP_MODE_CURRENCY_PROFIT) { out.swap_cash_buy = execution.swap_long; out.swap_cash_sell = execution.swap_short; out.swap_profit_currency = (execution.currency_profit != execution.account_currency); } else {reason = "BLOCKED:history.profile.swap_mode_requires_conversion_or_reopen"; return(false);} const ENUM_SYMBOL_INFO_DOUBLE swap_ids[] = {SYMBOL_SWAP_SUNDAY, SYMBOL_SWAP_MONDAY, SYMBOL_SWAP_TUESDAY, SYMBOL_SWAP_WEDNESDAY, SYMBOL_SWAP_THURSDAY, SYMBOL_SWAP_FRIDAY, SYMBOL_SWAP_SATURDAY }; for(int i = 0; i < 7; i++) if(!SymbolInfoDouble(execution.history_symbol, swap_ids[i], out.swap_factors[i]) || !MathIsValidNumber(out.swap_factors[i]) || out.swap_factors[i] < 0) {reason = StringFormat("BLOCKED:history.profile.swap_factor.%d", i); return(false);} for(int i = 0; i < 48; i++) if(!MathIsValidNumber((double)out.instrument[i])) {reason = StringFormat("BLOCKED:history.profile.instrument.%d", i); return(false);} if(!MathIsValidNumber(out.swap_cash_buy) || !MathIsValidNumber(out.swap_cash_sell)) {reason = "BLOCKED:history.profile.swap_cash_nonfinite"; return(false);} out.server_midnight = server_midnight; out.account_currency_digits = execution.account_currency_digits; out.account_currency = execution.account_currency; out.time_domain = (server_calendar ? "SERVER_CALENDAR_SECONDS_V1" : "UTC_SECONDS_V1"); out.payload = execution_payload; DPCCExecutionAppend(out.payload, "native_account_currency", out.account_currency); DPCCExecutionAppend(out.payload, "native_margin_contract", (frozen_fx ? "HISTORICAL_CLOSE_FX_V1" : (spot_fx ? "FOREX_SPOT_V1" : "EXTERNAL_SAME_CURRENCY_V2"))); DPCCExecutionAppend(out.payload, "context_recipe", "DPCC_H1_STREAM_NATIVE_V1"); DPCCExecutionAppend(out.payload, "price_tolerance_policy", "MIN_MODEL_TOLERANCE_AND_QUARTER_TRADE_TICK_SIZE_V1"); DPCCExecutionAppend(out.payload, "model_price_tolerance_upper_bound", StringFormat("%.9g", (double)limits[15])); DPCCExecutionAppend(out.payload, "cost_asof_policy", "CURRENT_TERMINAL_SNAPSHOT_MODELED_NOT_HISTORICAL_TARIFF"); DPCCExecutionAppend(out.payload, "spread_policy", "RAW_BAR_SPREAD_AT_ALL_FOUR_VERTICES"); DPCCExecutionAppend(out.payload, "clock_policy", server_calendar ? "HISTORICAL_SERVER_CALENDAR_V1" : (historical_clock_policy ? "HISTORICAL_POLICY_V1" : "ACCOUNT_OBSERVED_V1")); if(server_calendar) DPCCExecutionAppend(out.payload, "time_domain", out.time_domain); DPCCExecutionAppend(out.payload, "rollover_policy", server_midnight ? "SERVER_MIDNIGHT_DERIVED" : "UNCONFIRMED"); DPCCExecutionAppend(out.payload, "swap_rollover_policy", "SERVER_MIDNIGHT_MINUS_15S_ASSUMED"); DPCCExecutionAppend(out.payload, "account_leverage", (string)leverage); DPCCExecutionAppend(out.payload, "account_margin_mode", (string)margin_mode); for(int i = 0; i < 48; i++) DPCCExecutionAppend(out.payload, StringFormat("native_instrument_%02d", i), StringFormat("%.9g", out.instrument[i])); for(int i = 0; i < 24; i++) DPCCExecutionAppend(out.payload, StringFormat("model_limit_%02d", i), StringFormat("%.9g", out.limits[i])); for(int i = 0; i < 7; i++) DPCCExecutionAppend(out.payload, StringFormat("swap_factor_%d", i), StringFormat("%.17g", out.swap_factors[i])); DPCCExecutionAppend(out.payload, "swap_cash_buy", StringFormat("%.17g", out.swap_cash_buy)); DPCCExecutionAppend(out.payload, "swap_cash_sell", StringFormat("%.17g", out.swap_cash_sell)); return(DPCCCheckpointTextSHA(out.payload, out.sha256, reason)); } //+------------------------------------------------------------------+ //| Pack one factual H1 horizon; the native helper owns transitions | //+------------------------------------------------------------------+ bool DPCCBuildRawFinancialContext(const SDPCCConfig &config, const SDPCCHistoricalFinancialProfile &profile, const SDPCCStudyForecastBar &bars[], const int raw_index, const long &timeline_opens[], const long &timeline_closes[], const float &counts[], const float &calendar[], const long prior_time_close, float &context[], float &native_ohlc[], string &reason) { ArrayResize(context, 0); ArrayResize(native_ohlc, 0); SDPCCLayout layout, unused; uint tokens = 0; const int horizon = (int)config.shape.H; if(!profile.server_midnight) {reason = "BLOCKED:history.calendar.rollover_hour_unconfirmed"; return(false);} if(!DPCCCompositeLayouts(config.shape, layout, unused, tokens) || horizon < 1 || raw_index < 1 || raw_index > ArraySize(bars) - horizon || ArraySize(timeline_opens) != horizon || ArraySize(timeline_closes) != horizon || ArraySize(counts) != horizon || ArraySize(calendar) != horizon * (int)config.shape.E * 6 || layout.offsets[10] > INT_MAX || prior_time_close > timeline_opens[0] || prior_time_close < 0) {reason = "BLOCKED:history.stream.shape"; return(false);} float candidate[], ohlc[]; if(ArrayResize(candidate, (int)layout.offsets[10]) != (int)layout.offsets[10] || ArrayResize(ohlc, horizon * 4) != horizon * 4) {reason = "BLOCKED:history.stream.capacity"; return(false);} ArrayInitialize(candidate, 0.0f); for(int i = 0; i < 48; i++) candidate[(int)layout.offsets[5] + i] = profile.instrument[i]; for(int i = 0; i < 24; i++) candidate[(int)layout.offsets[6] + i] = profile.limits[i]; for(int h = 0; h < horizon; h++) { const int raw = raw_index + h; if(config.margin_fx_profile == DPCC_MARGIN_HISTORICAL_CLOSE_V1 && (raw < 1 || raw >= ArraySize(bars))) {reason = "BLOCKED:history.fx.previous_source_bar_required"; return(false);} const double spread = (double)bars[raw].spread * (double)profile.instrument[0]; if(bars[raw].spread < 0 || !MathIsValidNumber(spread) || !MathIsValidNumber((double)(float)spread) || timeline_closes[h] <= timeline_opens[h] || (h > 0 && timeline_opens[h] < timeline_closes[h - 1]) || counts[h] < 0 || counts[h] > (float)config.shape.E || MathFloor(counts[h]) != counts[h]) {reason = "BLOCKED:history.stream.spread_or_time_or_event_count"; return(false);} for(int c = 0; c < 4; c++) ohlc[h * 4 + c] = (float)bars[raw].ohlc[c]; candidate[(int)layout.offsets[8] + h] = counts[h]; for(uint k = 0; k < config.shape.K; k++) { const int offset = (int)layout.offsets[7] + ((int)k * horizon + h) * 24; for(int c = 0; c < 4; c++) candidate[offset + c] = (float)spread; candidate[offset + 13] = 1; candidate[offset + 14] = 1; candidate[offset + 15] = 1; candidate[offset + 16] = profile.instrument[23]; candidate[offset + 17] = profile.instrument[23]; if(config.margin_fx_profile == DPCC_MARGIN_HISTORICAL_CLOSE_V1) { if(ArraySize(profile.fx_source_open) != ArraySize(bars) || ArraySize(profile.fx_frozen) != (long)ArraySize(bars) * 4 || profile.fx_source_open[raw] != bars[raw - 1].server_open) {reason = "BLOCKED:history.fx.frozen_source_join"; return(false);} for(int c = 0; c < 4; c++) if(!MathIsValidNumber((double)profile.fx_frozen[raw * 4 + c]) || profile.fx_frozen[raw * 4 + c] <= 0) {reason = "BLOCKED:history.fx.frozen_numeric"; return(false);} candidate[offset + 13] = profile.fx_frozen[raw * 4]; candidate[offset + 14] = profile.fx_frozen[raw * 4 + 1]; candidate[offset + 15] = profile.fx_frozen[raw * 4 + 2]; candidate[offset + 23] = profile.fx_frozen[raw * 4 + 3]; } candidate[offset + 18] = (float)(timeline_closes[h] - timeline_opens[h]); candidate[offset + 19] = (float)(timeline_opens[h] - (h == 0 ? prior_time_close : timeline_closes[h - 1])); candidate[offset + 21] = counts[h]; for(int c = 0; c < (int)config.shape.E * 6; c++) candidate[(int)layout.offsets[9] + ((int)k * horizon + h) * (int)config.shape.E * 6 + c] = calendar[h * (int)config.shape.E * 6 + c]; } } const int account = (int)layout.offsets[4]; candidate[account + 15] = ohlc[0]; candidate[account + 16] = ohlc[0] + (float)((double)bars[raw_index].spread * profile.instrument[0]); candidate[(int)layout.offsets[5] + 14] = candidate[account + 16]; if(ArrayCopy(context, candidate) != ArraySize(candidate) || ArrayCopy(native_ohlc, ohlc) != ArraySize(ohlc)) {reason = "BLOCKED:history.stream.publish_capacity"; return(false);} reason = ""; return(true); } //+------------------------------------------------------------------+ //| Pack a causal Market decision into the published financial | //| layout | //+------------------------------------------------------------------+ bool DPCCBuildDecisionFinancialContext(const SDPCCConfig &config, CBufferFloat *native_ohlc, const float &account24[], const float &instrument48[], const float &limits24[], const float &execution_snapshot24[], const long decision_server_time, const long previous_close_server, const long &open_server[], const long &close_server[], const float &event_counts[], const float &calendar[], CBufferFloat &snapshot_work, CBufferFloat &context_work, CBufferFloat &flags_work, CBufferFloat &status_work, CBufferFloat *&context_out, string &reason) { context_out = NULL; reason = ""; //--- The owner allocates and retains these buffers for the admitted profile. SDPCCLayout layout; if(!DPCCValidateFixedConfig(config.shape, config) || !DPCCBuildRolloutLayout(config.shape, layout) || ArraySize(account24) != 24 || ArraySize(instrument48) != 48 || ArraySize(limits24) != 24 || ArraySize(execution_snapshot24) != 24 || ArraySize(open_server) != (int)config.shape.H || ArraySize(close_server) != (int)config.shape.H || ArraySize(event_counts) != (int)config.shape.H || ArraySize(calendar) != (int)layout.lengths[6] / (int)config.shape.K || decision_server_time <= 0 || previous_close_server < 0 || previous_close_server > decision_server_time || open_server[0] != decision_server_time) { reason = "decision_context.profile_or_shape"; return(false); } COpenCLMy *context = CheckPointer(native_ohlc) == POINTER_INVALID ? NULL : native_ohlc.GetOpenCL(); CBufferFloat *buffers[] = {native_ohlc, GetPointer(snapshot_work), GetPointer(context_work), GetPointer(flags_work), GetPointer(status_work) }; const ulong prices = layout.lengths[0]; const ulong tail = layout.offsets[7] - prices; const ulong counts[] = {prices, tail, layout.total, layout.total, 1}; for(int i = 0; i < ArraySize(buffers); i++) { if(CheckPointer(context) == POINTER_INVALID || CheckPointer(buffers[i]) == POINTER_INVALID || !NNQValid(context, buffers[i], counts[i]) || buffers[i].Total() != (int)counts[i] || context.GetBufferSize(buffers[i].GetIndex()) != counts[i] * sizeof(float)) { reason = "decision_context.workspace"; return(false); } for(int j = 0; j < i; j++) if(buffers[i] == buffers[j] || buffers[i].GetIndex() == buffers[j].GetIndex()) { reason = "decision_context.workspace_alias"; return(false); } } //--- Hold the current spread, costs, conversion and interest snapshot across the horizon. for(int i = 0; i < 24; i++) if(!MathIsValidNumber(account24[i]) || !MathIsValidNumber(limits24[i]) || !MathIsValidNumber(execution_snapshot24[i])) { reason = "decision_context.snapshot_nonfinite"; return(false); } for(int i = 0; i < 48; i++) if(!MathIsValidNumber(instrument48[i])) { reason = "decision_context.instrument_nonfinite"; return(false); } if(execution_snapshot24[0] != execution_snapshot24[1] || execution_snapshot24[0] != execution_snapshot24[2] || execution_snapshot24[0] != execution_snapshot24[3] || execution_snapshot24[16] != execution_snapshot24[17]) { reason = "decision_context.causal_snapshot_hold_required"; return(false); } for(uint h = 0; h < config.shape.H; h++) { const long prior_close = h == 0 ? previous_close_server : close_server[h - 1]; if(open_server[h] <= 0 || close_server[h] <= open_server[h] || prior_close > open_server[h] || !MathIsValidNumber(event_counts[h]) || event_counts[h] < 0 || event_counts[h] > (float)config.shape.E || MathFloor(event_counts[h]) != event_counts[h]) { reason = "decision_context.schedule_or_event_count"; return(false); } const long duration = close_server[h] - open_server[h]; const long gap = open_server[h] - prior_close; if(!MathIsValidNumber((float)duration) || !MathIsValidNumber((float)gap) || (double)(float)duration != (double)duration || (double)(float)gap != (double)gap) { reason = "decision_context.schedule_float_precision"; return(false); } for(uint event = 0; event < config.shape.E; event++) for(uint field = 0; field < 6; field++) { const float value = calendar[(h * config.shape.E + event) * 6 + field]; if(!MathIsValidNumber(value) || (event >= (uint)event_counts[h] && value != 0.0f)) { reason = "decision_context.calendar_nonfinite_or_unused"; return(false); } } } //--- Pack only immutable physical inputs; no factual future bar is accepted here. const int instrument_offset = (int)(layout.offsets[2] - prices); const int limits_offset = (int)(layout.offsets[3] - prices); const int execution_offset = (int)(layout.offsets[4] - prices); const int count_offset = (int)(layout.offsets[5] - prices); const int calendar_offset = (int)(layout.offsets[6] - prices); for(int i = 0; i < 24; i++) { if(!snapshot_work.Update(i, account24[i])) { reason = "decision_context.account_pack"; return(false); } if(!snapshot_work.Update(limits_offset + i, limits24[i])) { reason = "decision_context.limits_pack"; return(false); } } for(int i = 0; i < 48; i++) if(!snapshot_work.Update(instrument_offset + i, instrument48[i])) { reason = "decision_context.instrument_pack"; return(false); } for(uint h = 0; h < config.shape.H; h++) { //--- The account already includes events at or before the current decision. uint first_event = 0; uint decision_event_count = (uint)event_counts[h]; if(h == 0) while(first_event < decision_event_count && calendar[(h * config.shape.E + first_event) * 6] <= 0.0f) first_event++; decision_event_count -= first_event; if(!snapshot_work.Update(count_offset + (int)h, (float)decision_event_count)) { reason = "decision_context.count_pack"; return(false); } const long prior_close = h == 0 ? previous_close_server : close_server[h - 1]; for(uint k = 0; k < config.shape.K; k++) { const int row = (int)(k * config.shape.H + h); for(int field = 0; field < 24; field++) { float value = execution_snapshot24[field]; if(field == 18) value = (float)(close_server[h] - open_server[h]); else if(field == 19) value = (float)(open_server[h] - prior_close); else if(field == 21) value = (float)decision_event_count; if(!snapshot_work.Update(execution_offset + row * 24 + field, value)) { reason = "decision_context.execution_pack"; return(false); } } for(uint field = 0; field < config.shape.E * 6; field++) if(!snapshot_work.Update(calendar_offset + row * (int)config.shape.E * 6 + (int)field, (field / 6 < decision_event_count ? calendar[(h * config.shape.E + first_event + field / 6) * 6 + field % 6] : 0.0f))) { reason = "decision_context.calendar_pack"; return(false); } } } if(!snapshot_work.BufferWrite()) { reason = "decision_context.snapshot_upload"; return(false); } //--- Reuse native quote/OHLC packing; financial semantics remain in the rollout validator. CBufferFloat *pack_buffers[] = {GetPointer(snapshot_work), native_ohlc, GetPointer(context_work), GetPointer(flags_work) }; const uint pack_values[] = {config.shape.K, config.shape.H, config.shape.M, 0, (uint)execution_offset, (uint)layout.offsets[7], (uint)layout.total, config.price_basis, config.quote_adapter }; const float pack_reals[] = {(float)config.max_quote_age_seconds, (float)config.max_last_age_seconds}; const uint pack_global[] = {(uint)layout.total}; if(!NNQLaunch(context, def_k_DPCCCompositeQuoteR, pack_buffers, pack_values, pack_reals, pack_global)) { reason = "decision_context.native_pack"; return(false); } CBufferFloat *reduce_buffers[] = {GetPointer(flags_work), GetPointer(status_work)}; const uint reduce_values[] = {(uint)layout.total}; float reduce_reals[]; const uint reduce_global[] = {1}; if(!NNQLaunch(context, def_k_DPCCOperatorReduce, reduce_buffers, reduce_values, reduce_reals, reduce_global)) { reason = "decision_context.native_reduce"; return(false); } if(!status_work.BufferRead()) { reason = "decision_context.status_read"; return(false); } if(status_work[0] != 0.0f) { reason = "decision_context.native_status." + (string)status_work[0]; return(false); } //--- Publish the reusable device buffer only after the complete snapshot passes packing. context_out = GetPointer(context_work); return(true); } //+------------------------------------------------------------------+ //| Typed parse-only offline Oracle admission | //+------------------------------------------------------------------+ struct SDPCCOfflinePolicySeed { string schema, account_currency, source_sha256; long snapshot_bar, snapshot_utc, snapshot_server; float state[], risk_limits[]; SDPCCSimulatedPositionMetadata position; }; struct SDPCCOfflinePolicyProfile { uint version, horizon; string model_sha256, training_sha256, bank_sha256, time_index_sha256, decoder_sha256; string optimizer_name, optimizer_profile, bias_correction, gradient_convention, clip_location, zero_variance_update; double learning_rate, beta1, beta2, epsilon, weight_decay, gradient_clip; long initial_step, episode_id, first_decision_bar, last_decision_bar; float gamma, coordinate_deltas[]; ulong maximum_candidates, maximum_work; SDPCCOfflinePolicySeed seed; string seed_sha256, profile_sha256, fit_plan_sha256; }; struct SDPCCOfflineFitKey { int record_index; string window_id, record_sha256; long episode_id, decision_bar, decision_time; ulong decision_identity, context_identity, owner_identity; }; //--- Projection of already admitted materialized records; not a second bank reader. struct SDPCCOfflineFitWitness { bool admitted; string split, window_id, record_sha256; long episode_id, decision_bar, decision_time, horizon; ulong decision_identity, context_identity, owner_identity; long bar_indices[], bar_open_times[], bar_close_times[]; }; struct SDPCCOfflineFitPartition { string dataset_id; long bar_start, bar_end, start_utc, end_utc, validation_bar_start, validation_start_utc, embargo_bars; }; struct SDPCCOfflinePolicyAdmission { bool admitted; string seed_sha256, profile_sha256, fit_plan_sha256; ulong rows, candidate_bound, work_bound; }; //+------------------------------------------------------------------+ //| Clear the admission and publish explicit empty digest strings | //+------------------------------------------------------------------+ void DPCCClearOfflinePolicyAdmission(SDPCCOfflinePolicyAdmission &out) { ZeroMemory(out); out.seed_sha256 = ""; out.profile_sha256 = ""; out.fit_plan_sha256 = ""; } //+------------------------------------------------------------------+ //| Canonical length-framed fields and exact float32 arrays | //+------------------------------------------------------------------+ void DPCCOfflineField(string &wire, const string value) { wire += (string)StringLen(value) + ":" + value + "\n"; } //+------------------------------------------------------------------+ //| Function DPCCOfflineValidateFiniteFloats | //+------------------------------------------------------------------+ bool DPCCOfflineValidateFiniteFloats(const float &values[], string &reason) { for(int i = 0; i < ArraySize(values); i++) if(!MathIsValidNumber((double)values[i])) { reason = "BLOCKED:offline_policy.nonfinite_array"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCOfflineFloats | //+------------------------------------------------------------------+ bool DPCCOfflineFloats(string &wire, const float &values[], string &reason) { if(!DPCCOfflineValidateFiniteFloats(values, reason)) return(false); const int count = ArraySize(values); const string count_text = (string)count; const ulong count_length = (ulong)StringLen(count_text); const ulong required = (ulong)StringLen(wire) + 2 * count_length + 2 + (ulong)count * 11 + 1; if(required > UINT_MAX) { reason = "BLOCKED:offline_policy.wire_capacity"; return(false); } StringReserve(wire, (uint)required); DPCCOfflineField(wire, count_text); for(int i = 0; i < count; i++) { UDPCCObservedFloatBits value; value.value = values[i]; DPCCOfflineField(wire, StringFormat("%08x", value.bits)); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCOfflineLongs | //+------------------------------------------------------------------+ void DPCCOfflineLongs(string &wire, const long &values[]) { DPCCOfflineField(wire, (string)ArraySize(values)); for(int i = 0; i < ArraySize(values); i++) DPCCOfflineField(wire, (string)values[i]); } //+------------------------------------------------------------------+ //| Function DPCCOfflineBlocked | //+------------------------------------------------------------------+ bool DPCCOfflineBlocked(string &reason, const string prerequisite) { reason = "BLOCKED:offline_policy." + prerequisite; return(false); } //+------------------------------------------------------------------+ //| Recompute profile and ordered-plan digests without authorizing | //+------------------------------------------------------------------+ bool DPCCOfflinePolicyDigests(const SDPCCOfflinePolicyProfile &p, const SDPCCOfflineFitKey &keys[], string &seed_sha, string &profile_sha, string &plan_sha, string &reason) { seed_sha = ""; profile_sha = ""; plan_sha = ""; string wire = "DPCC_OFFLINE_SEED_V1\n"; DPCCOfflineField(wire, p.seed.schema); DPCCOfflineField(wire, p.seed.account_currency); DPCCOfflineField(wire, p.seed.source_sha256); DPCCOfflineField(wire, (string)p.seed.snapshot_bar); DPCCOfflineField(wire, (string)p.seed.snapshot_utc); DPCCOfflineField(wire, (string)p.seed.snapshot_server); if(!DPCCOfflineFloats(wire, p.seed.state, reason) || !DPCCOfflineFloats(wire, p.seed.risk_limits, reason)) return(false); DPCCOfflineField(wire, (string)p.seed.position.initialized); DPCCOfflineField(wire, (string)p.seed.position.snapshot_bar); DPCCOfflineField(wire, (string)p.seed.position.snapshot_server); DPCCOfflineField(wire, (string)p.seed.position.snapshot_time); DPCCOfflineField(wire, (string)p.seed.position.bar_server_open); DPCCOfflineField(wire, (string)p.seed.position.bar_server_close); DPCCOfflineField(wire, (string)p.seed.position.bar_open_time); DPCCOfflineField(wire, (string)p.seed.position.bar_close_time); DPCCOfflineField(wire, (string)p.seed.position.first_open_bar); DPCCOfflineField(wire, (string)p.seed.position.first_open_server); DPCCOfflineField(wire, (string)p.seed.position.first_open_time); float lots[] = {p.seed.position.signed_lots}; if(!DPCCOfflineFloats(wire, lots, reason) || !DPCCCheckpointTextSHA(wire, seed_sha, reason)) return(false); wire = "DPCC_OFFLINE_POLICY_V1\n"; DPCCOfflineField(wire, (string)p.version); DPCCOfflineField(wire, (string)p.horizon); const string sources[] = {p.model_sha256, p.training_sha256, p.bank_sha256, p.time_index_sha256, p.decoder_sha256}; for(int i = 0; i < ArraySize(sources); i++) DPCCOfflineField(wire, sources[i]); const string optimizer[] = {p.optimizer_name, p.optimizer_profile, p.bias_correction, p.gradient_convention, p.clip_location, p.zero_variance_update }; for(int i = 0; i < ArraySize(optimizer); i++) DPCCOfflineField(wire, optimizer[i]); const double numeric[] = {p.learning_rate, p.beta1, p.beta2, p.epsilon, p.weight_decay, p.gradient_clip}; for(int i = 0; i < ArraySize(numeric); i++) { if(!MathIsValidNumber(numeric[i])) return(DPCCOfflineBlocked(reason, "optimizer_numeric")); DPCCOfflineField(wire, StringFormat("%.17g", numeric[i])); } DPCCOfflineField(wire, (string)p.initial_step); DPCCOfflineField(wire, (string)p.episode_id); DPCCOfflineField(wire, (string)p.first_decision_bar); DPCCOfflineField(wire, (string)p.last_decision_bar); DPCCOfflineField(wire, (string)p.maximum_candidates); DPCCOfflineField(wire, (string)p.maximum_work); float discount[] = {p.gamma}; if(!DPCCOfflineFloats(wire, discount, reason) || !DPCCOfflineFloats(wire, p.coordinate_deltas, reason)) return(false); DPCCOfflineField(wire, seed_sha); if(!DPCCCheckpointTextSHA(wire, profile_sha, reason)) return(false); wire = "DPCC_OFFLINE_FIT_PLAN_V1\n"; DPCCOfflineField(wire, profile_sha); DPCCOfflineField(wire, (string)ArraySize(keys)); for(int i = 0; i < ArraySize(keys); i++) { DPCCOfflineField(wire, (string)i); DPCCOfflineField(wire, (string)keys[i].record_index); DPCCOfflineField(wire, keys[i].window_id); DPCCOfflineField(wire, keys[i].record_sha256); DPCCOfflineField(wire, (string)keys[i].episode_id); DPCCOfflineField(wire, (string)keys[i].decision_bar); DPCCOfflineField(wire, (string)keys[i].decision_time); DPCCOfflineField(wire, StringFormat("%016I64x", keys[i].decision_identity)); DPCCOfflineField(wire, StringFormat("%016I64x", keys[i].context_identity)); DPCCOfflineField(wire, StringFormat("%016I64x", keys[i].owner_identity)); } return(DPCCCheckpointTextSHA(wire, plan_sha, reason)); } //+------------------------------------------------------------------+ //| Admit one immutable profile and a complete chronological slice | //+------------------------------------------------------------------+ bool DPCCAdmitOfflinePolicy(const SDPCCOfflinePolicyProfile &p, const SDPCCOfflineFitKey &keys[], const SDPCCOfflineFitWitness &records[], const SDPCCOfflineFitPartition &fit, const SDPCCDatasetTimeIndex &index, const SDPCCConfig &model, const float &model_limits[], SDPCCOfflinePolicyAdmission &out, string &reason) { DPCCClearOfflinePolicyAdmission(out); reason = ""; //--- Require explicit values and the current native CNet optimizer contract. const int count = ArraySize(keys), materialized = ArraySize(records), bars = ArraySize(index.bar_index); if(p.version != 1 || p.horizon == 0 || p.horizon != model.shape.H || p.horizon > (INT_MAX - 1) / 12 || count <= 0 || materialized <= 0 || p.maximum_candidates == 0 || p.maximum_work == 0 || p.first_decision_bar < 0 || p.last_decision_bar < p.first_decision_bar || p.last_decision_bar - p.first_decision_bar != (long)count - 1 || ArraySize(p.coordinate_deltas) != 6 || !MathIsValidNumber((double)p.gamma) || p.gamma <= 0 || p.gamma > 1 || p.gamma != model.gamma) return(DPCCOfflineBlocked(reason, "required_profile_geometry_discount")); const string hashes[] = {p.model_sha256, p.training_sha256, p.bank_sha256, p.time_index_sha256, p.decoder_sha256, p.seed.source_sha256, p.seed_sha256, p.profile_sha256, p.fit_plan_sha256 }; for(int i = 0; i < ArraySize(hashes); i++) if(!DPCCArchitectureSHA256Encoding(hashes[i])) return(DPCCOfflineBlocked(reason, "required_source_digests")); for(int i = 0; i < 6; i++) if(!MathIsValidNumber((double)p.coordinate_deltas[i]) || p.coordinate_deltas[i] <= 0 || p.coordinate_deltas[i] > 1) return(DPCCOfflineBlocked(reason, "coordinate_delta")); if(p.optimizer_name != "ADAM" || p.optimizer_profile != "NN_NATIVE_CONV_ADAM_C6A8_V1" || p.learning_rate != 0.0003 || p.beta1 != 0.9 || p.beta2 != 0.999 || p.epsilon != 0 || p.weight_decay != 0 || p.gradient_clip != 0.01 || p.initial_step != 1 || p.bias_correction != "HOST_LT_FROM_T" || p.gradient_convention != "NEGATIVE_DERIVATIVE" || p.clip_location != "PER_WEIGHT_AFTER_TOKEN_SUM" || p.zero_variance_update != "ZERO_DELTA") return(DPCCOfflineBlocked(reason, "native_optimizer_contract")); if(p.seed.schema != "DPCC_NATIVE_STATE33_V1" || p.seed.account_currency == "" || p.seed.account_currency != AccountInfoString(ACCOUNT_CURRENCY) || ArraySize(p.seed.state) != 33 || ArraySize(p.seed.risk_limits) != 24 || ArraySize(model_limits) != 24 || p.seed.snapshot_bar != p.first_decision_bar || p.seed.snapshot_utc != keys[0].decision_time || p.seed.snapshot_server <= 0) return(DPCCOfflineBlocked(reason, "exact_starting_seed")); for(int i = 0; i < 33; i++) if(!MathIsValidNumber((double)p.seed.state[i])) return(DPCCOfflineBlocked(reason, "starting_state_numeric")); for(int i = 0; i < 24; i++) { UDPCCObservedFloatBits actual, expected; actual.value = p.seed.risk_limits[i]; expected.value = model_limits[i]; if(!MathIsValidNumber((double)actual.value) || !MathIsValidNumber((double)expected.value) || actual.bits != expected.bits) return(DPCCOfflineBlocked(reason, "starting_risk_model_binding")); } if(p.seed.state[0] != 1000.0f || p.seed.state[1] < 50.0f || p.seed.state[6] < 0 || p.seed.state[24] != 0 || p.seed.state[32] != 0) return(DPCCOfflineBlocked(reason, "starting_account_contract")); if(p.seed.state[2] == 0 && (p.seed.state[6] != 0 || p.seed.state[1] != p.seed.state[0])) return(DPCCOfflineBlocked(reason, "starting_flat_money_invariant")); if(!MathIsValidNumber((double)p.seed.position.signed_lots)) return(DPCCOfflineBlocked(reason, "starting_position_numeric")); if(p.seed.position.initialized) { if(p.seed.position.snapshot_bar != p.seed.snapshot_bar || p.seed.position.snapshot_time != p.seed.snapshot_utc || p.seed.position.snapshot_server != p.seed.snapshot_server || p.seed.position.bar_server_open != p.seed.snapshot_server || p.seed.position.bar_open_time != p.seed.snapshot_utc || p.seed.position.bar_server_close <= p.seed.snapshot_server || p.seed.position.bar_close_time <= p.seed.snapshot_utc || p.seed.position.signed_lots == 0 || p.seed.position.signed_lots != p.seed.state[2] || p.seed.position.first_open_bar < 0 || p.seed.position.first_open_server <= 0 || p.seed.position.first_open_time <= 0 || p.seed.position.first_open_bar > p.seed.snapshot_bar || p.seed.position.first_open_server > p.seed.snapshot_server || p.seed.position.first_open_time > p.seed.snapshot_utc) return(DPCCOfflineBlocked(reason, "starting_position_binding")); return(DPCCOfflineBlocked(reason, "starting_position_bootstrap_unsupported")); } else if(p.seed.state[2] != 0 || p.seed.state[3] != 0 || p.seed.state[4] != 0 || p.seed.state[5] != 0 || p.seed.position.signed_lots != 0 || p.seed.position.snapshot_bar != 0 || p.seed.position.snapshot_server != 0 || p.seed.position.snapshot_time != 0 || p.seed.position.bar_server_open != 0 || p.seed.position.bar_server_close != 0 || p.seed.position.bar_open_time != 0 || p.seed.position.bar_close_time != 0 || p.seed.position.first_open_bar != 0 || p.seed.position.first_open_server != 0 || p.seed.position.first_open_time != 0) return(DPCCOfflineBlocked(reason, "uninitialized_position_requires_explicit_flat")); //--- Bound admission scans and future candidate-step work before any traversal. const ulong candidates = 1 + (ulong)p.horizon * 12; if(candidates > p.maximum_candidates || candidates > p.maximum_work / (ulong)p.horizon / (ulong)count || (ulong)materialized > p.maximum_work / (ulong)count || (ulong)bars > p.maximum_work) return(DPCCOfflineBlocked(reason, "finite_candidate_work_cap")); const ulong scoring = candidates * (ulong)p.horizon * (ulong)count; const ulong scans = (ulong)materialized * (ulong)count; if(scoring > p.maximum_work - scans || scoring + scans > p.maximum_work - (ulong)bars) return(DPCCOfflineBlocked(reason, "finite_total_work_cap")); if(!DPCCValidateDatasetTimeIndex(index, fit.dataset_id, reason)) return(DPCCOfflineBlocked(reason, "time_index." + reason)); if(fit.bar_start < 0 || fit.bar_end <= fit.bar_start || fit.bar_end > bars || fit.start_utc != index.start_utc[(int)fit.bar_start] || fit.end_utc != index.end_utc[(int)fit.bar_end - 1] || fit.validation_bar_start < fit.bar_end || fit.validation_bar_start >= bars || fit.validation_start_utc != index.start_utc[(int)fit.validation_bar_start] || fit.embargo_bars < (long)p.horizon || fit.validation_bar_start - fit.bar_end < fit.embargo_bars || fit.validation_start_utc < fit.end_utc || p.first_decision_bar < fit.bar_start || p.last_decision_bar >= fit.bar_end || (long)p.horizon > fit.bar_end - p.last_decision_bar) return(DPCCOfflineBlocked(reason, "fit_partition_embargo_full_horizon")); //--- Join each exact key and reject missing, duplicate or ambiguous origins. for(int i = 0; i < count; i++) { const int record = keys[i].record_index; if(record < 0 || record >= materialized || keys[i].window_id == "" || !DPCCArchitectureSHA256Encoding(keys[i].record_sha256) || keys[i].episode_id != p.episode_id || keys[i].decision_bar != p.first_decision_bar + i || keys[i].decision_identity == 0 || keys[i].context_identity == 0 || keys[i].owner_identity == 0) return(DPCCOfflineBlocked(reason, "fit_key_identity_order")); int matches = 0; for(int j = 0; j < materialized; j++) if(records[j].window_id == keys[i].window_id || records[j].decision_bar == keys[i].decision_bar || records[j].decision_identity == keys[i].decision_identity) { if(j != record) return(DPCCOfflineBlocked(reason, "ambiguous_materialized_key")); matches++; } if(matches != 1 || !records[record].admitted || records[record].split != "fit" || records[record].window_id != keys[i].window_id || records[record].record_sha256 != keys[i].record_sha256 || records[record].episode_id != keys[i].episode_id || records[record].decision_bar != keys[i].decision_bar || records[record].decision_time != keys[i].decision_time || records[record].decision_identity != keys[i].decision_identity || records[record].context_identity != keys[i].context_identity || records[record].owner_identity != keys[i].owner_identity || records[record].horizon != (long)p.horizon || ArraySize(records[record].bar_indices) != (int)p.horizon || ArraySize(records[record].bar_open_times) != (int)p.horizon || ArraySize(records[record].bar_close_times) != (int)p.horizon || keys[i].decision_time != index.start_utc[(int)keys[i].decision_bar] || (i > 0 && keys[i].decision_time <= keys[i - 1].decision_time)) return(DPCCOfflineBlocked(reason, "exact_materialized_record_binding")); for(uint h = 0; h < p.horizon; h++) { const int b = (int)(keys[i].decision_bar + h); if(records[record].bar_indices[h] != (long)b || records[record].bar_open_times[h] != index.start_utc[b] || records[record].bar_close_times[h] != index.end_utc[b] || index.start_utc[b] < fit.start_utc || index.end_utc[b] > fit.end_utc) return(DPCCOfflineBlocked(reason, "factual_target_horizon")); } } string seed_sha = "", profile_sha = "", plan_sha = ""; if(!DPCCOfflinePolicyDigests(p, keys, seed_sha, profile_sha, plan_sha, reason) || seed_sha != p.seed_sha256 || profile_sha != p.profile_sha256 || plan_sha != p.fit_plan_sha256) return(DPCCOfflineBlocked(reason, "immutable_digest_binding")); out.admitted = true; out.seed_sha256 = seed_sha; out.profile_sha256 = profile_sha; out.fit_plan_sha256 = plan_sha; out.rows = (ulong)count; out.candidate_bound = candidates; out.work_bound = scoring + scans + (ulong)bars; return(true); } //+------------------------------------------------------------------+ //| Complete-row result; publication follows native update and | //| commit | //+------------------------------------------------------------------+ #ifndef DPCC_MARKET_CLOCK_BINDING_WORK_MQH #define DPCC_MARKET_CLOCK_BINDING_WORK_MQH #ifndef DPCC_CLOCK_SOURCE_MQH #define DPCC_CLOCK_SOURCE_MQH //+------------------------------------------------------------------+ //| BankMetadata.mqh: DPCC BankManifest and observed header closure | //+------------------------------------------------------------------+ #ifndef DPCC_BANK_METADATA_MQH #define DPCC_BANK_METADATA_MQH //+------------------------------------------------------------------+ //| DatasetMetadata.mqh: linked Dataset and data-evidence closure | //+------------------------------------------------------------------+ #ifndef DPCC_DATASET_METADATA_MQH #define DPCC_DATASET_METADATA_MQH //+------------------------------------------------------------------+ //| LinkedMetadata.mqh: DPCC central metadata link closure | //+------------------------------------------------------------------+ #ifndef DPCC_LINKED_METADATA_MQH #define DPCC_LINKED_METADATA_MQH //+------------------------------------------------------------------+ //| ArchitectureMetadata.mqh: DPCC META-08 JSON adapter | //+------------------------------------------------------------------+ #ifndef DPCC_ARCHITECTURE_METADATA_MQH #define DPCC_ARCHITECTURE_METADATA_MQH //+------------------------------------------------------------------+ //| Metadata.mqh: DPCC DatasetTimeIndex JSON adapter | //+------------------------------------------------------------------+ #ifndef DPCC_METADATA_MQH #define DPCC_METADATA_MQH //+------------------------------------------------------------------+ //| DPCCClearDatasetTimeIndex clears a typed metadata value | //+------------------------------------------------------------------+ void DPCCClearDatasetTimeIndex(SDPCCDatasetTimeIndex &index) { index.schema_version = 0; index.example_only = false; index.time_index_id = ""; index.dataset_id = ""; index.data_source = ""; index.timezone_mapping.path = ""; index.timezone_mapping.sha256 = ""; index.timezone_mapping.id = ""; ArrayResize(index.bar_index, 0); ArrayResize(index.start_utc, 0); ArrayResize(index.end_utc, 0); } //+------------------------------------------------------------------+ //| DPCCMetadataFail clears output and publishes one failure reason | //+------------------------------------------------------------------+ bool DPCCMetadataFail(SDPCCDatasetTimeIndex &index, string &reason, const string failure) { DPCCClearDatasetTimeIndex(index); reason = failure; return(false); } //+------------------------------------------------------------------+ //| DPCCReadExactNonnegativeLong reads an exact decimal long | //+------------------------------------------------------------------+ bool DPCCReadExactNonnegativeLong(CDPCCJsonDocument &document, const int node, long &value) { value = 0; if(document.Type(node) != DPCC_JSON_NUMBER) return(false); const string lexeme = document.Text(node); const int length = StringLen(lexeme); if(length <= 0) return(false); const bool negative_value = (StringGetCharacter(lexeme, 0) == '-'); const int mantissa_start = (negative_value ? 1 : 0); int exponent_pos = StringFind(lexeme, "e"); if(exponent_pos < 0) exponent_pos = StringFind(lexeme, "E"); const int mantissa_end = (exponent_pos < 0 ? length : exponent_pos); int exponent = 0; if(exponent_pos >= 0) { int position = exponent_pos + 1; bool exponent_negative = false; if(StringGetCharacter(lexeme, position) == '+' || StringGetCharacter(lexeme, position) == '-') { exponent_negative = (StringGetCharacter(lexeme, position) == '-'); position++; } for(; position < length; position++) { const int digit = (int)(StringGetCharacter(lexeme, position) - '0'); if(exponent > DPCC_JSON_MAX_SOURCE_CHARS) exponent = DPCC_JSON_MAX_SOURCE_CHARS + 1; else exponent = exponent * 10 + digit; } if(exponent_negative) exponent = -exponent; } string mantissa = StringSubstr(lexeme, mantissa_start, mantissa_end - mantissa_start); const int decimal_pos = StringFind(mantissa, "."); const int fraction = (decimal_pos < 0 ? 0 : StringLen(mantissa) - decimal_pos - 1); string digits = mantissa; if(decimal_pos >= 0) StringReplace(digits, ".", ""); int first = 0; while(first < StringLen(digits) && StringGetCharacter(digits, first) == '0') first++; if(first >= StringLen(digits)) return(true); if(negative_value) return(false); digits = StringSubstr(digits, first); int scale = exponent - fraction; if(scale < 0) { const int remove = -scale; if(remove > StringLen(digits)) return(false); for(int i = StringLen(digits) - remove; i < StringLen(digits); i++) if(StringGetCharacter(digits, i) != '0') return(false); digits = StringSubstr(digits, 0, StringLen(digits) - remove); scale = 0; if(StringLen(digits) == 0) return(true); } if(StringLen(digits) + scale > 19) return(false); for(int i = 0; i < StringLen(digits) + scale; i++) { const int digit = (i < StringLen(digits) ? (int)(StringGetCharacter(digits, i) - '0') : 0); if(value > (LONG_MAX - digit) / 10) return(false); value = value * 10 + digit; } return(true); } //+------------------------------------------------------------------+ //| DPCCReadRequiredString reads one required string property | //+------------------------------------------------------------------+ bool DPCCReadRequiredString(CDPCCJsonDocument &document, const int object_node, const string name, string &value, string &reason) { const int node = document.FindMember(object_node, name); if(node < 0) { reason = name; return(false); } if(document.Type(node) != DPCC_JSON_STRING) { reason = name + ".type"; return(false); } value = document.Text(node); return(true); } //+------------------------------------------------------------------+ //| DPCCReadRequiredLong reads one required exact integer property | //+------------------------------------------------------------------+ bool DPCCReadRequiredLong(CDPCCJsonDocument &document, const int object_node, const string name, long &value, string &reason, const string path) { const int node = document.FindMember(object_node, name); if(node < 0) { reason = path; return(false); } if(!DPCCReadExactNonnegativeLong(document, node, value)) { reason = path + ".integer"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCDecodeDatasetTimeIndex decodes and validates META-09 JSON | //+------------------------------------------------------------------+ bool DPCCDecodeDatasetTimeIndex(CDPCCJsonDocument &document, const int root, const string expected_dataset_id, SDPCCDatasetTimeIndex &index, string &reason) { DPCCClearDatasetTimeIndex(index); reason = ""; if(document.Type(root) != DPCC_JSON_OBJECT) return(DPCCMetadataFail(index, reason, "root.type")); if(document.ChildCount(root) != 7) return(DPCCMetadataFail(index, reason, "root.properties")); long schema_version = 0; if(!DPCCReadRequiredLong(document, root, "schema_version", schema_version, reason, "schema_version")) return(DPCCMetadataFail(index, reason, reason)); index.schema_version = schema_version; const int example_node = document.FindMember(root, "example_only"); if(example_node < 0) return(DPCCMetadataFail(index, reason, "example_only")); if(document.Type(example_node) != DPCC_JSON_BOOLEAN) return(DPCCMetadataFail(index, reason, "example_only.type")); index.example_only = (document.Text(example_node) == "true"); if(!DPCCReadRequiredString(document, root, "time_index_id", index.time_index_id, reason)) return(DPCCMetadataFail(index, reason, reason)); if(!DPCCReadRequiredString(document, root, "dataset_id", index.dataset_id, reason)) return(DPCCMetadataFail(index, reason, reason)); if(!DPCCReadRequiredString(document, root, "data_source", index.data_source, reason)) return(DPCCMetadataFail(index, reason, reason)); const int timezone_node = document.FindMember(root, "timezone_mapping"); if(timezone_node < 0) return(DPCCMetadataFail(index, reason, "timezone_mapping")); if(document.Type(timezone_node) != DPCC_JSON_OBJECT) return(DPCCMetadataFail(index, reason, "timezone_mapping.type")); if(document.ChildCount(timezone_node) != 3) return(DPCCMetadataFail(index, reason, "timezone_mapping.properties")); string field_reason = ""; if(!DPCCReadRequiredString(document, timezone_node, "path", index.timezone_mapping.path, field_reason)) return(DPCCMetadataFail(index, reason, "timezone_mapping." + field_reason)); if(!DPCCReadRequiredString(document, timezone_node, "sha256", index.timezone_mapping.sha256, field_reason)) return(DPCCMetadataFail(index, reason, "timezone_mapping." + field_reason)); if(!DPCCReadRequiredString(document, timezone_node, "id", index.timezone_mapping.id, field_reason)) return(DPCCMetadataFail(index, reason, "timezone_mapping." + field_reason)); const int bars_node = document.FindMember(root, "bars"); if(bars_node < 0) return(DPCCMetadataFail(index, reason, "bars")); if(document.Type(bars_node) != DPCC_JSON_ARRAY) return(DPCCMetadataFail(index, reason, "bars.type")); const int count = document.ChildCount(bars_node); if(count <= 0) return(DPCCMetadataFail(index, reason, "bars")); if(ArrayResize(index.bar_index, count) != count) return(DPCCMetadataFail(index, reason, "bars.bar_index.storage")); if(ArrayResize(index.start_utc, count) != count) return(DPCCMetadataFail(index, reason, "bars.start_utc.storage")); if(ArrayResize(index.end_utc, count) != count) return(DPCCMetadataFail(index, reason, "bars.end_utc.storage")); int bar_node = document.FirstChild(bars_node); for(int i = 0; i < count; i++) { const string base = StringFormat("bars[%d]", i); if(bar_node < 0 || document.Type(bar_node) != DPCC_JSON_OBJECT) return(DPCCMetadataFail(index, reason, base + ".type")); if(document.ChildCount(bar_node) != 3) return(DPCCMetadataFail(index, reason, base + ".properties")); if(!DPCCReadRequiredLong(document, bar_node, "bar_index", index.bar_index[i], reason, base + ".bar_index")) return(DPCCMetadataFail(index, reason, reason)); if(!DPCCReadRequiredLong(document, bar_node, "start_utc", index.start_utc[i], reason, base + ".start_utc")) return(DPCCMetadataFail(index, reason, reason)); if(!DPCCReadRequiredLong(document, bar_node, "end_utc", index.end_utc[i], reason, base + ".end_utc")) return(DPCCMetadataFail(index, reason, reason)); bar_node = document.NextSibling(bar_node); } string validation_reason = ""; if(!DPCCValidateDatasetTimeIndex(index, expected_dataset_id, validation_reason)) return(DPCCMetadataFail(index, reason, validation_reason)); reason = ""; return(true); } #endif //+------------------------------------------------------------------+ //| DPCCClearArchitectureContract clears every nested contract field | //+------------------------------------------------------------------+ void DPCCClearArchitectureContract(SDPCCArchitectureContract &contract) { contract.model_decoder.architecture = ""; contract.model_decoder.embedding = 0; contract.model_decoder.activation = ""; contract.model_decoder.backward_profile = ""; contract.model_denoiser.architecture = ""; contract.model_denoiser.embedding = 0; contract.model_denoiser.heads = 0; contract.model_denoiser.key_dimension = 0; contract.model_denoiser.depth = 0; contract.model_denoiser.padding_mask = false; contract.model_denoiser.graph_id = ""; contract.model_denoiser.normalization = ""; contract.model_denoiser.feedforward_activation = ""; contract.model_denoiser.temporal_mask = false; contract.model_denoiser.attention_backward = ""; contract.model_denoiser.graph_sha256 = ""; contract.model_denoiser.backward_profile = ""; contract.model_parallel.version = 0; contract.model_parallel.program = ""; contract.model_parallel.training_plan_batch = 0; contract.model_parallel.deployment_plan_batch = 0; contract.model_parallel.proposal_slots = 0; contract.model_parallel.logical_heads = 0; contract.model_parallel.key_dimension = 0; contract.model_parallel.shared_projection_weights = false; contract.model_parallel.self_attention_fold = ""; contract.model_parallel.cross_attention_fold = ""; contract.model_parallel.activation_lifetime = ""; contract.model_parallel.binary_layer_version = 0; contract.model_parallel.legacy_slice_use = ""; contract.training_decoder.architecture = ""; contract.training_decoder.embedding = 0; contract.training_decoder.activation = ""; contract.training_decoder.backward_profile = ""; contract.training_denoiser.architecture = ""; contract.training_denoiser.embedding = 0; contract.training_denoiser.heads = 0; contract.training_denoiser.key_dimension = 0; contract.training_denoiser.depth = 0; contract.training_denoiser.padding_mask = false; contract.training_denoiser.graph_id = ""; contract.training_denoiser.normalization = ""; contract.training_denoiser.feedforward_activation = ""; contract.training_denoiser.temporal_mask = false; contract.training_denoiser.attention_backward = ""; contract.training_denoiser.graph_sha256 = ""; contract.training_denoiser.backward_profile = ""; contract.training_parallel.version = 0; contract.training_parallel.program = ""; contract.training_parallel.training_plan_batch = 0; contract.training_parallel.deployment_plan_batch = 0; contract.training_parallel.proposal_slots = 0; contract.training_parallel.logical_heads = 0; contract.training_parallel.key_dimension = 0; contract.training_parallel.shared_projection_weights = false; contract.training_parallel.self_attention_fold = ""; contract.training_parallel.cross_attention_fold = ""; contract.training_parallel.activation_lifetime = ""; contract.training_parallel.binary_layer_version = 0; contract.training_parallel.legacy_slice_use = ""; } //+------------------------------------------------------------------+ //| DPCCArchitectureMetadataFail clears output and sets stable | //| reason | //+------------------------------------------------------------------+ void DPCCArchitectureMetadataFail(SDPCCArchitectureContract &contract, string &reason, const string failure) { DPCCClearArchitectureContract(contract); reason = failure; } //+------------------------------------------------------------------+ //| Find one required object along a normative JSON path | //+------------------------------------------------------------------+ bool DPCCArchitectureObject(CDPCCJsonDocument &document, const int parent, const string name, const string path, int &node, string &reason) { node = document.FindMember(parent, name); if(node < 0) { reason = path; return(false); } if(document.Type(node) != DPCC_JSON_OBJECT) { reason = path + ".type"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Read required typed META-08 string | //+------------------------------------------------------------------+ bool DPCCArchitectureString(CDPCCJsonDocument &document, const int object_node, const string name, const string path, string &value, string &reason) { const int node = document.FindMember(object_node, name); if(node < 0) { reason = path; return(false); } if(document.Type(node) != DPCC_JSON_STRING) { reason = path + ".type"; return(false); } value = document.Text(node); return(true); } //+------------------------------------------------------------------+ //| Read mathematical integer exactly and range-check before uint | //+------------------------------------------------------------------+ bool DPCCArchitectureUInt(CDPCCJsonDocument &document, const int object_node, const string name, const string path, uint &value, string &reason) { value = 0; const int node = document.FindMember(object_node, name); if(node < 0) { reason = path; return(false); } long exact = 0; if(!DPCCReadExactNonnegativeLong(document, node, exact)) { reason = path + ".integer"; return(false); } if(exact > 4294967295L) { reason = path + ".range"; return(false); } value = (uint)exact; return(true); } //+------------------------------------------------------------------+ //| Read required META-08 boolean | //+------------------------------------------------------------------+ bool DPCCArchitectureBool(CDPCCJsonDocument &document, const int object_node, const string name, const string path, bool &value, string &reason) { const int node = document.FindMember(object_node, name); if(node < 0) { reason = path; return(false); } if(document.Type(node) != DPCC_JSON_BOOLEAN) { reason = path + ".type"; return(false); } value = (document.Text(node) == "true"); return(true); } //+------------------------------------------------------------------+ //| Decode decoder architecture | //+------------------------------------------------------------------+ bool DPCCDecodeDecoderArchitecture(CDPCCJsonDocument &document, const int node, const string path, SDPCCDecoderArchitecture &value, string &reason) { return(DPCCArchitectureString(document, node, "architecture", path + ".architecture", value.architecture, reason) && DPCCArchitectureUInt(document, node, "embedding", path + ".embedding", value.embedding, reason) && DPCCArchitectureString(document, node, "activation", path + ".activation", value.activation, reason) && DPCCArchitectureString(document, node, "backward_profile", path + ".backward_profile", value.backward_profile, reason)); } //+------------------------------------------------------------------+ //| Decode denoiser architecture | //+------------------------------------------------------------------+ bool DPCCDecodeDenoiserArchitecture(CDPCCJsonDocument &document, const int node, const string path, SDPCCDenoiserArchitecture &value, string &reason) { return(DPCCArchitectureString(document, node, "architecture", path + ".architecture", value.architecture, reason) && DPCCArchitectureUInt(document, node, "embedding", path + ".embedding", value.embedding, reason) && DPCCArchitectureUInt(document, node, "heads", path + ".heads", value.heads, reason) && DPCCArchitectureUInt(document, node, "key_dimension", path + ".key_dimension", value.key_dimension, reason) && DPCCArchitectureUInt(document, node, "depth", path + ".depth", value.depth, reason) && DPCCArchitectureBool(document, node, "padding_mask", path + ".padding_mask", value.padding_mask, reason) && DPCCArchitectureString(document, node, "graph_id", path + ".graph_id", value.graph_id, reason) && DPCCArchitectureString(document, node, "normalization", path + ".normalization", value.normalization, reason) && DPCCArchitectureString(document, node, "feedforward_activation", path + ".feedforward_activation", value.feedforward_activation, reason) && DPCCArchitectureBool(document, node, "temporal_mask", path + ".temporal_mask", value.temporal_mask, reason) && DPCCArchitectureString(document, node, "attention_backward", path + ".attention_backward", value.attention_backward, reason) && DPCCArchitectureString(document, node, "graph_sha256", path + ".graph_sha256", value.graph_sha256, reason) && DPCCArchitectureString(document, node, "backward_profile", path + ".backward_profile", value.backward_profile, reason)); } //+------------------------------------------------------------------+ //| Decode parallel execution contract | //+------------------------------------------------------------------+ bool DPCCDecodeParallelExecution(CDPCCJsonDocument &document, const int node, const string path, SDPCCParallelExecution &value, string &reason) { return(DPCCArchitectureUInt(document, node, "version", path + ".version", value.version, reason) && DPCCArchitectureString(document, node, "program", path + ".program", value.program, reason) && DPCCArchitectureUInt(document, node, "training_plan_batch", path + ".training_plan_batch", value.training_plan_batch, reason) && DPCCArchitectureUInt(document, node, "deployment_plan_batch", path + ".deployment_plan_batch", value.deployment_plan_batch, reason) && DPCCArchitectureUInt(document, node, "proposal_slots", path + ".proposal_slots", value.proposal_slots, reason) && DPCCArchitectureUInt(document, node, "logical_heads", path + ".logical_heads", value.logical_heads, reason) && DPCCArchitectureUInt(document, node, "key_dimension", path + ".key_dimension", value.key_dimension, reason) && DPCCArchitectureBool(document, node, "shared_projection_weights", path + ".shared_projection_weights", value.shared_projection_weights, reason) && DPCCArchitectureString(document, node, "self_attention_fold", path + ".self_attention_fold", value.self_attention_fold, reason) && DPCCArchitectureString(document, node, "cross_attention_fold", path + ".cross_attention_fold", value.cross_attention_fold, reason) && DPCCArchitectureString(document, node, "activation_lifetime", path + ".activation_lifetime", value.activation_lifetime, reason) && DPCCArchitectureUInt(document, node, "binary_layer_version", path + ".binary_layer_version", value.binary_layer_version, reason) && DPCCArchitectureString(document, node, "legacy_slice_use", path + ".legacy_slice_use", value.legacy_slice_use, reason)); } //+------------------------------------------------------------------+ //| Extract schema-backed paths then apply the accepted META-08 gate | //+------------------------------------------------------------------+ bool DPCCDecodeArchitectureContract(CDPCCJsonDocument &model_document, const int model_root, CDPCCJsonDocument &training_document, const int training_root, const string expected_graph_sha256, SDPCCArchitectureContract &contract, string &reason) { DPCCClearArchitectureContract(contract); reason = ""; int composition = -1, model_decoder = -1, model_denoiser = -1, model_parallel = -1; int training_decoder = -1, training_denoiser = -1, training_parallel = -1; if(!DPCCArchitectureObject(model_document, model_root, "composition", "model.composition", composition, reason) || !DPCCArchitectureObject(model_document, composition, "decoder", "model.composition.decoder", model_decoder, reason) || !DPCCArchitectureObject(model_document, composition, "denoiser", "model.composition.denoiser", model_denoiser, reason) || !DPCCArchitectureObject(model_document, model_root, "parallel_execution", "model.parallel_execution", model_parallel, reason) || !DPCCArchitectureObject(training_document, training_root, "decoder", "training.decoder", training_decoder, reason) || !DPCCArchitectureObject(training_document, training_decoder, "architecture", "training.decoder.architecture", training_decoder, reason) || !DPCCArchitectureObject(training_document, training_root, "denoiser", "training.denoiser", training_denoiser, reason) || !DPCCArchitectureObject(training_document, training_denoiser, "architecture", "training.denoiser.architecture", training_denoiser, reason) || !DPCCArchitectureObject(training_document, training_root, "parallel_execution", "training.parallel_execution", training_parallel, reason)) { DPCCArchitectureMetadataFail(contract, reason, reason); return(false); } if(!DPCCDecodeDecoderArchitecture(model_document, model_decoder, "model.composition.decoder", contract.model_decoder, reason) || !DPCCDecodeDenoiserArchitecture(model_document, model_denoiser, "model.composition.denoiser", contract.model_denoiser, reason) || !DPCCDecodeParallelExecution(model_document, model_parallel, "model.parallel_execution", contract.model_parallel, reason) || !DPCCDecodeDecoderArchitecture(training_document, training_decoder, "training.decoder.architecture", contract.training_decoder, reason) || !DPCCDecodeDenoiserArchitecture(training_document, training_denoiser, "training.denoiser.architecture", contract.training_denoiser, reason) || !DPCCDecodeParallelExecution(training_document, training_parallel, "training.parallel_execution", contract.training_parallel, reason)) { DPCCArchitectureMetadataFail(contract, reason, reason); return(false); } if(!DPCCValidateArchitectureContract(contract, expected_graph_sha256, reason)) { DPCCArchitectureMetadataFail(contract, reason, reason); return(false); } reason = ""; return(true); } #endif //+------------------------------------------------------------------+ //| ReadinessMetadata.mqh: JSON adapters for DPCC resource evidence | //+------------------------------------------------------------------+ #ifndef DPCC_READINESS_METADATA_MQH #define DPCC_READINESS_METADATA_MQH //+------------------------------------------------------------------+ //| Function DPCCClearResourceMeasurement | //+------------------------------------------------------------------+ void DPCCClearResourceMeasurement(SDPCCResourceMeasurement &m) { m.measurement_id = ""; m.stage = ""; m.profile_id = ""; m.benchmark_case = ""; m.device = ""; m.driver = ""; m.opencl_version = ""; m.terminal_build = ""; m.projection_mode = ""; m.execution_program = ""; m.repeat_index = 0; m.warmup_completed = 0; m.projector_max_evaluations = 0; m.projector_max_iterations = 0; m.measured_started_at_utc = 0; m.shapes.K = 0; m.shapes.V = 0; m.shapes.H = 0; m.shapes.D = 0; m.shapes.M = 0; m.shapes.J = 0; m.shapes.E = 0; m.shapes.A = 0; m.shapes.B = 0; m.shapes.account_features = 0; m.attempts = 0; m.completed = 0; m.censored = 0; m.timeouts = 0; m.stale_decisions = 0; m.p50_ms = 0; m.p95_ms = 0; m.p99_ms = 0; m.max_ms = 0; m.max_decision_age_ms = 0; m.tracked_peak_bytes = 0; m.total_peak_bytes = 0; m.available_device_bytes = 0; m.total_memory_telemetry_available = false; } //+------------------------------------------------------------------+ //| Function DPCCClearResourceEvidence | //+------------------------------------------------------------------+ void DPCCClearResourceEvidence(SDPCCResourceEvidence &e) { e.evidence_id = ""; e.profile_ref_id = ""; e.profile_ref_sha256 = ""; DPCCClearResourceMeasurement(e.measurement); ArrayResize(e.samples, 0); } //+------------------------------------------------------------------+ //| Function DPCCResourceDecodeFail | //+------------------------------------------------------------------+ bool DPCCResourceDecodeFail(SDPCCResourceEvidence &e, string &reason, const string failure) { DPCCClearResourceEvidence(e); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCClearResourceProfile | //+------------------------------------------------------------------+ void DPCCClearResourceProfile(SDPCCResourceProfile &p) { p.readiness_profile_id = ""; p.calibration_state = ""; p.profile_id = ""; p.device = ""; p.driver = ""; p.opencl_version = ""; p.terminal_build = ""; p.frozen_at_utc = 0; p.shapes.K = 0; p.shapes.V = 0; p.shapes.H = 0; p.shapes.D = 0; p.shapes.M = 0; p.shapes.J = 0; p.shapes.E = 0; p.shapes.A = 0; p.shapes.B = 0; p.shapes.account_features = 0; p.projector_max_evaluations = 0; p.projector_max_iterations = 0; p.p95_limit_ms = 0; p.p99_limit_ms = 0; p.max_compute_ms = 0; p.max_decision_age_ms = 0; p.tracked_device_limit_bytes = 0; p.total_device_limit_bytes = 0; p.reserve_device_bytes = 0; p.warmup_decisions = 0; p.measured_decisions = 0; p.repeats = 0; p.censored_policy = ""; p.algorithmic_limit = ""; p.stress_policy = ""; p.execution_program = ""; ArrayResize(p.benchmark_cases, 0); } //+------------------------------------------------------------------+ //| Function DPCCResourceProfileDecodeFail | //+------------------------------------------------------------------+ bool DPCCResourceProfileDecodeFail(SDPCCResourceProfile &p, string &reason, const string failure) { DPCCClearResourceProfile(p); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCResourceReadString | //+------------------------------------------------------------------+ bool DPCCResourceReadString(CDPCCJsonDocument &d, const int object, const string name, string &value, string &reason, const string path) { const int node = d.FindMember(object, name); if(node < 0 || d.Type(node) != DPCC_JSON_STRING) { reason = path; return(false); } value = d.Text(node); return(true); } //+------------------------------------------------------------------+ //| Function DPCCResourceReadLong | //+------------------------------------------------------------------+ bool DPCCResourceReadLong(CDPCCJsonDocument &d, const int object, const string name, long &value, string &reason, const string path) { const int node = d.FindMember(object, name); if(node < 0 || !DPCCReadExactNonnegativeLong(d, node, value)) { reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCResourceReadDouble | //+------------------------------------------------------------------+ bool DPCCResourceReadDouble(CDPCCJsonDocument &d, const int object, const string name, double &value, string &reason, const string path) { const int node = d.FindMember(object, name); if(node < 0 || !d.TryNumber(node, value) || !DPCCResourceFiniteNonnegative(value)) { reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCResourceReadBool | //+------------------------------------------------------------------+ bool DPCCResourceReadBool(CDPCCJsonDocument &d, const int object, const string name, bool &value, string &reason, const string path) { const int node = d.FindMember(object, name); if(node < 0 || d.Type(node) != DPCC_JSON_BOOLEAN) { reason = path; return(false); } value = (d.Text(node) == "true"); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceShape | //+------------------------------------------------------------------+ bool DPCCDecodeResourceShape(CDPCCJsonDocument &d, const int node, SDPCCResourceShape &s, string &reason, const string path) { if(node < 0 || d.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } return(DPCCResourceReadLong(d, node, "K", s.K, reason, path + ".K") && DPCCResourceReadLong(d, node, "V", s.V, reason, path + ".V") && DPCCResourceReadLong(d, node, "H", s.H, reason, path + ".H") && DPCCResourceReadLong(d, node, "D", s.D, reason, path + ".D") && DPCCResourceReadLong(d, node, "M", s.M, reason, path + ".M") && DPCCResourceReadLong(d, node, "J", s.J, reason, path + ".J") && DPCCResourceReadLong(d, node, "E", s.E, reason, path + ".E") && DPCCResourceReadLong(d, node, "A", s.A, reason, path + ".A") && DPCCResourceReadLong(d, node, "B", s.B, reason, path + ".B") && DPCCResourceReadLong(d, node, "account_features", s.account_features, reason, path + ".account_features")); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceProfile | //+------------------------------------------------------------------+ bool DPCCDecodeResourceProfile(CDPCCJsonDocument &d, const int root, SDPCCResourceProfile &p, string &reason) { DPCCClearResourceProfile(p); reason = ""; const int r = d.FindMember(root, "resources"); if(r < 0 || d.Type(r) != DPCC_JSON_OBJECT) return(DPCCResourceProfileDecodeFail(p, reason, "resources")); if(!DPCCResourceReadString(d, root, "readiness_profile_id", p.readiness_profile_id, reason, "readiness_profile_id") || !DPCCResourceReadString(d, r, "calibration_state", p.calibration_state, reason, "resources.calibration_state") || !DPCCResourceReadLong(d, r, "frozen_at_utc", p.frozen_at_utc, reason, "resources.frozen_at_utc") || !DPCCResourceReadString(d, r, "profile_id", p.profile_id, reason, "resources.profile_id") || !DPCCResourceReadString(d, r, "device", p.device, reason, "resources.device") || !DPCCResourceReadString(d, r, "driver", p.driver, reason, "resources.driver") || !DPCCResourceReadString(d, r, "opencl_version", p.opencl_version, reason, "resources.opencl_version") || !DPCCResourceReadString(d, r, "terminal_build", p.terminal_build, reason, "resources.terminal_build") || !DPCCDecodeResourceShape(d, d.FindMember(r, "shapes"), p.shapes, reason, "resources.shapes") || !DPCCResourceReadLong(d, r, "projector_max_evaluations", p.projector_max_evaluations, reason, "resources.projector_max_evaluations") || !DPCCResourceReadLong(d, r, "projector_max_iterations", p.projector_max_iterations, reason, "resources.projector_max_iterations") || !DPCCResourceReadDouble(d, r, "p95_limit_ms", p.p95_limit_ms, reason, "resources.p95_limit_ms") || !DPCCResourceReadDouble(d, r, "p99_limit_ms", p.p99_limit_ms, reason, "resources.p99_limit_ms") || !DPCCResourceReadDouble(d, r, "max_compute_ms", p.max_compute_ms, reason, "resources.max_compute_ms") || !DPCCResourceReadDouble(d, r, "max_decision_age_ms", p.max_decision_age_ms, reason, "resources.max_decision_age_ms") || !DPCCResourceReadLong(d, r, "tracked_device_limit_bytes", p.tracked_device_limit_bytes, reason, "resources.tracked_device_limit_bytes") || !DPCCResourceReadLong(d, r, "total_device_limit_bytes", p.total_device_limit_bytes, reason, "resources.total_device_limit_bytes") || !DPCCResourceReadLong(d, r, "reserve_device_bytes", p.reserve_device_bytes, reason, "resources.reserve_device_bytes") || !DPCCResourceReadLong(d, r, "warmup_decisions", p.warmup_decisions, reason, "resources.warmup_decisions") || !DPCCResourceReadLong(d, r, "measured_decisions", p.measured_decisions, reason, "resources.measured_decisions") || !DPCCResourceReadLong(d, r, "repeats", p.repeats, reason, "resources.repeats") || !DPCCResourceReadString(d, r, "censored_policy", p.censored_policy, reason, "resources.censored_policy") || !DPCCResourceReadString(d, r, "algorithmic_limit", p.algorithmic_limit, reason, "resources.algorithmic_limit") || !DPCCResourceReadString(d, r, "stress_policy", p.stress_policy, reason, "resources.stress_policy") || !DPCCResourceReadString(d, r, "execution_program", p.execution_program, reason, "resources.execution_program")) return(DPCCResourceProfileDecodeFail(p, reason, reason)); const int cases = d.FindMember(r, "benchmark_cases"); if(cases < 0 || d.Type(cases) != DPCC_JSON_ARRAY || d.ChildCount(cases) <= 0) return(DPCCResourceProfileDecodeFail(p, reason, "resources.benchmark_cases")); ArrayResize(p.benchmark_cases, d.ChildCount(cases)); int child = d.FirstChild(cases); for(int i = 0; i < ArraySize(p.benchmark_cases); i++, child = d.NextSibling(child)) { if(child < 0 || d.Type(child) != DPCC_JSON_STRING) return(DPCCResourceProfileDecodeFail(p, reason, "resources.benchmark_cases")); p.benchmark_cases[i] = d.Text(child); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCParseGateVerdict | //+------------------------------------------------------------------+ bool DPCCParseGateVerdict(const string value, ENUM_DPCC_GATE_VERDICT &verdict) { if(value == "NOT_RUN") { verdict = DPCC_GATE_NOT_RUN; return(true); } if(value == "PASS") { verdict = DPCC_GATE_PASS; return(true); } if(value == "FAIL") { verdict = DPCC_GATE_FAIL; return(true); } if(value == "BLOCKED") { verdict = DPCC_GATE_BLOCKED; return(true); } return(false); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceReportClaims | //+------------------------------------------------------------------+ bool DPCCDecodeResourceReportClaims(CDPCCJsonDocument &d, const int root, SDPCCResourceReportClaims &claims, string &reason) { claims.evaluated_at_utc = 0; claims.pretrain_resources = DPCC_GATE_NOT_RUN; claims.full_resources = DPCC_GATE_NOT_RUN; reason = ""; if(!DPCCResourceReadLong(d, root, "evaluated_at_utc", claims.evaluated_at_utc, reason, "evaluated_at_utc")) return(false); const int pretrain = d.FindMember(root, "pretrain_resources"), full = d.FindMember(root, "full_resources"); string value = ""; if(pretrain < 0 || !DPCCResourceReadString(d, pretrain, "verdict", value, reason, "pretrain_resources.verdict") || !DPCCParseGateVerdict(value, claims.pretrain_resources)) { claims.evaluated_at_utc = 0; reason = "pretrain_resources.verdict"; return(false); } if(full < 0 || !DPCCResourceReadString(d, full, "verdict", value, reason, "full_resources.verdict") || !DPCCParseGateVerdict(value, claims.full_resources)) { claims.evaluated_at_utc = 0; claims.pretrain_resources = DPCC_GATE_NOT_RUN; reason = "full_resources.verdict"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCClearResourceReport | //+------------------------------------------------------------------+ void DPCCClearResourceReport(SDPCCResourceReport &report) { report.claims.evaluated_at_utc = 0; report.claims.pretrain_resources = DPCC_GATE_NOT_RUN; report.claims.full_resources = DPCC_GATE_NOT_RUN; report.profile_ref_id = ""; report.profile_ref_sha256 = ""; ArrayResize(report.rows, 0); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceMeasurement | //+------------------------------------------------------------------+ bool DPCCDecodeResourceMeasurement(CDPCCJsonDocument &d, const int m, SDPCCResourceMeasurement &v, string &reason) { DPCCClearResourceMeasurement(v); if(m < 0 || d.Type(m) != DPCC_JSON_OBJECT) { reason = "measurement"; return(false); } const bool success = (DPCCResourceReadString(d, m, "measurement_id", v.measurement_id, reason, "measurement.measurement_id") && DPCCResourceReadString(d, m, "stage", v.stage, reason, "measurement.stage") && DPCCResourceReadString(d, m, "profile_id", v.profile_id, reason, "measurement.profile_id") && DPCCResourceReadString(d, m, "benchmark_case", v.benchmark_case, reason, "measurement.benchmark_case") && DPCCResourceReadLong(d, m, "repeat_index", v.repeat_index, reason, "measurement.repeat_index") && DPCCResourceReadLong(d, m, "warmup_completed", v.warmup_completed, reason, "measurement.warmup_completed") && DPCCDecodeResourceShape(d, d.FindMember(m, "shapes"), v.shapes, reason, "measurement.shapes") && DPCCResourceReadLong(d, m, "projector_max_evaluations", v.projector_max_evaluations, reason, "measurement.projector_max_evaluations") && DPCCResourceReadLong(d, m, "projector_max_iterations", v.projector_max_iterations, reason, "measurement.projector_max_iterations") && DPCCResourceReadLong(d, m, "measured_started_at_utc", v.measured_started_at_utc, reason, "measurement.measured_started_at_utc") && DPCCResourceReadString(d, m, "device", v.device, reason, "measurement.device") && DPCCResourceReadString(d, m, "driver", v.driver, reason, "measurement.driver") && DPCCResourceReadString(d, m, "opencl_version", v.opencl_version, reason, "measurement.opencl_version") && DPCCResourceReadString(d, m, "terminal_build", v.terminal_build, reason, "measurement.terminal_build") && DPCCResourceReadString(d, m, "projection_mode", v.projection_mode, reason, "measurement.projection_mode") && DPCCResourceReadLong(d, m, "attempts", v.attempts, reason, "measurement.attempts") && DPCCResourceReadLong(d, m, "completed", v.completed, reason, "measurement.completed") && DPCCResourceReadLong(d, m, "censored", v.censored, reason, "measurement.censored") && DPCCResourceReadLong(d, m, "timeouts", v.timeouts, reason, "measurement.timeouts") && DPCCResourceReadLong(d, m, "stale_decisions", v.stale_decisions, reason, "measurement.stale_decisions") && DPCCResourceReadDouble(d, m, "p50_ms", v.p50_ms, reason, "measurement.p50_ms") && DPCCResourceReadDouble(d, m, "p95_ms", v.p95_ms, reason, "measurement.p95_ms") && DPCCResourceReadDouble(d, m, "p99_ms", v.p99_ms, reason, "measurement.p99_ms") && DPCCResourceReadDouble(d, m, "max_ms", v.max_ms, reason, "measurement.max_ms") && DPCCResourceReadDouble(d, m, "max_decision_age_ms", v.max_decision_age_ms, reason, "measurement.max_decision_age_ms") && DPCCResourceReadLong(d, m, "tracked_peak_bytes", v.tracked_peak_bytes, reason, "measurement.tracked_peak_bytes") && DPCCResourceReadLong(d, m, "total_peak_bytes", v.total_peak_bytes, reason, "measurement.total_peak_bytes") && DPCCResourceReadLong(d, m, "available_device_bytes", v.available_device_bytes, reason, "measurement.available_device_bytes") && DPCCResourceReadBool(d, m, "total_memory_telemetry_available", v.total_memory_telemetry_available, reason, "measurement.total_memory_telemetry_available") && DPCCResourceReadString(d, m, "execution_program", v.execution_program, reason, "measurement.execution_program")); if(!success) DPCCClearResourceMeasurement(v); return(success); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceReference | //+------------------------------------------------------------------+ bool DPCCDecodeResourceReference(CDPCCJsonDocument &d, const int node, SDPCCResourceReference &reference, string &reason, const string path) { reference.path = ""; reference.sha256 = ""; reference.id = ""; if(node < 0 || d.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } const bool success = (DPCCResourceReadString(d, node, "path", reference.path, reason, path + ".path") && DPCCResourceReadString(d, node, "sha256", reference.sha256, reason, path + ".sha256") && DPCCResourceReadString(d, node, "id", reference.id, reason, path + ".id")); if(!success) { reference.path = ""; reference.sha256 = ""; reference.id = ""; } return(success); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceReport | //+------------------------------------------------------------------+ bool DPCCDecodeResourceReport(CDPCCJsonDocument &d, const int root, SDPCCResourceReport &report, string &reason) { DPCCClearResourceReport(report); reason = ""; if(!DPCCDecodeResourceReportClaims(d, root, report.claims, reason)) return(false); const int profile_ref = d.FindMember(root, "readiness_profile"); SDPCCResourceReference profile_reference; if(!DPCCDecodeResourceReference(d, profile_ref, profile_reference, reason, "readiness_profile")) { DPCCClearResourceReport(report); return(false); } report.profile_ref_id = profile_reference.id; report.profile_ref_sha256 = profile_reference.sha256; const int rows = d.FindMember(root, "resource_measurements"); if(rows < 0 || d.Type(rows) != DPCC_JSON_ARRAY) { DPCCClearResourceReport(report); reason = "resource_measurements"; return(false); } const int row_count = d.ChildCount(rows); if(ArrayResize(report.rows, row_count) != row_count) { DPCCClearResourceReport(report); reason = "resource_measurements.capacity"; return(false); } int row_node = d.FirstChild(rows); for(int i = 0; i < row_count; i++, row_node = d.NextSibling(row_node)) { const string base = StringFormat("resource_measurements[%d]", i); if(row_node < 0 || !DPCCDecodeResourceMeasurement(d, row_node, report.rows[i].measurement, reason)) { DPCCClearResourceReport(report); reason = base + "." + reason; return(false); } const int refs = d.FindMember(row_node, "evidence"); if(refs < 0 || d.Type(refs) != DPCC_JSON_ARRAY) { DPCCClearResourceReport(report); reason = base + ".evidence"; return(false); } const int ref_count = d.ChildCount(refs); if(ArrayResize(report.rows[i].evidence, ref_count) != ref_count) { DPCCClearResourceReport(report); reason = base + ".evidence.capacity"; return(false); } int ref = d.FirstChild(refs); for(int j = 0; j < ref_count; j++, ref = d.NextSibling(ref)) if(!DPCCDecodeResourceReference(d, ref, report.rows[i].evidence[j], reason, base + StringFormat( ".evidence[%d]", j))) { DPCCClearResourceReport(report); return(false); } } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeResourceEvidence | //+------------------------------------------------------------------+ bool DPCCDecodeResourceEvidence(CDPCCJsonDocument &d, const int root, SDPCCResourceEvidence &e, string &reason) { DPCCClearResourceEvidence(e); reason = ""; if(!DPCCResourceReadString(d, root, "evidence_id", e.evidence_id, reason, "evidence_id")) return(DPCCResourceDecodeFail(e, reason, reason)); const int ref = d.FindMember(root, "readiness_profile"); if(ref < 0 || !DPCCResourceReadString(d, ref, "id", e.profile_ref_id, reason, "readiness_profile.id") || !DPCCResourceReadString(d, ref, "sha256", e.profile_ref_sha256, reason, "readiness_profile.sha256")) return(DPCCResourceDecodeFail(e, reason, reason)); if(!DPCCDecodeResourceMeasurement(d, d.FindMember(root, "measurement"), e.measurement, reason)) return(DPCCResourceDecodeFail(e, reason, reason)); const int samples = d.FindMember(root, "samples"); if(samples < 0 || d.Type(samples) != DPCC_JSON_ARRAY) return(DPCCResourceDecodeFail(e, reason, "samples")); ArrayResize(e.samples, d.ChildCount(samples)); int child = d.FirstChild(samples); for(int i = 0; i < ArraySize(e.samples); i++, child = d.NextSibling(child)) { if(child < 0 || !DPCCResourceReadDouble(d, child, "duration_ms", e.samples[i].duration_ms, reason, StringFormat( "samples[%d].duration_ms", i)) || !DPCCResourceReadDouble(d, child, "decision_age_ms", e.samples[i].decision_age_ms, reason, StringFormat( "samples[%d].decision_age_ms", i)) || !DPCCResourceReadBool(d, child, "censored", e.samples[i].censored, reason, StringFormat("samples[%d].censored", i)) || !DPCCResourceReadBool(d, child, "timeout", e.samples[i].timeout, reason, StringFormat("samples[%d].timeout", i)) || !DPCCResourceReadBool(d, child, "stale", e.samples[i].stale, reason, StringFormat("samples[%d].stale", i)) || !DPCCResourceReadLong(d, child, "tracked_bytes", e.samples[i].tracked_bytes, reason, StringFormat( "samples[%d].tracked_bytes", i)) || !DPCCResourceReadLong(d, child, "total_bytes", e.samples[i].total_bytes, reason, StringFormat( "samples[%d].total_bytes", i)) || !DPCCResourceReadLong(d, child, "available_bytes", e.samples[i].available_bytes, reason, StringFormat( "samples[%d].available_bytes", i))) return(DPCCResourceDecodeFail(e, reason, reason)); } return(true); } #endif #ifndef DPCC_JSON_SCHEMA_MQH #define DPCC_JSON_SCHEMA_MQH #define DPCC_JSON_SCHEMA_MAX_DEPTH 128 #define DPCC_JSON_SCHEMA_MAX_STEPS 1000000 //--- A deliberately bounded Draft 2020-12 subset for the closed DPCC v1.4 //--- metadata schemas. Unknown assertions fail as UNSUPPORTED_SCHEMA. class CDPCCJsonSchemaValidator { private: string m_reason; string m_path; int m_steps; string EscapePointer(const string value) { string result = ""; for(int i = 0; i < StringLen(value); i++) { ushort ch = StringGetCharacter(value, i); if(ch == 126) result += "~0"; else if(ch == 47) result += "~1"; else result += ShortToString(ch); } return(result); } bool Fail(const string reason, const string path) { if(m_reason == "") { m_reason = reason; m_path = path; } return(false); } bool Step(const string path) { m_steps++; if(m_steps > DPCC_JSON_SCHEMA_MAX_STEPS) return(Fail("BUDGET_EXCEEDED", path)); return(true); } int FindMemberBounded(CDPCCJsonDocument &doc, const int parent, const string name, const string path) { if(doc.Type(parent) != DPCC_JSON_OBJECT) return(-1); for(int child = doc.FirstChild(parent); child >= 0; child = doc.NextSibling(child)) { if(!Step(path)) return(-2); if(doc.Key(child) == name) return(child); } return(-1); } bool IsAllowedTypeName(const string name) { return(name == "object" || name == "array" || name == "string" || name == "number" || name == "integer" || name == "boolean" || name == "null"); } bool IsKnownKeyword(const string key) { return(key == "$schema" || key == "$id" || key == "title" || key == "description" || key == "type" || key == "const" || key == "enum" || key == "properties" || key == "required" || key == "additionalProperties" || key == "items" || key == "minItems" || key == "maxItems" || key == "minLength" || key == "maxLength" || key == "pattern" || key == "minimum" || key == "maximum" || key == "exclusiveMinimum" || key == "exclusiveMaximum" || key == "uniqueItems" || key == "anyOf"); } bool IsAnnotation(const string key) { return(key == "$schema" || key == "$id" || key == "title" || key == "description"); } bool IsJsonNumber(CDPCCJsonDocument &doc, const int node) { return(doc.Type(node) == DPCC_JSON_NUMBER); } bool IsIntegerLexeme(const string text) { string digits = ""; bool negative = false; long power = 0; if(!NormalizeNumber(text, digits, power, negative)) return(false); if(digits == "0") return(true); return(power >= 0); } bool NormalizeNumber(const string text, string &digits, long &power, bool &negative) { digits = ""; power = 0; negative = false; int length = StringLen(text); if(length <= 0) return(false); int pos = 0; if(StringGetCharacter(text, pos) == 45) { negative = true; pos++; } int fraction_digits = 0; bool in_fraction = false; while(pos < length) { ushort ch = StringGetCharacter(text, pos); if(ch == 46) { if(in_fraction) return(false); in_fraction = true; pos++; continue; } if(ch == 101 || ch == 69) break; if(ch < 48 || ch > 57) return(false); digits += ShortToString(ch); if(in_fraction) fraction_digits++; pos++; } long exponent = 0; if(pos < length) { pos++; bool exp_negative = false; if(pos < length && (StringGetCharacter(text, pos) == 43 || StringGetCharacter(text, pos) == 45)) { exp_negative = (StringGetCharacter(text, pos) == 45); pos++; } if(pos >= length) return(false); for(; pos < length; pos++) { ushort ch = StringGetCharacter(text, pos); if(ch < 48 || ch > 57) return(false); if(exponent > 1000000) return(false); exponent = exponent * 10 + (long)(ch - 48); } if(exponent > 1000000) return(false); if(exp_negative) exponent = -exponent; } power = exponent - (long)fraction_digits; int first = 0; while(first < StringLen(digits) && StringGetCharacter(digits, first) == 48) first++; if(first == StringLen(digits)) { digits = "0"; power = 0; negative = false; return(true); } if(first > 0) digits = StringSubstr(digits, first); while(StringLen(digits) > 1 && StringGetCharacter(digits, StringLen(digits) - 1) == 48) { digits = StringSubstr(digits, 0, StringLen(digits) - 1); power++; } return(true); } int StringCodePoints(const string value) { int count = 0; for(int i = 0; i < StringLen(value); i++) { ushort ch = StringGetCharacter(value, i); if(ch >= 0xD800 && ch <= 0xDBFF && i + 1 < StringLen(value)) { ushort next = StringGetCharacter(value, i + 1); if(next >= 0xDC00 && next <= 0xDFFF) i++; } count++; } return(count); } bool CompareNumberText(const string left, const string right, int &comparison) { string left_digits, right_digits; long left_power, right_power; bool left_negative, right_negative; if(!NormalizeNumber(left, left_digits, left_power, left_negative) || !NormalizeNumber(right, right_digits, right_power, right_negative)) return(false); if(left_digits == "0" && right_digits == "0") { comparison = 0; return(true); } if(left_digits == "0") { comparison = (right_negative ? 1 : -1); return(true); } if(right_digits == "0") { comparison = (left_negative ? -1 : 1); return(true); } if(left_negative != right_negative) { comparison = (left_negative ? -1 : 1); return(true); } long left_magnitude = (long)StringLen(left_digits) + left_power; long right_magnitude = (long)StringLen(right_digits) + right_power; if(left_magnitude != right_magnitude) comparison = (left_magnitude < right_magnitude ? -1 : 1); else { int length = MathMax(StringLen(left_digits), StringLen(right_digits)); comparison = 0; for(int i = 0; i < length; i++) { ushort lc = (i < StringLen(left_digits) ? StringGetCharacter(left_digits, i) : 48); ushort rc = (i < StringLen(right_digits) ? StringGetCharacter(right_digits, i) : 48); if(lc != rc) { comparison = (lc < rc ? -1 : 1); break; } } } if(left_negative) comparison = -comparison; return(true); } bool ReadBound(CDPCCJsonDocument &schema, const int schema_node, const string key, double &value, const bool integer_only, const string path) { int node = FindMemberBounded(schema, schema_node, key, path); if(node == -2) return(false); if(node < 0) return(true); if(!IsJsonNumber(schema, node) || !schema.TryNumber(node, value)) return(Fail("INVALID_SCHEMA", path + "/" + key)); if(integer_only && (!IsIntegerLexeme(schema.Text(node)) || value < 0.0 || value > 2147483647.0)) return(Fail("INVALID_SCHEMA", path + "/" + key)); return(true); } bool ValidateSchemaNode(CDPCCJsonDocument &schema, const int node, const string path, const int depth) { if(!Step(path)) return(false); if(depth > DPCC_JSON_SCHEMA_MAX_DEPTH) return(Fail("SCHEMA_DEPTH_LIMIT", path)); ENUM_DPCC_JSON_TYPE type = schema.Type(node); if(type == DPCC_JSON_BOOLEAN) return(true); if(type != DPCC_JSON_OBJECT) return(Fail("INVALID_SCHEMA", path)); for(int child = schema.FirstChild(node); child >= 0; child = schema.NextSibling(child)) { if(!Step(path)) return(false); string key = schema.Key(child); string keyword_path = path + "/" + EscapePointer(key); if(!IsKnownKeyword(key)) return(Fail("UNSUPPORTED_SCHEMA", keyword_path)); if(IsAnnotation(key)) { if(key == "$schema") { if(schema.Type(child) != DPCC_JSON_STRING || schema.Text(child) != "https://json-schema.org/draft/2020-12/schema") return(Fail("UNSUPPORTED_SCHEMA", keyword_path)); } else if(schema.Type(child) != DPCC_JSON_STRING) return(Fail("INVALID_SCHEMA", keyword_path)); continue; } if(key == "type") { if(schema.Type(child) == DPCC_JSON_STRING) { if(!IsAllowedTypeName(schema.Text(child))) return(Fail("INVALID_SCHEMA", keyword_path)); } else if(schema.Type(child) == DPCC_JSON_ARRAY) { if(schema.ChildCount(child) == 0) return(Fail("INVALID_SCHEMA", keyword_path)); int type_item = schema.FirstChild(child); for(int i = 0; type_item >= 0; type_item = schema.NextSibling(type_item), i++) { if(!Step(keyword_path)) return(false); int item = type_item; if(schema.Type(item) != DPCC_JSON_STRING || !IsAllowedTypeName(schema.Text(item))) return(Fail("INVALID_SCHEMA", keyword_path)); for(int prior = schema.FirstChild(child), j = 0; prior >= 0 && j < i; prior = schema.NextSibling( prior), j++) { if(!Step(keyword_path)) return(false); if(schema.Text(prior) == schema.Text(item)) return(Fail("INVALID_SCHEMA", keyword_path)); } } } else return(Fail("INVALID_SCHEMA", keyword_path)); } else if(key == "const") continue; else if(key == "enum") { if(schema.Type(child) != DPCC_JSON_ARRAY || schema.ChildCount(child) == 0) return(Fail("INVALID_SCHEMA", keyword_path)); int enum_item = schema.FirstChild(child); for(int i = 0; enum_item >= 0; enum_item = schema.NextSibling(enum_item), i++) { if(!Step(keyword_path)) return(false); for(int prior = schema.FirstChild(child), j = 0; prior >= 0 && j < i; prior = schema.NextSibling( prior), j++) { if(!Step(keyword_path)) return(false); if(ValueEqual(schema, prior, schema, enum_item, 0)) return(Fail("INVALID_SCHEMA", keyword_path)); if(m_reason == "BUDGET_EXCEEDED") return(false); } } } else if(key == "properties") { if(schema.Type(child) != DPCC_JSON_OBJECT) return(Fail("INVALID_SCHEMA", keyword_path)); for(int property = schema.FirstChild(child); property >= 0; property = schema.NextSibling( property)) { if(!Step(keyword_path)) return(false); if(!ValidateSchemaNode(schema, property, keyword_path + "/" + EscapePointer(schema.Key( property)), depth + 1)) return(false); } } else if(key == "required") { if(schema.Type(child) != DPCC_JSON_ARRAY) return(Fail("INVALID_SCHEMA", keyword_path)); int required_item = schema.FirstChild(child); for(int i = 0; required_item >= 0; required_item = schema.NextSibling(required_item), i++) { if(!Step(keyword_path)) return(false); int required = required_item; if(schema.Type(required) != DPCC_JSON_STRING) return(Fail("INVALID_SCHEMA", keyword_path)); for(int prior = schema.FirstChild(child), j = 0; prior >= 0 && j < i; prior = schema.NextSibling(prior), j++) { if(!Step(keyword_path)) return(false); if(schema.Text(prior) == schema.Text(required)) return(Fail("INVALID_SCHEMA", keyword_path)); } } } else if(key == "additionalProperties" || key == "items") { if(key == "additionalProperties" && (schema.Type(child) == DPCC_JSON_BOOLEAN)) continue; if(key == "items" && schema.Type(child) == DPCC_JSON_ARRAY) return(Fail("UNSUPPORTED_SCHEMA", keyword_path)); if(!ValidateSchemaNode(schema, child, keyword_path, depth + 1)) return(false); } else if(key == "minItems" || key == "maxItems" || key == "minLength" || key == "maxLength") { double value = 0.0; if(!ReadBound(schema, node, key, value, true, path)) return(false); } else if(key == "minimum" || key == "maximum" || key == "exclusiveMinimum" || key == "exclusiveMaximum") { double value = 0.0; if(!ReadBound(schema, node, key, value, false, path)) return(false); } else if(key == "pattern") { if(schema.Type(child) != DPCC_JSON_STRING) return(Fail("INVALID_SCHEMA", keyword_path)); string pattern = schema.Text(child); if(pattern != "^[0-9a-f]{16}$" && pattern != "^[0-9a-f]{40}$" && pattern != "^[0-9a-f]{64}$") return(Fail("UNSUPPORTED_SCHEMA", keyword_path)); } else if(key == "uniqueItems") { if(schema.Type(child) != DPCC_JSON_BOOLEAN) return(Fail("INVALID_SCHEMA", keyword_path)); } else if(key == "anyOf") { if(schema.Type(child) != DPCC_JSON_ARRAY || schema.ChildCount(child) == 0) return(Fail("INVALID_SCHEMA", keyword_path)); int branch = schema.FirstChild(child); for(int i = 0; branch >= 0; branch = schema.NextSibling(branch), i++) { if(!Step(keyword_path)) return(false); if(!ValidateSchemaNode(schema, branch, keyword_path + "/" + IntegerToString(i), depth + 1)) return(false); } } } return(true); } bool TypeMatches(CDPCCJsonDocument &instance, const int node, const string name) { ENUM_DPCC_JSON_TYPE actual = instance.Type(node); if(name == "object") return(actual == DPCC_JSON_OBJECT); if(name == "array") return(actual == DPCC_JSON_ARRAY); if(name == "string") return(actual == DPCC_JSON_STRING); if(name == "number") return(actual == DPCC_JSON_NUMBER); if(name == "integer") return(actual == DPCC_JSON_NUMBER && IsIntegerLexeme(instance.Text(node))); if(name == "boolean") return(actual == DPCC_JSON_BOOLEAN); if(name == "null") return(actual == DPCC_JSON_NULL); return(false); } bool ValueEqual(CDPCCJsonDocument &a, const int an, CDPCCJsonDocument &b, const int bn, const int depth) { if(depth > DPCC_JSON_SCHEMA_MAX_DEPTH || !Step("")) return(false); ENUM_DPCC_JSON_TYPE at = a.Type(an), bt = b.Type(bn); if(at != bt) return(false); if(at == DPCC_JSON_NULL) return(true); if(at == DPCC_JSON_BOOLEAN || at == DPCC_JSON_STRING) return(a.Text(an) == b.Text(bn)); if(at == DPCC_JSON_NUMBER) { int comparison = 0; if(!CompareNumberText(a.Text(an), b.Text(bn), comparison)) { Fail("BUDGET_EXCEEDED", ""); return(false); } return(comparison == 0); } if(at == DPCC_JSON_ARRAY) { if(a.ChildCount(an) != b.ChildCount(bn)) return(false); int left_child = a.FirstChild(an), right_child = b.FirstChild(bn); while(left_child >= 0 && right_child >= 0) { if(!Step("")) return(false); if(!ValueEqual(a, left_child, b, right_child, depth + 1)) return(false); left_child = a.NextSibling(left_child); right_child = b.NextSibling(right_child); } return(true); } if(at == DPCC_JSON_OBJECT) { if(a.ChildCount(an) != b.ChildCount(bn)) return(false); for(int member = a.FirstChild(an); member >= 0; member = a.NextSibling(member)) { if(!Step("")) return(false); int other = FindMemberBounded(b, bn, a.Key(member), ""); if(other == -2) return(false); if(other < 0 || !ValueEqual(a, member, b, other, depth + 1)) return(false); } return(true); } return(false); } bool PatternMatches(const string value, const string pattern) { int expected_length = 0; if(pattern == "^[0-9a-f]{16}$") expected_length = 16; else if(pattern == "^[0-9a-f]{40}$") expected_length = 40; else if(pattern == "^[0-9a-f]{64}$") expected_length = 64; else return(false); if(StringLen(value) != expected_length) return(false); for(int i = 0; i < expected_length; i++) { ushort ch = StringGetCharacter(value, i); if(!((ch >= 48 && ch <= 57) || (ch >= 97 && ch <= 102))) return(false); } return(true); } bool ValidateNode(CDPCCJsonDocument &instance, const int instance_node, CDPCCJsonDocument &schema, const int schema_node, const string path, const int depth) { if(!Step(path)) return(false); if(depth > DPCC_JSON_SCHEMA_MAX_DEPTH) return(Fail("DEPTH_LIMIT", path)); ENUM_DPCC_JSON_TYPE schema_type = schema.Type(schema_node); if(schema_type == DPCC_JSON_BOOLEAN) return(schema.Text(schema_node) == "true" || Fail("INSTANCE_FALSE_SCHEMA", path)); int keyword = FindMemberBounded(schema, schema_node, "type", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(keyword >= 0) { bool match = false; if(schema.Type(keyword) == DPCC_JSON_STRING) match = TypeMatches(instance, instance_node, schema.Text(keyword)); else for(int type_item = schema.FirstChild(keyword); type_item >= 0; type_item = schema.NextSibling(type_item)) { if(!Step(path)) return(false); if(TypeMatches(instance, instance_node, schema.Text(type_item))) match = true; } if(!match) return(Fail("INSTANCE_TYPE", path)); } keyword = FindMemberBounded(schema, schema_node, "const", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(keyword >= 0 && !ValueEqual(instance, instance_node, schema, keyword, 0)) return(Fail("INSTANCE_CONST", path)); if(m_reason == "BUDGET_EXCEEDED") return(false); keyword = FindMemberBounded(schema, schema_node, "enum", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(keyword >= 0) { bool found = false; for(int enum_item = schema.FirstChild(keyword); enum_item >= 0; enum_item = schema.NextSibling(enum_item)) { if(!Step(path)) return(false); if(ValueEqual(instance, instance_node, schema, enum_item, 0)) found = true; if(m_reason == "BUDGET_EXCEEDED") return(false); } if(!found) return(Fail("INSTANCE_ENUM", path)); } bool numeric = (instance.Type(instance_node) == DPCC_JSON_NUMBER); if(numeric) { int comparison = 0; int bound_node = FindMemberBounded(schema, schema_node, "minimum", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0) { if(!CompareNumberText(instance.Text(instance_node), schema.Text(bound_node), comparison)) return(Fail("BUDGET_EXCEEDED", path)); if(comparison < 0) return(Fail("INSTANCE_MINIMUM", path)); } bound_node = FindMemberBounded(schema, schema_node, "maximum", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0) { if(!CompareNumberText(instance.Text(instance_node), schema.Text(bound_node), comparison)) return(Fail("BUDGET_EXCEEDED", path)); if(comparison > 0) return(Fail("INSTANCE_MAXIMUM", path)); } bound_node = FindMemberBounded(schema, schema_node, "exclusiveMinimum", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0) { if(!CompareNumberText(instance.Text(instance_node), schema.Text(bound_node), comparison)) return(Fail("BUDGET_EXCEEDED", path)); if(comparison <= 0) return(Fail("INSTANCE_EXCLUSIVE_MINIMUM", path)); } bound_node = FindMemberBounded(schema, schema_node, "exclusiveMaximum", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0) { if(!CompareNumberText(instance.Text(instance_node), schema.Text(bound_node), comparison)) return(Fail("BUDGET_EXCEEDED", path)); if(comparison >= 0) return(Fail("INSTANCE_EXCLUSIVE_MAXIMUM", path)); } } if(instance.Type(instance_node) == DPCC_JSON_STRING) { int length = StringCodePoints(instance.Text(instance_node)); double bound = 0.0; int bound_node = FindMemberBounded(schema, schema_node, "minLength", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0 && schema.TryNumber(bound_node, bound) && length < (int)bound) return(Fail("INSTANCE_MIN_LENGTH", path)); bound_node = FindMemberBounded(schema, schema_node, "maxLength", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0 && schema.TryNumber(bound_node, bound) && length > (int)bound) return(Fail("INSTANCE_MAX_LENGTH", path)); bound_node = FindMemberBounded(schema, schema_node, "pattern", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0 && !PatternMatches(instance.Text(instance_node), schema.Text(bound_node))) return(Fail("INSTANCE_PATTERN", path)); } if(instance.Type(instance_node) == DPCC_JSON_ARRAY) { double bound = 0.0; int bound_node = FindMemberBounded(schema, schema_node, "minItems", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0 && schema.TryNumber(bound_node, bound) && instance.ChildCount(instance_node) < (int)bound) return(Fail("INSTANCE_MIN_ITEMS", path)); bound_node = FindMemberBounded(schema, schema_node, "maxItems", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(bound_node >= 0 && schema.TryNumber(bound_node, bound) && instance.ChildCount(instance_node) > (int)bound) return(Fail("INSTANCE_MAX_ITEMS", path)); int unique = FindMemberBounded(schema, schema_node, "uniqueItems", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(unique >= 0 && schema.Text(unique) == "true") { int unique_index = 0; for(int item = instance.FirstChild(instance_node); item >= 0; item = instance.NextSibling(item), unique_index++) { if(!Step(path)) return(false); for(int prior = instance.FirstChild(instance_node); prior >= 0 && prior != item; prior = instance.NextSibling(prior)) { if(!Step(path)) return(false); if(ValueEqual(instance, prior, instance, item, 0)) return(Fail("INSTANCE_UNIQUE_ITEMS", path + "/" + IntegerToString(unique_index))); if(m_reason == "BUDGET_EXCEEDED") return(false); } } } int item_schema = FindMemberBounded(schema, schema_node, "items", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(item_schema >= 0) { int item_index = 0; for(int item = instance.FirstChild(instance_node); item >= 0; item = instance.NextSibling(item), item_index++) { if(!Step(path)) return(false); if(!ValidateNode(instance, item, schema, item_schema, path + "/" + IntegerToString(item_index), depth + 1)) return(false); } } } if(instance.Type(instance_node) == DPCC_JSON_OBJECT) { int required = FindMemberBounded(schema, schema_node, "required", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(required >= 0) { for(int required_item = schema.FirstChild(required); required_item >= 0; required_item = schema.NextSibling( required_item)) { if(!Step(path)) return(false); string name = schema.Text(required_item); int found = FindMemberBounded(instance, instance_node, name, path); if(found == -2) return(false); if(found < 0) return(Fail("INSTANCE_REQUIRED", path + "/" + EscapePointer(name))); } } int properties = FindMemberBounded(schema, schema_node, "properties", path); if(m_reason == "BUDGET_EXCEEDED") return(false); int additional = FindMemberBounded(schema, schema_node, "additionalProperties", path); if(m_reason == "BUDGET_EXCEEDED") return(false); for(int member = instance.FirstChild(instance_node); member >= 0; member = instance.NextSibling(member)) { if(!Step(path)) return(false); string name = instance.Key(member); int property_schema = (properties >= 0 ? FindMemberBounded(schema, properties, name, path) : -1); if(property_schema == -2) return(false); if(property_schema >= 0) { if(!ValidateNode(instance, member, schema, property_schema, path + "/" + EscapePointer(name), depth + 1)) return(false); } else if(additional >= 0) { if(schema.Type(additional) == DPCC_JSON_BOOLEAN) { if(schema.Text(additional) == "false") return(Fail("INSTANCE_ADDITIONAL_PROPERTY", path + "/" + EscapePointer(name))); } else if(!ValidateNode(instance, member, schema, additional, path + "/" + EscapePointer(name), depth + 1)) return(false); } } } int any_of = FindMemberBounded(schema, schema_node, "anyOf", path); if(m_reason == "BUDGET_EXCEEDED") return(false); if(any_of >= 0) { bool matched = false; int branch_index = 0; for(int branch = schema.FirstChild(any_of); branch >= 0 && !matched; branch = schema.NextSibling(branch), branch_index++) { if(!Step(path)) return(false); string prior_reason = m_reason, prior_path = m_path; m_reason = ""; m_path = ""; matched = ValidateNode(instance, instance_node, schema, branch, path, depth + 1); string branch_reason = m_reason, branch_path = m_path; m_reason = prior_reason; m_path = prior_path; if(branch_reason == "BUDGET_EXCEEDED") return(Fail(branch_reason, branch_path)); } if(!matched) return(Fail("INSTANCE_ANY_OF", path)); } if(m_reason != "") return(false); return(true); } public: CDPCCJsonSchemaValidator(void) { m_reason = ""; m_path = ""; m_steps = 0; } bool Validate(CDPCCJsonDocument &instance, const int instance_node, CDPCCJsonDocument &schema, const int schema_node, string &reason, string &instance_path) { m_reason = ""; m_path = ""; m_steps = 0; reason = ""; instance_path = ""; if(instance.Type(instance_node) == DPCC_JSON_INVALID) { reason = "INVALID_INSTANCE"; return(false); } if(schema.Type(schema_node) == DPCC_JSON_INVALID) { reason = "INVALID_SCHEMA"; return(false); } if(schema.Type(schema_node) == DPCC_JSON_OBJECT) { int dialect = FindMemberBounded(schema, schema_node, "$schema", ""); if(dialect == -2) { reason = m_reason; instance_path = m_path; return(false); } if(dialect < 0 || schema.Type(dialect) != DPCC_JSON_STRING || schema.Text(dialect) != "https://json-schema.org/draft/2020-12/schema") { reason = "UNSUPPORTED_SCHEMA"; instance_path = "/$schema"; return(false); } } if(!ValidateSchemaNode(schema, schema_node, "", 0)) { reason = m_reason; instance_path = m_path; return(false); } if(!ValidateNode(instance, instance_node, schema, schema_node, "", 0)) { reason = m_reason; instance_path = m_path; return(false); } reason = ""; instance_path = ""; return(true); } }; #endif struct SDPCCCentralLinkResult { string model_id; string integration_id; string training_id; string profile_id; string model_path; string integration_path; string training_path; string profile_path; string model_sha256; string integration_sha256; string training_sha256; string profile_sha256; }; struct SDPCCCentralTypeIds { long decoder; long rollout; long projection; long denoiser; long dpcc; }; struct SDPCCCentralModel { bool example_only; string model_id; string repository; string base_commit; string local_changes[]; SDPCCResourceReference integration; SDPCCResourceReference training; SDPCCResourceReference profile; SDPCCCentralTypeIds type_ids; SDPCCResourceShape shapes; long projector_max_evaluations; long projector_max_iterations; }; struct SDPCCCentralIntegration { bool example_only; string integration_id; string repository; string base_commit; string local_changes[]; SDPCCCentralTypeIds type_ids; string component[]; string status[]; int evidence_count[]; string backup_state; long backup_created_at_utc; string backup_content_sha256; long backup_file_count; bool backup_index; bool backup_worktree; bool backup_untracked; string restore_check; bool restore_evidence_present; bool excluded_all_preserved; }; struct SDPCCCentralTraining { bool example_only; string training_id; SDPCCResourceReference profile; }; //+------------------------------------------------------------------+ //| Function DPCCClearCentralLinkResult | //+------------------------------------------------------------------+ void DPCCClearCentralLinkResult(SDPCCCentralLinkResult &result) { result.model_id = ""; result.integration_id = ""; result.training_id = ""; result.profile_id = ""; result.model_path = ""; result.integration_path = ""; result.training_path = ""; result.profile_path = ""; result.model_sha256 = ""; result.integration_sha256 = ""; result.training_sha256 = ""; result.profile_sha256 = ""; } //+------------------------------------------------------------------+ //| Function DPCCCentralFail | //+------------------------------------------------------------------+ bool DPCCCentralFail(SDPCCCentralLinkResult &result, string &reason, const string failure) { DPCCClearCentralLinkResult(result); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCCentralBool | //+------------------------------------------------------------------+ bool DPCCCentralBool(CDPCCJsonDocument &document, const int object, const string name, bool &value, string &reason, const string path) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_BOOLEAN) { reason = path; return(false); } value = (document.Text(node) == "true"); return(true); } //+------------------------------------------------------------------+ //| Function DPCCCentralObject | //+------------------------------------------------------------------+ bool DPCCCentralObject(CDPCCJsonDocument &document, const int parent, const string name, int &node, string &reason, const string path) { node = document.FindMember(parent, name); if(node < 0 || document.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCCentralStringArray | //+------------------------------------------------------------------+ bool DPCCCentralStringArray(CDPCCJsonDocument &document, const int object, const string name, string &values[], string &reason, const string path) { ArrayResize(values, 0); const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_ARRAY) { reason = path; return(false); } const int count = document.ChildCount(node); if(ArrayResize(values, count) != count) { reason = path + ".capacity"; return(false); } int child = document.FirstChild(node); for(int i = 0; i < count; i++, child = document.NextSibling(child)) { if(child < 0 || document.Type(child) != DPCC_JSON_STRING) { ArrayResize(values, 0); reason = path + ".item"; return(false); } values[i] = document.Text(child); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeCentralTypeIds | //+------------------------------------------------------------------+ bool DPCCDecodeCentralTypeIds(CDPCCJsonDocument &document, const int root, SDPCCCentralTypeIds &ids, string &reason, const string path) { int node = -1; if(!DPCCCentralObject(document, root, "type_ids", node, reason, path)) return(false); return(DPCCResourceReadLong(document, node, "CNeuronScenarioDecoderOCL", ids.decoder, reason, path + ".decoder") && DPCCResourceReadLong(document, node, "CNeuronScenarioRolloutOCL", ids.rollout, reason, path + ".rollout") && DPCCResourceReadLong(document, node, "CNeuronPlanProjectionOCL", ids.projection, reason, path + ".projection") && DPCCResourceReadLong(document, node, "CNeuronPlanDenoiserOCL", ids.denoiser, reason, path + ".denoiser") && DPCCResourceReadLong(document, node, "CNeuronDPCCOCL", ids.dpcc, reason, path + ".dpcc")); } //+------------------------------------------------------------------+ //| Function DPCCCentralTypeIdsEqual | //+------------------------------------------------------------------+ bool DPCCCentralTypeIdsEqual(const SDPCCCentralTypeIds &left, const SDPCCCentralTypeIds &right) { return(left.decoder == right.decoder && left.rollout == right.rollout && left.projection == right.projection && left.denoiser == right.denoiser && left.dpcc == right.dpcc); } //+------------------------------------------------------------------+ //| Function DPCCCentralTypeIdsUnique | //+------------------------------------------------------------------+ bool DPCCCentralTypeIdsUnique(const SDPCCCentralTypeIds &ids) { long values[5]; values[0] = ids.decoder; values[1] = ids.rollout; values[2] = ids.projection; values[3] = ids.denoiser; values[4] = ids.dpcc; for(int i = 0; i < 5; i++) for(int j = i + 1; j < 5; j++) if(values[i] == values[j]) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCCentralStringsEqual | //+------------------------------------------------------------------+ bool DPCCCentralStringsEqual(const string &left[], const string &right[]) { if(ArraySize(left) != ArraySize(right)) return(false); for(int i = 0; i < ArraySize(left); i++) if(left[i] != right[i]) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeCentralModel | //+------------------------------------------------------------------+ bool DPCCDecodeCentralModel(CDPCCJsonDocument &document, SDPCCCentralModel &model, string &reason) { const int root = document.Root(); int implementation = -1, training = -1, projection = -1; if(!DPCCCentralBool(document, root, "example_only", model.example_only, reason, "model.example_only") || !DPCCResourceReadString(document, root, "model_id", model.model_id, reason, "model.model_id") || !DPCCResourceReadString(document, root, "repository", model.repository, reason, "model.repository") || !DPCCCentralObject(document, root, "implementation", implementation, reason, "model.implementation") || !DPCCResourceReadString(document, implementation, "base_commit", model.base_commit, reason, "model.implementation.base_commit") || !DPCCCentralStringArray(document, implementation, "local_changes", model.local_changes, reason, "model.implementation.local_changes") || !DPCCDecodeResourceReference(document, document.FindMember(implementation, "integration_manifest"), model.integration, reason, "model.implementation.integration_manifest") || !DPCCDecodeCentralTypeIds(document, root, model.type_ids, reason, "model.type_ids") || !DPCCDecodeResourceShape(document, document.FindMember(root, "shapes"), model.shapes, reason, "model.shapes") || !DPCCCentralObject(document, root, "projection", projection, reason, "model.projection") || !DPCCResourceReadLong(document, projection, "max_evaluations", model.projector_max_evaluations, reason, "model.projection.max_evaluations") || !DPCCResourceReadLong(document, projection, "max_iterations", model.projector_max_iterations, reason, "model.projection.max_iterations") || !DPCCCentralObject(document, root, "training", training, reason, "model.training") || !DPCCDecodeResourceReference(document, document.FindMember(training, "training_config"), model.training, reason, "model.training.training_config") || !DPCCDecodeResourceReference(document, document.FindMember(root, "readiness_profile"), model.profile, reason, "model.readiness_profile")) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeCentralIntegration | //+------------------------------------------------------------------+ bool DPCCDecodeCentralIntegration(CDPCCJsonDocument &document, SDPCCCentralIntegration &integration, string &reason) { const int root = document.Root(); if(!DPCCCentralBool(document, root, "example_only", integration.example_only, reason, "integration.example_only") || !DPCCResourceReadString(document, root, "integration_id", integration.integration_id, reason, "integration.integration_id") || !DPCCResourceReadString(document, root, "repository", integration.repository, reason, "integration.repository") || !DPCCResourceReadString(document, root, "base_commit", integration.base_commit, reason, "integration.base_commit") || !DPCCCentralStringArray(document, root, "local_changes", integration.local_changes, reason, "integration.local_changes") || !DPCCDecodeCentralTypeIds(document, root, integration.type_ids, reason, "integration.type_ids")) return(false); const int rows = document.FindMember(root, "source_reconciliation"); if(rows < 0 || document.Type(rows) != DPCC_JSON_ARRAY) { reason = "integration.source_reconciliation"; return(false); } const int count = document.ChildCount(rows); if(ArrayResize(integration.component, count) != count || ArrayResize(integration.status, count) != count || ArrayResize(integration.evidence_count, count) != count) { reason = "integration.source_reconciliation.capacity"; return(false); } int row = document.FirstChild(rows); for(int i = 0; i < count; i++, row = document.NextSibling(row)) { if(row < 0 || document.Type(row) != DPCC_JSON_OBJECT || !DPCCResourceReadString(document, row, "component", integration.component[i], reason, "integration.source_reconciliation.component") || !DPCCResourceReadString(document, row, "status", integration.status[i], reason, "integration.source_reconciliation.status")) return(false); const int evidence = document.FindMember(row, "evidence"); if(evidence < 0 || document.Type(evidence) != DPCC_JSON_ARRAY) { reason = "integration.source_reconciliation.evidence"; return(false); } integration.evidence_count[i] = document.ChildCount(evidence); } int backup = -1; if(!DPCCCentralObject(document, root, "worktree_backup", backup, reason, "integration.worktree_backup") || !DPCCResourceReadString(document, backup, "state", integration.backup_state, reason, "integration.worktree_backup.state") || !DPCCResourceReadLong(document, backup, "created_at_utc", integration.backup_created_at_utc, reason, "integration.worktree_backup.created_at_utc") || !DPCCResourceReadString(document, backup, "content_sha256", integration.backup_content_sha256, reason, "integration.worktree_backup.content_sha256") || !DPCCResourceReadLong(document, backup, "file_count", integration.backup_file_count, reason, "integration.worktree_backup.file_count") || !DPCCCentralBool(document, backup, "includes_index_diff", integration.backup_index, reason, "integration.worktree_backup.includes_index_diff") || !DPCCCentralBool(document, backup, "includes_worktree_files", integration.backup_worktree, reason, "integration.worktree_backup.includes_worktree_files") || !DPCCCentralBool(document, backup, "includes_untracked", integration.backup_untracked, reason, "integration.worktree_backup.includes_untracked") || !DPCCResourceReadString(document, backup, "restore_check", integration.restore_check, reason, "integration.worktree_backup.restore_check")) return(false); const int restore = document.FindMember(backup, "restore_evidence"); integration.restore_evidence_present = (restore >= 0 && document.Type(restore) != DPCC_JSON_NULL); integration.excluded_all_preserved = true; const int excluded = document.FindMember(backup, "excluded_items"); if(excluded < 0 || document.Type(excluded) != DPCC_JSON_ARRAY) { reason = "integration.worktree_backup.excluded_items"; return(false); } for(int item = document.FirstChild(excluded); item >= 0; item = document.NextSibling(item)) { bool preserved = false; if(!DPCCCentralBool(document, item, "preserved_elsewhere", preserved, reason, "integration.worktree_backup.excluded_items.preserved_elsewhere")) return(false); if(!preserved) integration.excluded_all_preserved = false; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeCentralTraining | //+------------------------------------------------------------------+ bool DPCCDecodeCentralTraining(CDPCCJsonDocument &document, SDPCCCentralTraining &training, string &reason) { const int root = document.Root(); return(DPCCCentralBool(document, root, "example_only", training.example_only, reason, "training.example_only") && DPCCResourceReadString(document, root, "training_id", training.training_id, reason, "training.training_id") && DPCCDecodeResourceReference(document, document.FindMember(root, "readiness_profile"), training.profile, reason, "training.readiness_profile")); } //+------------------------------------------------------------------+ //| Function DPCCCentralSchemaValidate | //+------------------------------------------------------------------+ bool DPCCCentralSchemaValidate(CDPCCJsonDocument &document, const string source, const string schema_source, string &reason, const string role) { CDPCCJsonDocument schema; if(!document.Parse(source, reason)) { reason = role + ".json." + reason; return(false); } if(!schema.Parse(schema_source, reason)) { reason = role + ".schema." + reason; return(false); } string path = ""; CDPCCJsonSchemaValidator validator; if(!validator.Validate(document, document.Root(), schema, schema.Root(), reason, path)) { reason = role + ".schema_validation." + reason + "." + path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCCentralObservedSource | //+------------------------------------------------------------------+ bool DPCCCentralObservedSource(const string source, const string observed_sha256, string &reason, const string role) { uchar bytes[]; int count = StringToCharArray(source, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(count <= 0) { reason = role + ".bytes"; return(false); } if(bytes[count - 1] == 0) count--; if(ArrayResize(bytes, count) != count) { reason = role + ".bytes.capacity"; return(false); } string digest = ""; if(!DPCCSHA256Lower(bytes, digest, reason)) { reason = role + "." + reason; return(false); } if(digest != observed_sha256) { reason = role + ".observed_sha256"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralIntegration | //+------------------------------------------------------------------+ bool DPCCValidateCentralIntegration(const SDPCCCentralModel &model, const SDPCCCentralIntegration &integration, string &reason) { if(model.integration.id != integration.integration_id) reason = "model.integration.id"; else if(model.repository != integration.repository) reason = "model.integration.repository"; else if(model.base_commit != integration.base_commit) reason = "model.integration.base_commit"; else if(!DPCCCentralStringsEqual(model.local_changes, integration.local_changes)) reason = "model.integration.local_changes"; else if(!DPCCCentralTypeIdsEqual(model.type_ids, integration.type_ids)) reason = "model.integration.type_ids"; else if(!DPCCCentralTypeIdsUnique(model.type_ids)) reason = "model.type_ids.duplicate"; if(reason != "") return(false); const string required[8] = {"CNet", "CNeuronBaseOCL", "ScenarioForecast", "UShape", "CrossAttention", "RevINDenorm", "FQF_SRM", "AccountTransition" }; for(int r = 0; r < 8; r++) { int matches = 0; for(int i = 0; i < ArraySize(integration.component); i++) if(integration.component[i] == required[r]) matches++; if(matches != 1) { reason = "integration.source_reconciliation." + required[r]; return(false); } } if(ArraySize(integration.component) != 8) { reason = "integration.source_reconciliation.count"; return(false); } for(int i = 0; i < ArraySize(integration.status); i++) if((integration.status[i] == "MATCH" || integration.status[i] == "COMPATIBLE") && integration.evidence_count[i] <= 0) { reason = "integration.source_reconciliation.evidence"; return(false); } if(integration.backup_state == "VERIFIED") { if(integration.backup_created_at_utc <= 0 || integration.backup_file_count <= 0 || integration.backup_content_sha256 == "0000000000000000000000000000000000000000000000000000000000000000" || !integration.backup_index || !integration.backup_worktree || !integration.backup_untracked || integration.restore_check != "PASS" || !integration.restore_evidence_present || !integration.excluded_all_preserved) { reason = "integration.worktree_backup.verified"; return(false); } } return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralProfile | //+------------------------------------------------------------------+ bool DPCCValidateCentralProfile(const SDPCCCentralModel &model, const SDPCCCentralTraining &training, const SDPCCResourceProfile &profile, const SDPCCArchitectureContract &architecture, const string profile_model_id, const string profile_training_id, string &reason) { reason = ""; if(profile_model_id != model.model_id) reason = "profile.model_id"; else if(profile_training_id != training.training_id) reason = "profile.training_id"; else if(profile.execution_program != architecture.model_parallel.program) reason = "profile.resources.execution_program"; else if(profile.shapes.K != model.shapes.K) reason = "profile.resources.shapes.K"; else if(profile.shapes.V != model.shapes.V) reason = "profile.resources.shapes.V"; else if(profile.shapes.H != model.shapes.H) reason = "profile.resources.shapes.H"; else if(profile.shapes.D != model.shapes.D) reason = "profile.resources.shapes.D"; else if(profile.shapes.M != model.shapes.M) reason = "profile.resources.shapes.M"; else if(profile.shapes.J != model.shapes.J) reason = "profile.resources.shapes.J"; else if(profile.shapes.E != model.shapes.E) reason = "profile.resources.shapes.E"; else if(profile.shapes.A != model.shapes.A) reason = "profile.resources.shapes.A"; else if(profile.shapes.B != model.shapes.B) reason = "profile.resources.shapes.B"; else if(profile.shapes.account_features != model.shapes.account_features) reason = "profile.resources.shapes.account_features"; else if(profile.projector_max_evaluations != model.projector_max_evaluations) reason = "profile.resources.projector_max_evaluations"; else if(profile.projector_max_iterations != model.projector_max_iterations) reason = "profile.resources.projector_max_iterations"; return(reason == ""); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralTyped | //+------------------------------------------------------------------+ bool DPCCValidateCentralTyped(const SDPCCCentralModel &model, const SDPCCCentralIntegration &integration, const SDPCCCentralTraining &training, const SDPCCResourceProfile &profile, const SDPCCArchitectureContract &architecture, const bool profile_example, const string profile_model_id, const string profile_training_id, const bool production, const string integration_sha256, const string training_sha256, const string profile_sha256, string &reason) { reason = ""; if(production && (model.example_only || integration.example_only || training.example_only || profile_example)) { reason = "central.example_only"; return(false); } if(model.integration.sha256 != integration_sha256) { reason = "model.integration.sha256"; return(false); } if(model.training.sha256 != training_sha256) { reason = "model.training.sha256"; return(false); } if(model.profile.sha256 != profile_sha256) { reason = "model.profile.sha256"; return(false); } if(!DPCCValidateCentralIntegration(model, integration, reason)) return(false); if(model.training.id != training.training_id) { reason = "model.training.id"; return(false); } if(model.profile.id != profile.readiness_profile_id) { reason = "model.profile.id"; return(false); } if(training.profile.id != model.profile.id || training.profile.sha256 != model.profile.sha256) { reason = "training.readiness_profile"; return(false); } if(!DPCCValidateArchitectureContract(architecture, architecture.model_denoiser.graph_sha256, reason)) return(false); return(DPCCValidateCentralProfile(model, training, profile, architecture, profile_model_id, profile_training_id, reason)); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralDocuments | //+------------------------------------------------------------------+ bool DPCCValidateCentralDocuments(CDPCCJsonDocument &model_document, CDPCCJsonDocument &integration_document, CDPCCJsonDocument &training_document, CDPCCJsonDocument &profile_document, const bool production, const string model_sha256, const string integration_sha256, const string training_sha256, const string profile_sha256, SDPCCCentralLinkResult &result, string &reason) { DPCCClearCentralLinkResult(result); reason = ""; SDPCCCentralModel model; SDPCCCentralIntegration integration; SDPCCCentralTraining training; SDPCCResourceProfile profile; if(!DPCCDecodeCentralModel(model_document, model, reason) || !DPCCDecodeCentralIntegration(integration_document, integration, reason) || !DPCCDecodeCentralTraining(training_document, training, reason) || !DPCCDecodeResourceProfile(profile_document, profile_document.Root(), profile, reason)) return(DPCCCentralFail(result, reason, reason)); bool profile_example = false; string profile_model_id = "", profile_training_id = ""; if(!DPCCCentralBool(profile_document, profile_document.Root(), "example_only", profile_example, reason, "profile.example_only") || !DPCCResourceReadString(profile_document, profile_document.Root(), "model_id", profile_model_id, reason, "profile.model_id") || !DPCCResourceReadString(profile_document, profile_document.Root(), "training_id", profile_training_id, reason, "profile.training_id")) return(DPCCCentralFail(result, reason, reason)); SDPCCArchitectureContract architecture; const int composition = model_document.FindMember(model_document.Root(), "composition"); const int denoiser = model_document.FindMember(composition, "denoiser"); string graph_sha = ""; if(!DPCCResourceReadString(model_document, denoiser, "graph_sha256", graph_sha, reason, "model.composition.denoiser.graph_sha256") || !DPCCDecodeArchitectureContract(model_document, model_document.Root(), training_document, training_document.Root(), graph_sha, architecture, reason)) return(DPCCCentralFail(result, reason, reason)); if(!DPCCValidateCentralTyped(model, integration, training, profile, architecture, profile_example, profile_model_id, profile_training_id, production, integration_sha256, training_sha256, profile_sha256, reason)) return(DPCCCentralFail(result, reason, reason)); result.model_id = model.model_id; result.integration_id = integration.integration_id; result.training_id = training.training_id; result.profile_id = profile.readiness_profile_id; result.model_sha256 = model_sha256; result.integration_sha256 = integration_sha256; result.training_sha256 = training_sha256; result.profile_sha256 = profile_sha256; return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralSources | //+------------------------------------------------------------------+ bool DPCCValidateCentralSources(const string model_source, const string integration_source, const string training_source, const string profile_source, const string model_schema, const string integration_schema, const string training_schema, const string profile_schema, const bool production, const string model_sha256, const string integration_sha256, const string training_sha256, const string profile_sha256, SDPCCCentralLinkResult &result, string &reason) { DPCCClearCentralLinkResult(result); reason = ""; if(!DPCCCentralObservedSource(model_source, model_sha256, reason, "model") || !DPCCCentralObservedSource(integration_source, integration_sha256, reason, "integration") || !DPCCCentralObservedSource(training_source, training_sha256, reason, "training") || !DPCCCentralObservedSource(profile_source, profile_sha256, reason, "profile")) return(false); CDPCCJsonDocument model_document, integration_document, training_document, profile_document; if(!DPCCCentralSchemaValidate(model_document, model_source, model_schema, reason, "model") || !DPCCCentralSchemaValidate(integration_document, integration_source, integration_schema, reason, "integration") || !DPCCCentralSchemaValidate(training_document, training_source, training_schema, reason, "training") || !DPCCCentralSchemaValidate(profile_document, profile_source, profile_schema, reason, "profile")) return(false); return(DPCCValidateCentralDocuments(model_document, integration_document, training_document, profile_document, production, model_sha256, integration_sha256, training_sha256, profile_sha256, result, reason)); } //+------------------------------------------------------------------+ //| Function DPCCCentralBytesSource | //+------------------------------------------------------------------+ bool DPCCCentralBytesSource(const uchar &bytes[], string &source, string &reason, const string role) { if(ArraySize(bytes) <= 0) { reason = role + ".bytes.empty"; return(false); } source = CharArrayToString(bytes, 0, WHOLE_ARRAY, CP_UTF8); if(source == "") { reason = role + ".bytes.utf8"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateCentralPaths | //+------------------------------------------------------------------+ bool DPCCValidateCentralPaths(const string model_path, const string integration_path, const string training_path, const string profile_path, string &reason) { reason = ""; if(model_path == integration_path || model_path == training_path || model_path == profile_path) reason = "central.path.cycle"; else if(integration_path == training_path || integration_path == profile_path || training_path == profile_path) reason = "central.path.role_conflict"; return(reason == ""); } //+------------------------------------------------------------------+ //| Function DPCCResolveCentralMetadata | //+------------------------------------------------------------------+ bool DPCCResolveCentralMetadata(const string package_root_relative, const string model_package_path, const bool common_files, const bool production, const string caller_observed_model_sha256, const string model_schema, const string integration_schema, const string training_schema, const string profile_schema, SDPCCCentralLinkResult &result, string &reason) { DPCCClearCentralLinkResult(result); reason = ""; string model_path = ""; if(!DPCCNormalizePackagePath(package_root_relative, "root.json", model_package_path, model_path, reason)) return(false); uchar model_bytes[]; if(!DPCCReadArtifactBytes(model_path, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, model_bytes, reason)) return(false); string model_sha = ""; if(!DPCCSHA256Lower(model_bytes, model_sha, reason) || model_sha != caller_observed_model_sha256) return(DPCCCentralFail(result, reason, "model.observed_sha256")); string model_source = ""; if(!DPCCCentralBytesSource(model_bytes, model_source, reason, "model")) return(false); CDPCCJsonDocument model_document; if(!DPCCCentralSchemaValidate(model_document, model_source, model_schema, reason, "model")) return(false); SDPCCCentralModel model; if(!DPCCDecodeCentralModel(model_document, model, reason)) return(false); SDPCCArtifactResolveResult integration_artifact, training_artifact, profile_artifact; if(!DPCCResolveArtifact(package_root_relative, model_package_path, model.integration, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, integration_artifact, reason) || !DPCCResolveArtifact(package_root_relative, model_package_path, model.training, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, training_artifact, reason) || !DPCCResolveArtifact(package_root_relative, model_package_path, model.profile, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, profile_artifact, reason)) return(DPCCCentralFail(result, reason, reason)); if(!DPCCValidateCentralPaths(model_path, integration_artifact.canonical_path, training_artifact.canonical_path, profile_artifact.canonical_path, reason)) return(DPCCCentralFail(result, reason, reason)); string integration_source = "", training_source = "", profile_source = ""; if(!DPCCCentralBytesSource(integration_artifact.bytes, integration_source, reason, "integration") || !DPCCCentralBytesSource(training_artifact.bytes, training_source, reason, "training") || !DPCCCentralBytesSource(profile_artifact.bytes, profile_source, reason, "profile")) return(DPCCCentralFail(result, reason, reason)); if(!DPCCValidateCentralSources(model_source, integration_source, training_source, profile_source, model_schema, integration_schema, training_schema, profile_schema, production, model_sha, integration_artifact.observed_sha256, training_artifact.observed_sha256, profile_artifact.observed_sha256, result, reason)) return(false); result.model_path = model_path; result.integration_path = integration_artifact.canonical_path; result.training_path = training_artifact.canonical_path; result.profile_path = profile_artifact.canonical_path; return(true); } #endif //+------------------------------------------------------------------+ //| DataReadinessMetadata.mqh: JSON adapters for SEM-11 | //+------------------------------------------------------------------+ #ifndef DPCC_DATA_READINESS_METADATA_MQH #define DPCC_DATA_READINESS_METADATA_MQH //+------------------------------------------------------------------+ //| DataReadiness.mqh: typed DPCC data-readiness core | //+------------------------------------------------------------------+ #ifndef DPCC_DATA_READINESS_MQH #define DPCC_DATA_READINESS_MQH #define DPCC_DATA_CATEGORY_COUNT 7 enum ENUM_DPCC_DATA_CATEGORY { DPCC_DATA_OPEN = 0, DPCC_DATA_HOLD = 1, DPCC_DATA_REDUCE = 2, DPCC_DATA_ADD = 3, DPCC_DATA_CLOSE = 4, DPCC_DATA_REVERSE = 5, DPCC_DATA_FLAT = 6 }; struct SDPCCDataRareThreshold { string event; long min_count; string zero_reason; }; struct SDPCCDataRareObservation { string event; long count; }; struct SDPCCDataQualityPolicy { long min_valid_windows; double max_invalid_fraction; long min_category_counts[DPCC_DATA_CATEGORY_COUNT]; bool min_category_present[DPCC_DATA_CATEGORY_COUNT]; string zero_minimum_justifications[DPCC_DATA_CATEGORY_COUNT]; SDPCCDataRareThreshold rare_event_min_counts[]; string result_selection_scope; bool validation_test_profit_filter; bool mandatory_losing_fraction; bool counts_before_after_required; }; struct SDPCCDataProfile { bool example_only; string readiness_profile_id, dataset_id, training_id; SDPCCResourceReference dataset_manifest, rho_implementation; string forecast_id; SDPCCDataQualityPolicy quality; }; struct SDPCCDataObservation { bool valid_windows_present, total_windows_present, counts_present, rare_event_counts_present; long valid_windows, total_windows; bool category_present[DPCC_DATA_CATEGORY_COUNT]; long category_counts[DPCC_DATA_CATEGORY_COUNT]; SDPCCDataRareObservation rare_event_counts[]; }; struct SDPCCDataSplit { SDPCCDataObservation before, after; }; struct SDPCCDataEvidence { bool example_only; string evidence_id; SDPCCResourceReference readiness_profile, dataset_manifest, bank_manifest, training_config, rho_implementation; string forecast_id, normalization_id; bool sources_match_present, rho_source_verified_present, filtered_validation_test_present, observed_at_utc_present; bool sources_match, rho_source_verified, filtered_validation_test; long observed_at_utc; SDPCCDataSplit fit, validation, test; }; struct SDPCCDataBindings { string readiness_profile_id, readiness_profile_sha256; string dataset_id, dataset_sha256; string bank_id, bank_sha256; string training_id, training_sha256; string rho_id, rho_sha256; string forecast_id, normalization_id; }; struct SDPCCDataReport { bool example_only; long evaluated_at_utc; ENUM_DPCC_GATE_VERDICT declared_data; SDPCCResourceReference readiness_profile; SDPCCResourceReference data_evidence[]; bool source_distribution_present, rho_extraction_present; SDPCCResourceReference source_distribution_report, rho_extraction_report; SDPCCResourceReference validated_inputs[]; }; //+------------------------------------------------------------------+ //| Function DPCCDataCategoryName | //+------------------------------------------------------------------+ string DPCCDataCategoryName(const int index) { switch(index) { case DPCC_DATA_OPEN: return("OPEN"); case DPCC_DATA_HOLD: return("HOLD"); case DPCC_DATA_REDUCE: return("REDUCE"); case DPCC_DATA_ADD: return("ADD"); case DPCC_DATA_CLOSE: return("CLOSE"); case DPCC_DATA_REVERSE: return("REVERSE"); case DPCC_DATA_FLAT: return("FLAT"); } return(""); } //+------------------------------------------------------------------+ //| Function DPCCDataCategoryIndex | //+------------------------------------------------------------------+ int DPCCDataCategoryIndex(const string name) { for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) if(DPCCDataCategoryName(i) == name) return(i); return(-1); } //+------------------------------------------------------------------+ //| Function DPCCDataFinite | //+------------------------------------------------------------------+ bool DPCCDataFinite(const double value) { return(MathIsValidNumber(value) && value > -DBL_MAX && value < DBL_MAX); } //+------------------------------------------------------------------+ //| Function DPCCDataNonemptyText | //+------------------------------------------------------------------+ bool DPCCDataNonemptyText(const string value) { string copy = value; StringTrimLeft(copy); StringTrimRight(copy); return(StringLen(copy) > 0); } //+------------------------------------------------------------------+ //| Function DPCCDataLowerSha256 | //+------------------------------------------------------------------+ bool DPCCDataLowerSha256(const string value) { if(StringLen(value) != 64) return(false); for(int i = 0; i < 64; i++) { const ushort ch = StringGetCharacter(value, i); if(!((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'))) return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateDataPolicy | //+------------------------------------------------------------------+ bool DPCCValidateDataPolicy(const SDPCCDataProfile &profile, string &reason) { reason = ""; const SDPCCDataQualityPolicy policy = profile.quality; if(policy.min_valid_windows < 0) { reason = "data_quality.min_valid_windows"; return(false); } if(!DPCCDataFinite(policy.max_invalid_fraction) || policy.max_invalid_fraction < 0.0 || policy.max_invalid_fraction > 1.0) { reason = "data_quality.max_invalid_fraction"; return(false); } for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) { const string name = DPCCDataCategoryName(i); if(!policy.min_category_present[i]) { reason = "data_quality.min_category_counts." + name; return(false); } if(policy.min_category_counts[i] < 0) { reason = "data_quality.min_category_counts." + name; return(false); } if(policy.min_category_counts[i] == 0 && !DPCCDataNonemptyText(policy.zero_minimum_justifications[i])) { reason = "data_quality.zero_minimum_justifications." + name; return(false); } } for(int i = 0; i < ArraySize(policy.rare_event_min_counts); i++) { const SDPCCDataRareThreshold threshold = policy.rare_event_min_counts[i]; if(!DPCCDataNonemptyText(threshold.event) || threshold.min_count < 0) { reason = "data_quality.rare_event_min_counts"; return(false); } if(threshold.min_count == 0 && !DPCCDataNonemptyText(threshold.zero_reason)) { reason = "data_quality.rare_event_min_counts.zero_reason"; return(false); } for(int j = 0; j < i; j++) if(policy.rare_event_min_counts[j].event == threshold.event) { reason = "data_quality.rare_event_min_counts.duplicate"; return(false); } } if(policy.result_selection_scope != "FIT_ONLY") { reason = "data_quality.result_selection_scope"; return(false); } if(policy.validation_test_profit_filter) { reason = "data_quality.validation_test_profit_filter"; return(false); } if(policy.mandatory_losing_fraction) { reason = "data_quality.mandatory_losing_fraction"; return(false); } if(!policy.counts_before_after_required) { reason = "data_quality.counts_before_after_required"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataFindRare | //+------------------------------------------------------------------+ int DPCCDataFindRare(const SDPCCDataObservation &observation, const string event) { for(int i = 0; i < ArraySize(observation.rare_event_counts); i++) if(observation.rare_event_counts[i].event == event) return(i); return(-1); } //+------------------------------------------------------------------+ //| Function DPCCDataObservationHasDuplicateRare | //+------------------------------------------------------------------+ bool DPCCDataObservationHasDuplicateRare(const SDPCCDataObservation &observation) { for(int i = 0; i < ArraySize(observation.rare_event_counts); i++) for(int j = 0; j < i; j++) if(observation.rare_event_counts[i].event == observation.rare_event_counts[j].event) return(true); return(false); } //+------------------------------------------------------------------+ //| Function DPCCDataObservationContainersPresent | //+------------------------------------------------------------------+ bool DPCCDataObservationContainersPresent(const SDPCCDataObservation &observation, string &reason) { if(!observation.valid_windows_present || !observation.total_windows_present || !observation.counts_present || !observation.rare_event_counts_present) { reason = "data.observation.missing"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataObservationMembersPresent | //+------------------------------------------------------------------+ bool DPCCDataObservationMembersPresent(const SDPCCDataQualityPolicy &policy, const SDPCCDataObservation &observation, string &reason) { for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) if(!observation.category_present[i]) { reason = "data.counts." + DPCCDataCategoryName(i); return(false); } for(int i = 0; i < ArraySize(policy.rare_event_min_counts); i++) if(DPCCDataFindRare(observation, policy.rare_event_min_counts[i].event) < 0) { reason = "data.rare_event_counts." + policy.rare_event_min_counts[i].event; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataObservationNumeric | //+------------------------------------------------------------------+ bool DPCCDataObservationNumeric(const SDPCCDataObservation &observation, string &reason) { if(observation.valid_windows < 0 || observation.total_windows < 0) { reason = "data.windows.numeric"; return(false); } if(observation.valid_windows > observation.total_windows) { reason = "data.windows.order"; return(false); } for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) if(observation.category_counts[i] < 0) { reason = "data.counts." + DPCCDataCategoryName(i) + ".numeric"; return(false); } for(int i = 0; i < ArraySize(observation.rare_event_counts); i++) if(observation.rare_event_counts[i].count < 0) { reason = "data.rare_event_counts.numeric"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCEvaluateDataObservation | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCEvaluateDataObservation(const SDPCCDataQualityPolicy &policy, const SDPCCDataObservation &observation, const bool sources_match_present, const bool rho_source_verified_present, const bool filtered_validation_test_present, const bool sources_match, const bool rho_source_verified, const bool filtered_validation_test, string &reason) { reason = ""; if(!sources_match_present || !rho_source_verified_present || !filtered_validation_test_present || !DPCCDataObservationContainersPresent(observation, reason)) { reason = "data.root.missing"; return(DPCC_GATE_BLOCKED); } if(!sources_match || !rho_source_verified) { reason = "data.provenance"; return(DPCC_GATE_BLOCKED); } if(filtered_validation_test) { reason = "data.filtered_validation_test"; return(DPCC_GATE_FAIL); } if(!DPCCDataObservationMembersPresent(policy, observation, reason)) return(DPCC_GATE_BLOCKED); if(!DPCCDataObservationNumeric(observation, reason)) return(DPCC_GATE_FAIL); if(observation.total_windows <= 0) { reason = "data.total_windows"; return(DPCC_GATE_FAIL); } if(observation.valid_windows < policy.min_valid_windows) { reason = "data.valid_windows"; return(DPCC_GATE_FAIL); } const double invalid_fraction = 1.0 - (double)observation.valid_windows / (double)observation.total_windows; if(invalid_fraction > policy.max_invalid_fraction) { reason = "data.invalid_fraction"; return(DPCC_GATE_FAIL); } for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) if(observation.category_counts[i] < policy.min_category_counts[i]) { reason = "data.counts." + DPCCDataCategoryName(i); return(DPCC_GATE_FAIL); } for(int i = 0; i < ArraySize(policy.rare_event_min_counts); i++) { const int found = DPCCDataFindRare(observation, policy.rare_event_min_counts[i].event); if(observation.rare_event_counts[found].count < policy.rare_event_min_counts[i].min_count) { reason = "data.rare_event_counts." + policy.rare_event_min_counts[i].event; return(DPCC_GATE_FAIL); } } return(DPCC_GATE_PASS); } //+------------------------------------------------------------------+ //| Function DPCCDataObservationEqual | //+------------------------------------------------------------------+ bool DPCCDataObservationEqual(const SDPCCDataObservation &left, const SDPCCDataObservation &right) { if(left.valid_windows_present != right.valid_windows_present || left.total_windows_present != right.total_windows_present || left.counts_present != right.counts_present || left.rare_event_counts_present != right.rare_event_counts_present || left.valid_windows != right.valid_windows || left.total_windows != right.total_windows) return(false); for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) if(left.category_present[i] != right.category_present[i] || left.category_counts[i] != right.category_counts[i]) return(false); if(ArraySize(left.rare_event_counts) != ArraySize(right.rare_event_counts)) return(false); for(int i = 0; i < ArraySize(left.rare_event_counts); i++) if(left.rare_event_counts[i].event != right.rare_event_counts[i].event || left.rare_event_counts[i].count != right.rare_event_counts[i].count) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataReferenceMatches | //+------------------------------------------------------------------+ bool DPCCDataReferenceMatches(const SDPCCResourceReference &reference, const string expected_id, const string expected_sha256, string &reason, const string path) { if(!DPCCDataLowerSha256(reference.sha256) || !DPCCDataLowerSha256(expected_sha256)) { reason = path + ".sha256"; return(false); } if(reference.id != expected_id) { reason = path + ".id"; return(false); } if(reference.sha256 != expected_sha256) { reason = path + ".sha256"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBindDataEvidence | //+------------------------------------------------------------------+ bool DPCCBindDataEvidence(const SDPCCDataProfile &profile, const SDPCCDataEvidence &evidence, const SDPCCDataBindings &bindings, string &reason) { reason = ""; if(profile.readiness_profile_id != bindings.readiness_profile_id) { reason = "profile.readiness_profile_id"; return(false); } if(profile.dataset_id != bindings.dataset_id || profile.dataset_manifest.id != bindings.dataset_id || profile.dataset_manifest.sha256 != bindings.dataset_sha256) { reason = "profile.dataset_manifest"; return(false); } if(profile.training_id != bindings.training_id) { reason = "profile.training_id"; return(false); } if(profile.forecast_id != bindings.forecast_id || profile.rho_implementation.id != bindings.rho_id || profile.rho_implementation.sha256 != bindings.rho_sha256) { reason = "profile.rho_provenance"; return(false); } if(!DPCCDataReferenceMatches(evidence.readiness_profile, bindings.readiness_profile_id, bindings.readiness_profile_sha256, reason, "readiness_profile") || !DPCCDataReferenceMatches(evidence.dataset_manifest, bindings.dataset_id, bindings.dataset_sha256, reason, "dataset_manifest") || !DPCCDataReferenceMatches(evidence.bank_manifest, bindings.bank_id, bindings.bank_sha256, reason, "bank_manifest") || !DPCCDataReferenceMatches(evidence.training_config, bindings.training_id, bindings.training_sha256, reason, "training_config") || !DPCCDataReferenceMatches(evidence.rho_implementation, bindings.rho_id, bindings.rho_sha256, reason, "rho_implementation")) return(false); if(evidence.forecast_id != bindings.forecast_id) { reason = "forecast_id"; return(false); } if(evidence.normalization_id != bindings.normalization_id) { reason = "normalization_id"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCEvaluateDataEvidence | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCEvaluateDataEvidence(const SDPCCDataProfile &profile, const SDPCCDataEvidence &evidence, const SDPCCDataBindings &bindings, string &reason) { reason = ""; if(!DPCCValidateDataPolicy(profile, reason)) return(DPCC_GATE_BLOCKED); if(!DPCCBindDataEvidence(profile, evidence, bindings, reason)) return(DPCC_GATE_BLOCKED); if(!evidence.sources_match_present || !evidence.rho_source_verified_present || !evidence.filtered_validation_test_present || !evidence.observed_at_utc_present) { reason = "data.root.missing"; return(DPCC_GATE_BLOCKED); } if(!DPCCDataObservationContainersPresent(evidence.fit.before, reason) || !DPCCDataObservationContainersPresent(evidence.fit.after, reason) || !DPCCDataObservationContainersPresent(evidence.validation.before, reason) || !DPCCDataObservationContainersPresent(evidence.validation.after, reason) || !DPCCDataObservationContainersPresent(evidence.test.before, reason) || !DPCCDataObservationContainersPresent(evidence.test.after, reason)) return(DPCC_GATE_BLOCKED); if(!evidence.sources_match || !evidence.rho_source_verified) { reason = "data.provenance"; return(DPCC_GATE_BLOCKED); } if(evidence.filtered_validation_test) { reason = "data.filtered_validation_test"; return(DPCC_GATE_FAIL); } if(!DPCCDataObservationMembersPresent(profile.quality, evidence.fit.before, reason) || !DPCCDataObservationMembersPresent(profile.quality, evidence.fit.after, reason) || !DPCCDataObservationMembersPresent(profile.quality, evidence.validation.before, reason) || !DPCCDataObservationMembersPresent(profile.quality, evidence.validation.after, reason) || !DPCCDataObservationMembersPresent(profile.quality, evidence.test.before, reason) || !DPCCDataObservationMembersPresent(profile.quality, evidence.test.after, reason)) return(DPCC_GATE_BLOCKED); if(!DPCCDataObservationNumeric(evidence.fit.before, reason) || !DPCCDataObservationNumeric(evidence.fit.after, reason) || !DPCCDataObservationNumeric(evidence.validation.before, reason) || !DPCCDataObservationNumeric(evidence.validation.after, reason) || !DPCCDataObservationNumeric(evidence.test.before, reason) || !DPCCDataObservationNumeric(evidence.test.after, reason)) return(DPCC_GATE_FAIL); if(DPCCDataObservationHasDuplicateRare(evidence.fit.before) || DPCCDataObservationHasDuplicateRare(evidence.fit.after) || DPCCDataObservationHasDuplicateRare(evidence.validation.before) || DPCCDataObservationHasDuplicateRare(evidence.validation.after) || DPCCDataObservationHasDuplicateRare(evidence.test.before) || DPCCDataObservationHasDuplicateRare(evidence.test.after)) { reason = "data.rare_event_counts.duplicate"; return(DPCC_GATE_BLOCKED); } if(evidence.fit.after.total_windows > evidence.fit.before.total_windows || evidence.validation.after.total_windows > evidence.validation.before.total_windows || evidence.test.after.total_windows > evidence.test.before.total_windows) { reason = "data.selection.window_count"; return(DPCC_GATE_BLOCKED); } if(!DPCCDataObservationEqual(evidence.validation.before, evidence.validation.after) || !DPCCDataObservationEqual(evidence.test.before, evidence.test.after)) { reason = "data.validation_test.selection"; return(DPCC_GATE_BLOCKED); } return(DPCCEvaluateDataObservation(profile.quality, evidence.fit.after, evidence.sources_match_present, evidence.rho_source_verified_present, evidence.filtered_validation_test_present, evidence.sources_match, evidence.rho_source_verified, evidence.filtered_validation_test, reason)); } //+------------------------------------------------------------------+ //| Function DPCCDataReferenceInSet | //+------------------------------------------------------------------+ bool DPCCDataReferenceInSet(const SDPCCResourceReference &reference, const SDPCCResourceReference &set[]) { for(int i = 0; i < ArraySize(set); i++) if(set[i].id == reference.id && set[i].sha256 == reference.sha256 && DPCCDataLowerSha256(set[i].sha256)) return(true); return(false); } //+------------------------------------------------------------------+ //| Function DPCCDeriveDataReport | //+------------------------------------------------------------------+ ENUM_DPCC_GATE_VERDICT DPCCDeriveDataReport(const SDPCCDataProfile &profile, const SDPCCDataEvidence &evidence, const SDPCCDataBindings &bindings, const SDPCCDataReport &report, const string observed_evidence_sha256, const SDPCCResourceReference &observed_data_evidence[], const bool production, string &reason) { reason = ""; if(profile.readiness_profile_id != bindings.readiness_profile_id) { reason = "readiness_profile.id"; return(DPCC_GATE_BLOCKED); } if(!DPCCDataReferenceMatches(report.readiness_profile, bindings.readiness_profile_id, bindings.readiness_profile_sha256, reason, "readiness_profile")) return(DPCC_GATE_BLOCKED); if(production && (report.example_only || profile.example_only || (report.source_distribution_present && evidence.example_only))) { reason = "example_only"; return(DPCC_GATE_BLOCKED); } if(!report.source_distribution_present) { if(report.declared_data == DPCC_GATE_NOT_RUN) return(DPCC_GATE_NOT_RUN); reason = "source_distribution_report"; return(DPCC_GATE_BLOCKED); } if(!DPCCDataReferenceMatches(report.source_distribution_report, evidence.evidence_id, observed_evidence_sha256, reason, "source_distribution_report")) return(DPCC_GATE_BLOCKED); if(report.evaluated_at_utc < evidence.observed_at_utc) { reason = "evaluated_at_utc"; return(DPCC_GATE_BLOCKED); } const ENUM_DPCC_GATE_VERDICT derived = DPCCEvaluateDataEvidence(profile, evidence, bindings, reason); if(derived != DPCC_GATE_PASS) return(derived); if(ArraySize(report.data_evidence) == 0) { reason = "data.evidence"; return(DPCC_GATE_BLOCKED); } for(int i = 0; i < ArraySize(report.data_evidence); i++) if(!DPCCDataReferenceInSet(report.data_evidence[i], observed_data_evidence)) { reason = "data.evidence"; return(DPCC_GATE_BLOCKED); } if(!report.rho_extraction_present || report.rho_extraction_report.id != evidence.rho_implementation.id || report.rho_extraction_report.sha256 != evidence.rho_implementation.sha256 || !DPCCDataLowerSha256(report.rho_extraction_report.sha256)) { reason = "rho_extraction_report"; return(DPCC_GATE_BLOCKED); } SDPCCResourceReference required[3]; required[0] = evidence.dataset_manifest; required[1] = evidence.bank_manifest; required[2] = evidence.training_config; for(int i = 0; i < 3; i++) if(!DPCCDataReferenceInSet(required[i], report.validated_inputs)) { reason = "validated_inputs"; return(DPCC_GATE_BLOCKED); } return(DPCC_GATE_PASS); } //+------------------------------------------------------------------+ //| Function DPCCDataDeclaredAgrees | //+------------------------------------------------------------------+ bool DPCCDataDeclaredAgrees(const ENUM_DPCC_GATE_VERDICT declared_verdict, const ENUM_DPCC_GATE_VERDICT derived_verdict, string &reason) { if(declared_verdict != derived_verdict) { reason = "data.verdict"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCEvaluateDataReportClaim | //+------------------------------------------------------------------+ bool DPCCEvaluateDataReportClaim(const SDPCCDataProfile &profile, const SDPCCDataEvidence &evidence, const SDPCCDataBindings &bindings, const SDPCCDataReport &report, const string observed_evidence_sha256, const SDPCCResourceReference &observed_data_evidence[], const bool production, ENUM_DPCC_GATE_VERDICT &derived, string &reason) { derived = DPCCDeriveDataReport(profile, evidence, bindings, report, observed_evidence_sha256, observed_data_evidence, production, reason); if(report.declared_data != derived) { reason = "data.verdict"; return(false); } return(true); } #endif //+------------------------------------------------------------------+ //| Function DPCCClearDataObservation | //+------------------------------------------------------------------+ void DPCCClearDataObservation(SDPCCDataObservation &observation) { observation.valid_windows_present = false; observation.total_windows_present = false; observation.counts_present = false; observation.rare_event_counts_present = false; observation.valid_windows = 0; observation.total_windows = 0; for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) { observation.category_present[i] = false; observation.category_counts[i] = 0; } ArrayResize(observation.rare_event_counts, 0); } //+------------------------------------------------------------------+ //| Function DPCCClearDataPolicy | //+------------------------------------------------------------------+ void DPCCClearDataPolicy(SDPCCDataQualityPolicy &policy) { policy.min_valid_windows = 0; policy.max_invalid_fraction = 0.0; for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) { policy.min_category_counts[i] = 0; policy.min_category_present[i] = false; policy.zero_minimum_justifications[i] = ""; } ArrayResize(policy.rare_event_min_counts, 0); policy.result_selection_scope = ""; policy.validation_test_profit_filter = false; policy.mandatory_losing_fraction = false; policy.counts_before_after_required = false; } //+------------------------------------------------------------------+ //| Function DPCCClearDataProfile | //+------------------------------------------------------------------+ void DPCCClearDataProfile(SDPCCDataProfile &profile) { profile.example_only = false; profile.readiness_profile_id = ""; profile.dataset_id = ""; profile.training_id = ""; profile.dataset_manifest.path = ""; profile.dataset_manifest.sha256 = ""; profile.dataset_manifest.id = ""; profile.rho_implementation.path = ""; profile.rho_implementation.sha256 = ""; profile.rho_implementation.id = ""; profile.forecast_id = ""; DPCCClearDataPolicy(profile.quality); } //+------------------------------------------------------------------+ //| Function DPCCClearDataEvidence | //+------------------------------------------------------------------+ void DPCCClearDataEvidence(SDPCCDataEvidence &evidence) { evidence.example_only = false; evidence.evidence_id = ""; evidence.readiness_profile.path = ""; evidence.readiness_profile.sha256 = ""; evidence.readiness_profile.id = ""; evidence.dataset_manifest.path = ""; evidence.dataset_manifest.sha256 = ""; evidence.dataset_manifest.id = ""; evidence.bank_manifest.path = ""; evidence.bank_manifest.sha256 = ""; evidence.bank_manifest.id = ""; evidence.training_config.path = ""; evidence.training_config.sha256 = ""; evidence.training_config.id = ""; evidence.rho_implementation.path = ""; evidence.rho_implementation.sha256 = ""; evidence.rho_implementation.id = ""; evidence.forecast_id = ""; evidence.normalization_id = ""; evidence.sources_match_present = false; evidence.rho_source_verified_present = false; evidence.filtered_validation_test_present = false; evidence.observed_at_utc_present = false; evidence.sources_match = false; evidence.rho_source_verified = false; evidence.filtered_validation_test = false; evidence.observed_at_utc = 0; DPCCClearDataObservation(evidence.fit.before); DPCCClearDataObservation(evidence.fit.after); DPCCClearDataObservation(evidence.validation.before); DPCCClearDataObservation(evidence.validation.after); DPCCClearDataObservation(evidence.test.before); DPCCClearDataObservation(evidence.test.after); } //+------------------------------------------------------------------+ //| Function DPCCClearDataReport | //+------------------------------------------------------------------+ void DPCCClearDataReport(SDPCCDataReport &report) { report.example_only = false; report.evaluated_at_utc = 0; report.declared_data = DPCC_GATE_NOT_RUN; report.readiness_profile.path = ""; report.readiness_profile.sha256 = ""; report.readiness_profile.id = ""; ArrayResize(report.data_evidence, 0); report.source_distribution_present = false; report.rho_extraction_present = false; report.source_distribution_report.path = ""; report.source_distribution_report.sha256 = ""; report.source_distribution_report.id = ""; report.rho_extraction_report.path = ""; report.rho_extraction_report.sha256 = ""; report.rho_extraction_report.id = ""; ArrayResize(report.validated_inputs, 0); } //+------------------------------------------------------------------+ //| Function DPCCDataDecodeFail | //+------------------------------------------------------------------+ bool DPCCDataDecodeFail(SDPCCDataEvidence &evidence, string &reason, const string failure) { DPCCClearDataEvidence(evidence); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCDataProfileDecodeFail | //+------------------------------------------------------------------+ bool DPCCDataProfileDecodeFail(SDPCCDataProfile &profile, string &reason, const string failure) { DPCCClearDataProfile(profile); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCDataReportDecodeFail | //+------------------------------------------------------------------+ bool DPCCDataReportDecodeFail(SDPCCDataReport &report, string &reason, const string failure) { DPCCClearDataReport(report); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCDataReadBool | //+------------------------------------------------------------------+ bool DPCCDataReadBool(CDPCCJsonDocument &document, const int object_node, const string name, bool &value, string &reason, const string path) { const int node = document.FindMember(object_node, name); if(node < 0 || document.Type(node) != DPCC_JSON_BOOLEAN) { reason = path; return(false); } value = (document.Text(node) == "true"); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataReadString | //+------------------------------------------------------------------+ bool DPCCDataReadString(CDPCCJsonDocument &document, const int object_node, const string name, string &value, string &reason, const string path) { const int node = document.FindMember(object_node, name); if(node < 0 || document.Type(node) != DPCC_JSON_STRING) { reason = path; return(false); } value = document.Text(node); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataReadLong | //+------------------------------------------------------------------+ bool DPCCDataReadLong(CDPCCJsonDocument &document, const int object_node, const string name, long &value, string &reason, const string path) { const int node = document.FindMember(object_node, name); if(node < 0 || !DPCCReadExactNonnegativeLong(document, node, value)) { reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDataReadDouble | //+------------------------------------------------------------------+ bool DPCCDataReadDouble(CDPCCJsonDocument &document, const int object_node, const string name, double &value, string &reason, const string path) { const int node = document.FindMember(object_node, name); if(node < 0 || !document.TryNumber(node, value) || !DPCCDataFinite(value)) { reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataPolicy | //+------------------------------------------------------------------+ bool DPCCDecodeDataPolicy(CDPCCJsonDocument &document, const int root, SDPCCDataQualityPolicy &policy, string &reason) { DPCCClearDataPolicy(policy); const int quality = document.FindMember(root, "data_quality"); if(quality < 0 || document.Type(quality) != DPCC_JSON_OBJECT) { reason = "data_quality"; return(false); } if(!DPCCDataReadLong(document, quality, "min_valid_windows", policy.min_valid_windows, reason, "data_quality.min_valid_windows") || !DPCCDataReadDouble(document, quality, "max_invalid_fraction", policy.max_invalid_fraction, reason, "data_quality.max_invalid_fraction")) { DPCCClearDataPolicy(policy); return(false); } const int counts = document.FindMember(quality, "min_category_counts"); const int justifications = document.FindMember(quality, "zero_minimum_justifications"); if(counts < 0 || justifications < 0 || document.Type(counts) != DPCC_JSON_OBJECT || document.Type(justifications) != DPCC_JSON_OBJECT) { DPCCClearDataPolicy(policy); reason = "data_quality.categories"; return(false); } for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) { const string name = DPCCDataCategoryName(i); if(!DPCCDataReadLong(document, counts, name, policy.min_category_counts[i], reason, "data_quality.min_category_counts." + name) || !DPCCDataReadString(document, justifications, name, policy.zero_minimum_justifications[i], reason, "data_quality.zero_minimum_justifications." + name)) { DPCCClearDataPolicy(policy); return(false); } policy.min_category_present[i] = true; } const int rare = document.FindMember(quality, "rare_event_min_counts"); if(rare < 0 || document.Type(rare) != DPCC_JSON_ARRAY) { DPCCClearDataPolicy(policy); reason = "data_quality.rare_event_min_counts"; return(false); } const int rare_count = document.ChildCount(rare); if(ArrayResize(policy.rare_event_min_counts, rare_count) != rare_count) { DPCCClearDataPolicy(policy); reason = "data_quality.rare_event_min_counts.capacity"; return(false); } int item = document.FirstChild(rare); for(int i = 0; i < rare_count; i++, item = document.NextSibling(item)) { if(item < 0 || document.Type(item) != DPCC_JSON_OBJECT || !DPCCDataReadString(document, item, "event", policy.rare_event_min_counts[i].event, reason, "data_quality.rare_event_min_counts.event") || !DPCCDataReadLong(document, item, "min_count", policy.rare_event_min_counts[i].min_count, reason, "data_quality.rare_event_min_counts.min_count") || !DPCCDataReadString(document, item, "zero_reason", policy.rare_event_min_counts[i].zero_reason, reason, "data_quality.rare_event_min_counts.zero_reason")) { DPCCClearDataPolicy(policy); return(false); } } if(!DPCCDataReadString(document, quality, "result_selection_scope", policy.result_selection_scope, reason, "data_quality.result_selection_scope") || !DPCCDataReadBool(document, quality, "validation_test_profit_filter", policy.validation_test_profit_filter, reason, "data_quality.validation_test_profit_filter") || !DPCCDataReadBool(document, quality, "mandatory_losing_fraction", policy.mandatory_losing_fraction, reason, "data_quality.mandatory_losing_fraction") || !DPCCDataReadBool(document, quality, "counts_before_after_required", policy.counts_before_after_required, reason, "data_quality.counts_before_after_required")) { DPCCClearDataPolicy(policy); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataProfile | //+------------------------------------------------------------------+ bool DPCCDecodeDataProfile(CDPCCJsonDocument &document, const int root, SDPCCDataProfile &profile, string &reason) { DPCCClearDataProfile(profile); if(!DPCCDataReadBool(document, root, "example_only", profile.example_only, reason, "example_only") || !DPCCDataReadString(document, root, "readiness_profile_id", profile.readiness_profile_id, reason, "readiness_profile_id") || !DPCCDataReadString(document, root, "dataset_id", profile.dataset_id, reason, "dataset_id") || !DPCCDataReadString(document, root, "training_id", profile.training_id, reason, "training_id") || !DPCCDecodeResourceReference(document, document.FindMember(root, "dataset_manifest"), profile.dataset_manifest, reason, "dataset_manifest")) return(DPCCDataProfileDecodeFail(profile, reason, reason)); const int rho = document.FindMember(root, "rho_provenance"); if(rho < 0 || document.Type(rho) != DPCC_JSON_OBJECT || !DPCCDataReadString(document, rho, "forecast_id", profile.forecast_id, reason, "rho_provenance.forecast_id") || !DPCCDecodeResourceReference(document, document.FindMember(rho, "implementation_ref"), profile.rho_implementation, reason, "rho_provenance.implementation_ref") || !DPCCDecodeDataPolicy(document, root, profile.quality, reason)) return(DPCCDataProfileDecodeFail(profile, reason, reason)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataObservation | //+------------------------------------------------------------------+ bool DPCCDecodeDataObservation(CDPCCJsonDocument &document, const int node, SDPCCDataObservation &observation, string &reason, const string path) { DPCCClearDataObservation(observation); if(node < 0 || document.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } if(!DPCCDataReadLong(document, node, "valid_windows", observation.valid_windows, reason, path + ".valid_windows")) return(false); observation.valid_windows_present = true; if(!DPCCDataReadLong(document, node, "total_windows", observation.total_windows, reason, path + ".total_windows")) { DPCCClearDataObservation(observation); return(false); } observation.total_windows_present = true; const int counts = document.FindMember(node, "counts"); if(counts < 0 || document.Type(counts) != DPCC_JSON_OBJECT) { DPCCClearDataObservation(observation); reason = path + ".counts"; return(false); } observation.counts_present = true; for(int i = 0; i < DPCC_DATA_CATEGORY_COUNT; i++) { const string name = DPCCDataCategoryName(i); if(!DPCCDataReadLong(document, counts, name, observation.category_counts[i], reason, path + ".counts." + name)) { DPCCClearDataObservation(observation); return(false); } observation.category_present[i] = true; } const int rare = document.FindMember(node, "rare_event_counts"); if(rare < 0 || document.Type(rare) != DPCC_JSON_ARRAY) { DPCCClearDataObservation(observation); reason = path + ".rare_event_counts"; return(false); } observation.rare_event_counts_present = true; const int rare_count = document.ChildCount(rare); if(ArrayResize(observation.rare_event_counts, rare_count) != rare_count) { DPCCClearDataObservation(observation); reason = path + ".rare_event_counts.capacity"; return(false); } int item = document.FirstChild(rare); for(int i = 0; i < rare_count; i++, item = document.NextSibling(item)) { if(item < 0 || document.Type(item) != DPCC_JSON_OBJECT || !DPCCDataReadString(document, item, "event", observation.rare_event_counts[i].event, reason, path + ".rare_event_counts.event") || !DPCCDataReadLong(document, item, "count", observation.rare_event_counts[i].count, reason, path + ".rare_event_counts.count")) { DPCCClearDataObservation(observation); return(false); } } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataSplit | //+------------------------------------------------------------------+ bool DPCCDecodeDataSplit(CDPCCJsonDocument &document, const int node, SDPCCDataSplit &split, string &reason, const string path) { DPCCClearDataObservation(split.before); DPCCClearDataObservation(split.after); if(node < 0 || document.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } if(!DPCCDecodeDataObservation(document, document.FindMember(node, "before"), split.before, reason, path + ".before") || !DPCCDecodeDataObservation(document, document.FindMember(node, "after"), split.after, reason, path + ".after")) { const string failure = reason; DPCCClearDataObservation(split.before); DPCCClearDataObservation(split.after); reason = failure; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataEvidence | //+------------------------------------------------------------------+ bool DPCCDecodeDataEvidence(CDPCCJsonDocument &document, const int root, SDPCCDataEvidence &evidence, string &reason) { DPCCClearDataEvidence(evidence); if(!DPCCDataReadBool(document, root, "example_only", evidence.example_only, reason, "example_only") || !DPCCDataReadString(document, root, "evidence_id", evidence.evidence_id, reason, "evidence_id") || !DPCCDecodeResourceReference(document, document.FindMember(root, "readiness_profile"), evidence.readiness_profile, reason, "readiness_profile") || !DPCCDecodeResourceReference(document, document.FindMember(root, "dataset_manifest"), evidence.dataset_manifest, reason, "dataset_manifest") || !DPCCDecodeResourceReference(document, document.FindMember(root, "bank_manifest"), evidence.bank_manifest, reason, "bank_manifest") || !DPCCDecodeResourceReference(document, document.FindMember(root, "training_config"), evidence.training_config, reason, "training_config") || !DPCCDataReadString(document, root, "forecast_id", evidence.forecast_id, reason, "forecast_id") || !DPCCDataReadString(document, root, "normalization_id", evidence.normalization_id, reason, "normalization_id") || !DPCCDecodeResourceReference(document, document.FindMember(root, "rho_implementation"), evidence.rho_implementation, reason, "rho_implementation")) return(DPCCDataDecodeFail(evidence, reason, reason)); if(!DPCCDataReadBool(document, root, "sources_match", evidence.sources_match, reason, "sources_match")) return(DPCCDataDecodeFail(evidence, reason, reason)); evidence.sources_match_present = true; if(!DPCCDataReadBool(document, root, "rho_source_verified", evidence.rho_source_verified, reason, "rho_source_verified")) return(DPCCDataDecodeFail(evidence, reason, reason)); evidence.rho_source_verified_present = true; if(!DPCCDataReadBool(document, root, "filtered_validation_test", evidence.filtered_validation_test, reason, "filtered_validation_test")) return(DPCCDataDecodeFail(evidence, reason, reason)); evidence.filtered_validation_test_present = true; if(!DPCCDataReadLong(document, root, "observed_at_utc", evidence.observed_at_utc, reason, "observed_at_utc")) return(DPCCDataDecodeFail(evidence, reason, reason)); evidence.observed_at_utc_present = true; const int splits = document.FindMember(root, "splits"); if(splits < 0 || document.Type(splits) != DPCC_JSON_OBJECT || !DPCCDecodeDataSplit(document, document.FindMember(splits, "fit"), evidence.fit, reason, "splits.fit") || !DPCCDecodeDataSplit(document, document.FindMember(splits, "validation"), evidence.validation, reason, "splits.validation") || !DPCCDecodeDataSplit(document, document.FindMember(splits, "test"), evidence.test, reason, "splits.test")) return(DPCCDataDecodeFail(evidence, reason, reason)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataReferenceArray | //+------------------------------------------------------------------+ bool DPCCDecodeDataReferenceArray(CDPCCJsonDocument &document, const int node, SDPCCResourceReference &references[], string &reason, const string path) { ArrayResize(references, 0); if(node < 0 || document.Type(node) != DPCC_JSON_ARRAY) { reason = path; return(false); } const int count = document.ChildCount(node); if(ArrayResize(references, count) != count) { reason = path + ".capacity"; return(false); } int item = document.FirstChild(node); for(int i = 0; i < count; i++, item = document.NextSibling(item)) if(!DPCCDecodeResourceReference(document, item, references[i], reason, path)) { ArrayResize(references, 0); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeNullableDataReference | //+------------------------------------------------------------------+ bool DPCCDecodeNullableDataReference(CDPCCJsonDocument &document, const int node, bool &present, SDPCCResourceReference &reference, string &reason, const string path) { present = false; reference.path = ""; reference.sha256 = ""; reference.id = ""; if(node < 0) { reason = path; return(false); } if(document.Type(node) == DPCC_JSON_NULL) return(true); if(!DPCCDecodeResourceReference(document, node, reference, reason, path)) return(false); present = true; return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDataReport | //+------------------------------------------------------------------+ bool DPCCDecodeDataReport(CDPCCJsonDocument &document, const int root, SDPCCDataReport &report, string &reason) { DPCCClearDataReport(report); if(!DPCCDataReadBool(document, root, "example_only", report.example_only, reason, "example_only") || !DPCCDataReadLong(document, root, "evaluated_at_utc", report.evaluated_at_utc, reason, "evaluated_at_utc") || !DPCCDecodeResourceReference(document, document.FindMember(root, "readiness_profile"), report.readiness_profile, reason, "readiness_profile")) return(DPCCDataReportDecodeFail(report, reason, reason)); const int data = document.FindMember(root, "data"); string verdict = ""; if(data < 0 || document.Type(data) != DPCC_JSON_OBJECT || !DPCCDataReadString(document, data, "verdict", verdict, reason, "data.verdict") || !DPCCParseGateVerdict(verdict, report.declared_data)) return(DPCCDataReportDecodeFail(report, reason, "data.verdict")); if(!DPCCDecodeDataReferenceArray(document, document.FindMember(data, "evidence"), report.data_evidence, reason, "data.evidence") || !DPCCDecodeNullableDataReference(document, document.FindMember(root, "source_distribution_report"), report.source_distribution_present, report.source_distribution_report, reason, "source_distribution_report") || !DPCCDecodeNullableDataReference(document, document.FindMember(root, "rho_extraction_report"), report.rho_extraction_present, report.rho_extraction_report, reason, "rho_extraction_report") || !DPCCDecodeDataReferenceArray(document, document.FindMember(root, "validated_inputs"), report.validated_inputs, reason, "validated_inputs")) return(DPCCDataReportDecodeFail(report, reason, reason)); return(true); } #endif struct SDPCCDatasetInterval { long start_utc; long end_utc; long bar_start; long bar_end; }; struct SDPCCDatasetManifest { bool example_only; string dataset_id; string data_source; string normalization_id; SDPCCResourceReference timezone_mapping; SDPCCResourceReference time_index; SDPCCResourceShape shapes; long embargo_bars; SDPCCDatasetInterval fit; SDPCCDatasetInterval validation; SDPCCDatasetInterval test; SDPCCDatasetInterval normalization_fit; }; struct SDPCCDatasetContinuation { SDPCCResourceReference model_dataset; string training_dataset_id; string training_normalization_id; long rho_k; }; struct SDPCCDatasetLinkResult { string dataset_id; string time_index_id; string evidence_id; string rho_id; string dataset_path; string time_index_path; string evidence_path; string rho_path; string dataset_sha256; string time_index_sha256; string evidence_sha256; string rho_sha256; }; //+------------------------------------------------------------------+ //| Function DPCCClearDatasetInterval | //+------------------------------------------------------------------+ void DPCCClearDatasetInterval(SDPCCDatasetInterval &interval) { interval.start_utc = 0; interval.end_utc = 0; interval.bar_start = 0; interval.bar_end = 0; } //+------------------------------------------------------------------+ //| Function DPCCClearDatasetManifest | //+------------------------------------------------------------------+ void DPCCClearDatasetManifest(SDPCCDatasetManifest &dataset) { dataset.example_only = false; dataset.dataset_id = ""; dataset.data_source = ""; dataset.normalization_id = ""; dataset.timezone_mapping.path = ""; dataset.timezone_mapping.sha256 = ""; dataset.timezone_mapping.id = ""; dataset.time_index.path = ""; dataset.time_index.sha256 = ""; dataset.time_index.id = ""; dataset.shapes.K = 0; dataset.shapes.V = 0; dataset.shapes.H = 0; dataset.shapes.D = 0; dataset.shapes.M = 0; dataset.shapes.J = 0; dataset.shapes.E = 0; dataset.shapes.A = 0; dataset.shapes.B = 0; dataset.shapes.account_features = 0; dataset.embargo_bars = 0; DPCCClearDatasetInterval(dataset.fit); DPCCClearDatasetInterval(dataset.validation); DPCCClearDatasetInterval(dataset.test); DPCCClearDatasetInterval(dataset.normalization_fit); } //+------------------------------------------------------------------+ //| Function DPCCClearDatasetContinuation | //+------------------------------------------------------------------+ void DPCCClearDatasetContinuation(SDPCCDatasetContinuation &continuation) { continuation.model_dataset.path = ""; continuation.model_dataset.sha256 = ""; continuation.model_dataset.id = ""; continuation.training_dataset_id = ""; continuation.training_normalization_id = ""; continuation.rho_k = 0; } //+------------------------------------------------------------------+ //| Function DPCCClearDatasetLinkResult | //+------------------------------------------------------------------+ void DPCCClearDatasetLinkResult(SDPCCDatasetLinkResult &result) { result.dataset_id = ""; result.time_index_id = ""; result.evidence_id = ""; result.rho_id = ""; result.dataset_path = ""; result.time_index_path = ""; result.evidence_path = ""; result.rho_path = ""; result.dataset_sha256 = ""; result.time_index_sha256 = ""; result.evidence_sha256 = ""; result.rho_sha256 = ""; } //+------------------------------------------------------------------+ //| Function DPCCDatasetFail | //+------------------------------------------------------------------+ bool DPCCDatasetFail(SDPCCDatasetLinkResult &result, string &reason, const string failure) { DPCCClearDatasetLinkResult(result); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCDatasetDecodeInterval | //+------------------------------------------------------------------+ bool DPCCDatasetDecodeInterval(CDPCCJsonDocument &document, const int node, SDPCCDatasetInterval &interval, string &reason, const string path) { DPCCClearDatasetInterval(interval); if(node < 0 || document.Type(node) != DPCC_JSON_OBJECT) { reason = path; return(false); } if(!DPCCResourceReadLong(document, node, "start_utc", interval.start_utc, reason, path + ".start_utc") || !DPCCResourceReadLong(document, node, "end_utc", interval.end_utc, reason, path + ".end_utc") || !DPCCResourceReadLong(document, node, "bar_start", interval.bar_start, reason, path + ".bar_start") || !DPCCResourceReadLong(document, node, "bar_end", interval.bar_end, reason, path + ".bar_end")) { DPCCClearDatasetInterval(interval); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDatasetManifest | //+------------------------------------------------------------------+ bool DPCCDecodeDatasetManifest(CDPCCJsonDocument &document, const int root, SDPCCDatasetManifest &dataset, string &reason) { DPCCClearDatasetManifest(dataset); reason = ""; int splits = -1; if(!DPCCCentralBool(document, root, "example_only", dataset.example_only, reason, "dataset.example_only") || !DPCCResourceReadString(document, root, "dataset_id", dataset.dataset_id, reason, "dataset.dataset_id") || !DPCCResourceReadString(document, root, "data_source", dataset.data_source, reason, "dataset.data_source") || !DPCCResourceReadString(document, root, "normalization_id", dataset.normalization_id, reason, "dataset.normalization_id") || !DPCCDecodeResourceReference(document, document.FindMember(root, "timezone_mapping"), dataset.timezone_mapping, reason, "dataset.timezone_mapping") || !DPCCDecodeResourceReference(document, document.FindMember(root, "time_index"), dataset.time_index, reason, "dataset.time_index") || !DPCCDecodeResourceShape(document, document.FindMember(root, "shapes"), dataset.shapes, reason, "dataset.shapes") || !DPCCResourceReadLong(document, root, "embargo_bars", dataset.embargo_bars, reason, "dataset.embargo_bars") || !DPCCCentralObject(document, root, "splits", splits, reason, "dataset.splits") || !DPCCDatasetDecodeInterval(document, document.FindMember(splits, "fit"), dataset.fit, reason, "dataset.splits.fit") || !DPCCDatasetDecodeInterval(document, document.FindMember(splits, "validation"), dataset.validation, reason, "dataset.splits.validation") || !DPCCDatasetDecodeInterval(document, document.FindMember(splits, "test"), dataset.test, reason, "dataset.splits.test") || !DPCCDatasetDecodeInterval(document, document.FindMember(root, "normalization_fit_interval"), dataset.normalization_fit, reason, "dataset.normalization_fit_interval")) { DPCCClearDatasetManifest(dataset); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeDatasetContinuation | //+------------------------------------------------------------------+ bool DPCCDecodeDatasetContinuation(CDPCCJsonDocument &model_document, CDPCCJsonDocument &training_document, CDPCCJsonDocument &profile_document, SDPCCDatasetContinuation &continuation, string &reason) { DPCCClearDatasetContinuation(continuation); reason = ""; int training = -1, rho = -1; if(!DPCCCentralObject(model_document, model_document.Root(), "training", training, reason, "model.training") || !DPCCDecodeResourceReference(model_document, model_document.FindMember(training, "dataset_manifest"), continuation.model_dataset, reason, "model.dataset_manifest") || !DPCCResourceReadString(training_document, training_document.Root(), "dataset_id", continuation.training_dataset_id, reason, "training.dataset_id") || !DPCCResourceReadString(training_document, training_document.Root(), "normalization_id", continuation.training_normalization_id, reason, "training.normalization_id") || !DPCCCentralObject(profile_document, profile_document.Root(), "rho_provenance", rho, reason, "profile.rho_provenance") || !DPCCResourceReadLong(profile_document, rho, "K", continuation.rho_k, reason, "profile.rho_provenance.K")) { DPCCClearDatasetContinuation(continuation); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDatasetShapeEqual | //+------------------------------------------------------------------+ bool DPCCDatasetShapeEqual(const SDPCCResourceShape &dataset, const SDPCCResourceShape &profile, string &reason) { if(dataset.K != profile.K) reason = "dataset.shapes.K"; else if(dataset.V != profile.V) reason = "dataset.shapes.V"; else if(dataset.H != profile.H) reason = "dataset.shapes.H"; else if(dataset.D != profile.D) reason = "dataset.shapes.D"; else if(dataset.M != profile.M) reason = "dataset.shapes.M"; else if(dataset.J != profile.J) reason = "dataset.shapes.J"; else if(dataset.E != profile.E) reason = "dataset.shapes.E"; else if(dataset.A != profile.A) reason = "dataset.shapes.A"; else if(dataset.B != profile.B) reason = "dataset.shapes.B"; else if(dataset.account_features != profile.account_features) reason = "dataset.shapes.account_features"; return(reason == ""); } //+------------------------------------------------------------------+ //| Function DPCCDatasetValidateInterval | //+------------------------------------------------------------------+ bool DPCCDatasetValidateInterval(const SDPCCDatasetInterval &interval, const SDPCCDatasetTimeIndex &time_index, string &reason, const string path) { const int count = ArraySize(time_index.bar_index); if(interval.bar_start < 0 || interval.bar_start >= interval.bar_end) { reason = path + ".bar_start"; return(false); } if(interval.bar_end > count) { reason = path + ".bar_end"; return(false); } const int first = (int)interval.bar_start; const int last = (int)interval.bar_end - 1; if(interval.start_utc != time_index.start_utc[first]) { reason = path + ".start_utc"; return(false); } if(interval.end_utc != time_index.end_utc[last]) { reason = path + ".end_utc"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCValidateDatasetTimeClosure | //+------------------------------------------------------------------+ bool DPCCValidateDatasetTimeClosure(const SDPCCDatasetManifest &dataset, const SDPCCDatasetTimeIndex &time_index, const SDPCCResourceProfile &profile, const long rho_k, const bool production, string &reason) { reason = ""; string index_reason = ""; if(!DPCCValidateDatasetTimeIndex(time_index, dataset.dataset_id, index_reason)) { reason = index_reason; return(false); } if(production && dataset.example_only) { reason = "dataset.example_only"; return(false); } if(production && time_index.example_only) { reason = "time_index.example_only"; return(false); } if(dataset.time_index.id != time_index.time_index_id) { reason = "dataset.time_index.id"; return(false); } if(dataset.data_source != time_index.data_source) { reason = "dataset.data_source"; return(false); } if(dataset.timezone_mapping.id != time_index.timezone_mapping.id) { reason = "dataset.timezone_mapping.id"; return(false); } if(dataset.timezone_mapping.sha256 != time_index.timezone_mapping.sha256) { reason = "dataset.timezone_mapping.sha256"; return(false); } if(!DPCCDatasetShapeEqual(dataset.shapes, profile.shapes, reason)) return(false); if(rho_k != dataset.shapes.K) { reason = "profile.rho_provenance.K"; return(false); } if(!DPCCDatasetValidateInterval(dataset.fit, time_index, reason, "dataset.splits.fit") || !DPCCDatasetValidateInterval(dataset.validation, time_index, reason, "dataset.splits.validation") || !DPCCDatasetValidateInterval(dataset.test, time_index, reason, "dataset.splits.test") || !DPCCDatasetValidateInterval(dataset.normalization_fit, time_index, reason, "dataset.normalization_fit_interval")) return(false); if(dataset.fit.bar_end > dataset.validation.bar_start || dataset.fit.end_utc > dataset.validation.start_utc) { reason = "dataset.splits.validation.order"; return(false); } if(dataset.validation.bar_end > dataset.test.bar_start || dataset.validation.end_utc > dataset.test.start_utc) { reason = "dataset.splits.test.order"; return(false); } if(dataset.embargo_bars < dataset.shapes.H) { reason = "dataset.embargo_bars.H"; return(false); } if(dataset.validation.bar_start - dataset.fit.bar_end < dataset.embargo_bars) { reason = "dataset.splits.validation.embargo"; return(false); } if(dataset.test.bar_start - dataset.validation.bar_end < dataset.embargo_bars) { reason = "dataset.splits.test.embargo"; return(false); } if(dataset.normalization_fit.bar_start < dataset.fit.bar_start || dataset.normalization_fit.bar_end > dataset.fit.bar_end) { reason = "dataset.normalization_fit_interval.fit"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCClearDataBindings | //+------------------------------------------------------------------+ void DPCCClearDataBindings(SDPCCDataBindings &bindings) { bindings.readiness_profile_id = ""; bindings.readiness_profile_sha256 = ""; bindings.dataset_id = ""; bindings.dataset_sha256 = ""; bindings.bank_id = ""; bindings.bank_sha256 = ""; bindings.training_id = ""; bindings.training_sha256 = ""; bindings.rho_id = ""; bindings.rho_sha256 = ""; bindings.forecast_id = ""; bindings.normalization_id = ""; } //+------------------------------------------------------------------+ //| Function DPCCValidateDatasetEvidenceClosure | //+------------------------------------------------------------------+ bool DPCCValidateDatasetEvidenceClosure(const SDPCCCentralLinkResult ¢ral, const SDPCCDatasetContinuation &continuation, const SDPCCDatasetManifest &dataset, const SDPCCDataProfile &profile, const SDPCCDataEvidence &evidence, const string dataset_sha256, const string rho_sha256, const string bank_id, const string bank_sha256, const bool production, SDPCCDataBindings &bindings, ENUM_DPCC_GATE_VERDICT &verdict, string &reason) { DPCCClearDataBindings(bindings); verdict = DPCC_GATE_BLOCKED; reason = ""; if(continuation.model_dataset.id != dataset.dataset_id) { reason = "model.dataset_manifest.id"; return(false); } if(continuation.model_dataset.sha256 != dataset_sha256) { reason = "model.dataset_manifest.sha256"; return(false); } if(continuation.training_dataset_id != dataset.dataset_id) { reason = "training.dataset_id"; return(false); } if(continuation.training_normalization_id != dataset.normalization_id) { reason = "training.normalization_id"; return(false); } bindings.readiness_profile_id = central.profile_id; bindings.readiness_profile_sha256 = central.profile_sha256; bindings.dataset_id = dataset.dataset_id; bindings.dataset_sha256 = dataset_sha256; bindings.bank_id = bank_id; bindings.bank_sha256 = bank_sha256; bindings.training_id = central.training_id; bindings.training_sha256 = central.training_sha256; bindings.rho_id = profile.rho_implementation.id; bindings.rho_sha256 = rho_sha256; bindings.forecast_id = profile.forecast_id; bindings.normalization_id = dataset.normalization_id; verdict = DPCCEvaluateDataEvidence(profile, evidence, bindings, reason); if(verdict != DPCC_GATE_PASS) { DPCCClearDataBindings(bindings); return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCDatasetRelativePath | //+------------------------------------------------------------------+ bool DPCCDatasetRelativePath(const string root, const string canonical, string &relative, string &reason) { relative = ""; const string prefix = root + "/"; if(StringFind(canonical, prefix) != 0 || StringLen(canonical) <= StringLen(prefix)) { reason = "path.relative"; return(false); } relative = StringSubstr(canonical, StringLen(prefix)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCDatasetReadSource | //+------------------------------------------------------------------+ bool DPCCDatasetReadSource(const string path, const bool common_files, const string expected_sha, string &source, string &observed_sha, string &reason, const string role) { source = ""; observed_sha = ""; uchar bytes[]; if(!DPCCReadArtifactBytes(path, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, bytes, reason)) { reason = role + "." + reason; return(false); } if(!DPCCSHA256Lower(bytes, observed_sha, reason)) { reason = role + "." + reason; return(false); } if(observed_sha != expected_sha) { reason = role + ".sha256"; return(false); } if(!DPCCCentralBytesSource(bytes, source, reason, role)) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCResolveDatasetClosure | //+------------------------------------------------------------------+ bool DPCCResolveDatasetClosure(const string package_root_relative, const bool common_files, const bool production, const SDPCCCentralLinkResult ¢ral, const string model_schema, const string training_schema, const string profile_schema, const string dataset_schema, const string time_index_schema, const string evidence_schema, const string evidence_package_path, const string evidence_sha256, const string bank_id, const string bank_sha256, SDPCCDatasetLinkResult &result, ENUM_DPCC_GATE_VERDICT &verdict, string &reason) { DPCCClearDatasetLinkResult(result); verdict = DPCC_GATE_BLOCKED; reason = ""; string model_source = "", training_source = "", profile_source = ""; string observed_model = "", observed_training = "", observed_profile = ""; if(!DPCCDatasetReadSource(central.model_path, common_files, central.model_sha256, model_source, observed_model, reason, "model") || !DPCCDatasetReadSource(central.training_path, common_files, central.training_sha256, training_source, observed_training, reason, "training") || !DPCCDatasetReadSource(central.profile_path, common_files, central.profile_sha256, profile_source, observed_profile, reason, "profile")) return(DPCCDatasetFail(result, reason, reason)); CDPCCJsonDocument model_document, training_document, profile_document; if(!DPCCCentralSchemaValidate(model_document, model_source, model_schema, reason, "model") || !DPCCCentralSchemaValidate(training_document, training_source, training_schema, reason, "training") || !DPCCCentralSchemaValidate(profile_document, profile_source, profile_schema, reason, "profile")) return(DPCCDatasetFail(result, reason, reason)); SDPCCDatasetContinuation continuation; SDPCCResourceProfile resource_profile; SDPCCDataProfile data_profile; if(!DPCCDecodeDatasetContinuation(model_document, training_document, profile_document, continuation, reason) || !DPCCDecodeResourceProfile(profile_document, profile_document.Root(), resource_profile, reason) || !DPCCDecodeDataProfile(profile_document, profile_document.Root(), data_profile, reason)) return(DPCCDatasetFail(result, reason, reason)); string profile_relative = "", model_relative = ""; if(!DPCCDatasetRelativePath(package_root_relative, central.profile_path, profile_relative, reason) || !DPCCDatasetRelativePath(package_root_relative, central.model_path, model_relative, reason)) return(DPCCDatasetFail(result, reason, reason)); SDPCCArtifactResolveResult dataset_artifact, model_dataset_artifact; if(!DPCCResolveArtifact(package_root_relative, profile_relative, data_profile.dataset_manifest, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, dataset_artifact, reason)) return(DPCCDatasetFail(result, reason, "dataset.sha256")); if(!DPCCResolveArtifact(package_root_relative, model_relative, continuation.model_dataset, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, model_dataset_artifact, reason)) return(DPCCDatasetFail(result, reason, "model.dataset_manifest.sha256")); string dataset_source = ""; if(!DPCCCentralBytesSource(dataset_artifact.bytes, dataset_source, reason, "dataset")) return(DPCCDatasetFail(result, reason, reason)); CDPCCJsonDocument dataset_document; if(!DPCCCentralSchemaValidate(dataset_document, dataset_source, dataset_schema, reason, "dataset")) return(DPCCDatasetFail(result, reason, reason)); SDPCCDatasetManifest dataset; if(!DPCCDecodeDatasetManifest(dataset_document, dataset_document.Root(), dataset, reason)) return(DPCCDatasetFail(result, reason, reason)); string dataset_relative = ""; if(!DPCCDatasetRelativePath(package_root_relative, dataset_artifact.canonical_path, dataset_relative, reason)) return(DPCCDatasetFail(result, reason, reason)); SDPCCArtifactResolveResult index_artifact; if(!DPCCResolveArtifact(package_root_relative, dataset_relative, dataset.time_index, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, index_artifact, reason)) return(DPCCDatasetFail(result, reason, "time_index.sha256")); string index_source = ""; if(!DPCCCentralBytesSource(index_artifact.bytes, index_source, reason, "time_index")) return(DPCCDatasetFail(result, reason, reason)); CDPCCJsonDocument index_document; if(!DPCCCentralSchemaValidate(index_document, index_source, time_index_schema, reason, "time_index")) return(DPCCDatasetFail(result, reason, reason)); SDPCCDatasetTimeIndex time_index; if(!DPCCDecodeDatasetTimeIndex(index_document, index_document.Root(), dataset.dataset_id, time_index, reason)) return(DPCCDatasetFail(result, reason, reason)); string index_relative = ""; if(!DPCCDatasetRelativePath(package_root_relative, index_artifact.canonical_path, index_relative, reason)) return(DPCCDatasetFail(result, reason, reason)); SDPCCResourceReference index_timezone; index_timezone.path = time_index.timezone_mapping.path; index_timezone.sha256 = time_index.timezone_mapping.sha256; index_timezone.id = time_index.timezone_mapping.id; SDPCCArtifactResolveResult dataset_timezone, index_timezone_artifact; if(!DPCCResolveArtifact(package_root_relative, dataset_relative, dataset.timezone_mapping, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, dataset_timezone, reason) || !DPCCResolveArtifact(package_root_relative, index_relative, index_timezone, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, index_timezone_artifact, reason)) return(DPCCDatasetFail(result, reason, "timezone.sha256")); if(dataset_timezone.canonical_path != index_timezone_artifact.canonical_path) return(DPCCDatasetFail(result, reason, "dataset.timezone_mapping.path")); string evidence_path = ""; if(!DPCCNormalizePackagePath(package_root_relative, "root.json", evidence_package_path, evidence_path, reason)) return(DPCCDatasetFail(result, reason, reason)); string evidence_source = "", observed_evidence = ""; if(!DPCCDatasetReadSource(evidence_path, common_files, evidence_sha256, evidence_source, observed_evidence, reason, "evidence")) return(DPCCDatasetFail(result, reason, reason)); CDPCCJsonDocument evidence_document; if(!DPCCCentralSchemaValidate(evidence_document, evidence_source, evidence_schema, reason, "evidence")) return(DPCCDatasetFail(result, reason, reason)); SDPCCDataEvidence evidence; if(!DPCCDecodeDataEvidence(evidence_document, evidence_document.Root(), evidence, reason)) return(DPCCDatasetFail(result, reason, reason)); SDPCCArtifactResolveResult evidence_profile, evidence_training, evidence_dataset; if(!DPCCResolveArtifact(package_root_relative, evidence_package_path, evidence.readiness_profile, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, evidence_profile, reason) || !DPCCResolveArtifact(package_root_relative, evidence_package_path, evidence.training_config, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, evidence_training, reason) || !DPCCResolveArtifact(package_root_relative, evidence_package_path, evidence.dataset_manifest, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, evidence_dataset, reason)) return(DPCCDatasetFail(result, reason, "evidence.reference")); SDPCCArtifactResolveResult rho_artifact, evidence_rho; if(!DPCCResolveArtifact(package_root_relative, profile_relative, data_profile.rho_implementation, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, rho_artifact, reason)) return(DPCCDatasetFail(result, reason, "rho.sha256")); if(!DPCCResolveArtifact(package_root_relative, evidence_package_path, evidence.rho_implementation, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, evidence_rho, reason)) return(DPCCDatasetFail(result, reason, "rho.sha256")); if(!DPCCValidateDatasetTimeClosure(dataset, time_index, resource_profile, continuation.rho_k, production, reason)) return(DPCCDatasetFail(result, reason, reason)); SDPCCDataBindings bindings; if(!DPCCValidateDatasetEvidenceClosure(central, continuation, dataset, data_profile, evidence, dataset_artifact.observed_sha256, rho_artifact.observed_sha256, bank_id, bank_sha256, production, bindings, verdict, reason)) return(DPCCDatasetFail(result, reason, reason)); result.dataset_id = dataset.dataset_id; result.time_index_id = time_index.time_index_id; result.evidence_id = evidence.evidence_id; result.rho_id = data_profile.rho_implementation.id; result.dataset_path = dataset_artifact.canonical_path; result.time_index_path = index_artifact.canonical_path; result.evidence_path = evidence_path; result.rho_path = rho_artifact.canonical_path; result.dataset_sha256 = dataset_artifact.observed_sha256; result.time_index_sha256 = index_artifact.observed_sha256; result.evidence_sha256 = observed_evidence; result.rho_sha256 = rho_artifact.observed_sha256; return(true); } #endif struct SDPCCBankMetadataResult { string bank_id; string dataset_id; string time_index_id; string header_sha256; string bank_sha256; long record_count; }; //+------------------------------------------------------------------+ //| Function DPCCClearBankMetadataResult | //+------------------------------------------------------------------+ void DPCCClearBankMetadataResult(SDPCCBankMetadataResult &result) { result.bank_id = ""; result.dataset_id = ""; result.time_index_id = ""; result.header_sha256 = ""; result.bank_sha256 = ""; result.record_count = 0; } //+------------------------------------------------------------------+ //| Function DPCCBankMetadataFail | //+------------------------------------------------------------------+ bool DPCCBankMetadataFail(SDPCCBankMetadataResult &result, string &reason, const string failure) { DPCCClearBankMetadataResult(result); reason = failure; return(false); } //+------------------------------------------------------------------+ //| Function DPCCBankParseExactNumber | //+------------------------------------------------------------------+ bool DPCCBankParseExactNumber(const string source, bool &negative, string &digits, long &scale) { negative = false; digits = ""; scale = 0; const int length = StringLen(source); if(length <= 0) return(false); int start = 0; if(StringGetCharacter(source, 0) == '-') { negative = true; start = 1; } int exponent_pos = -1; for(int i = start; i < length; i++) { const ushort ch = StringGetCharacter(source, i); if(ch == 'e' || ch == 'E') { exponent_pos = i; break; } } const int mantissa_end = (exponent_pos < 0 ? length : exponent_pos); bool decimal_seen = false, has_digit = false; long fractional_digits = 0; for(int i = start; i < mantissa_end; i++) { const ushort ch = StringGetCharacter(source, i); if(ch == '.') { if(decimal_seen) return(false); decimal_seen = true; continue; } if(ch < '0' || ch > '9') return(false); has_digit = true; if(decimal_seen) fractional_digits++; } if(!has_digit) return(false); digits = StringSubstr(source, start, mantissa_end - start); StringReplace(digits, ".", ""); long exponent = 0; if(exponent_pos >= 0) { int pos = exponent_pos + 1; bool exponent_negative = false; if(pos < length && (StringGetCharacter(source, pos) == '+' || StringGetCharacter(source, pos) == '-')) { exponent_negative = (StringGetCharacter(source, pos) == '-'); pos++; } if(pos >= length) return(false); const long maximum = 9000000000000000000; long magnitude = 0; for(; pos < length; pos++) { const ushort ch = StringGetCharacter(source, pos); if(ch < '0' || ch > '9') return(false); const int digit = (int)(ch - '0'); if(magnitude > (maximum - digit) / 10) return(false); magnitude = magnitude * 10 + digit; } exponent = (exponent_negative ? -magnitude : magnitude); } const long maximum_scale = 9000000000000000000; if(exponent < -maximum_scale + fractional_digits) return(false); scale = exponent - fractional_digits; int leading = 0; const int digit_length = StringLen(digits); while(leading < digit_length - 1 && StringGetCharacter(digits, leading) == '0') leading++; if(leading > 0) digits = StringSubstr(digits, leading); if(digits == "0") { negative = false; scale = 0; return(true); } const int normalized_length = StringLen(digits); int trailing = 0; while(trailing < normalized_length && StringGetCharacter(digits, normalized_length - 1 - trailing) == '0') trailing++; if(trailing > 0) { if(scale > maximum_scale - trailing) return(false); digits = StringSubstr(digits, 0, normalized_length - trailing); scale += trailing; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankExactNumberEqual | //+------------------------------------------------------------------+ bool DPCCBankExactNumberEqual(const string left, const string right) { bool left_negative = false, right_negative = false; string left_digits = "", right_digits = ""; long left_scale = 0, right_scale = 0; if(!DPCCBankParseExactNumber(left, left_negative, left_digits, left_scale) || !DPCCBankParseExactNumber(right, right_negative, right_digits, right_scale)) return(false); return(left_negative == right_negative && left_digits == right_digits && left_scale == right_scale); } //+------------------------------------------------------------------+ //| Function DPCCBankJsonEqual | //+------------------------------------------------------------------+ bool DPCCBankJsonEqual(CDPCCJsonDocument &left, const int left_node, CDPCCJsonDocument &right, const int right_node, const int depth) { if(depth > DPCC_JSON_MAX_DEPTH) return(false); const ENUM_DPCC_JSON_TYPE left_type = left.Type(left_node); const ENUM_DPCC_JSON_TYPE right_type = right.Type(right_node); if(left_type != right_type || left_type == DPCC_JSON_INVALID) return(false); if(left_type == DPCC_JSON_NULL) return(true); if(left_type == DPCC_JSON_BOOLEAN || left_type == DPCC_JSON_STRING) return(left.Text(left_node) == right.Text(right_node)); if(left_type == DPCC_JSON_NUMBER) return(DPCCBankExactNumberEqual(left.Text(left_node), right.Text(right_node))); if(left_type == DPCC_JSON_ARRAY) { if(left.ChildCount(left_node) != right.ChildCount(right_node)) return(false); int left_child = left.FirstChild(left_node); int right_child = right.FirstChild(right_node); while(left_child >= 0 && right_child >= 0) { if(!DPCCBankJsonEqual(left, left_child, right, right_child, depth + 1)) return(false); left_child = left.NextSibling(left_child); right_child = right.NextSibling(right_child); } return(left_child < 0 && right_child < 0); } if(left_type == DPCC_JSON_OBJECT) { if(left.ChildCount(left_node) != right.ChildCount(right_node)) return(false); for(int member = left.FirstChild(left_node); member >= 0; member = left.NextSibling(member)) { const int other = right.FindMember(right_node, left.Key(member)); if(other < 0 || !DPCCBankJsonEqual(left, member, right, other, depth + 1)) return(false); } return(true); } return(false); } //+------------------------------------------------------------------+ //| Function DPCCBankShapeEqual | //+------------------------------------------------------------------+ bool DPCCBankShapeEqual(const SDPCCResourceShape &left, const SDPCCResourceShape &right) { return(left.K == right.K && left.V == right.V && left.H == right.H && left.D == right.D && left.M == right.M && left.J == right.J && left.E == right.E && left.A == right.A && left.B == right.B && left.account_features == right.account_features); } //+------------------------------------------------------------------+ //| Function DPCCBankReadReference | //+------------------------------------------------------------------+ bool DPCCBankReadReference(CDPCCJsonDocument &document, const int object, const string name, SDPCCResourceReference &reference, string &reason, const string path) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_OBJECT || !DPCCResourceReadString(document, node, "path", reference.path, reason, path + ".path") || !DPCCResourceReadString(document, node, "sha256", reference.sha256, reason, path + ".sha256") || !DPCCResourceReadString(document, node, "id", reference.id, reason, path + ".id")) { if(reason == "") reason = path; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankReadString | //+------------------------------------------------------------------+ bool DPCCBankReadString(CDPCCJsonDocument &document, const int object, const string name, string &value, string &reason, const string path) { return(DPCCResourceReadString(document, object, name, value, reason, path)); } //+------------------------------------------------------------------+ //| Function DPCCBankSourceHash | //+------------------------------------------------------------------+ bool DPCCBankSourceHash(const string source, string &digest, string &reason, const string role) { uchar bytes[]; const int count = StringToCharArray(source, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(count <= 0) { reason = role + ".bytes"; return(false); } const int length = (bytes[count - 1] == 0 ? count - 1 : count); if(ArrayResize(bytes, length) != length || !DPCCSHA256Lower(bytes, digest, reason)) { reason = role + ".sha256"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankCheckShape | //+------------------------------------------------------------------+ bool DPCCBankCheckShape(CDPCCJsonDocument &header, const int header_root, const SDPCCResourceShape &expected, string &reason) { SDPCCResourceShape shape; if(!DPCCDecodeResourceShape(header, header.FindMember(header_root, "shapes"), shape, reason, "header.shapes")) return(false); if(!DPCCBankShapeEqual(shape, expected)) { reason = "header.shapes.dataset"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankCheckPriceContract | //+------------------------------------------------------------------+ bool DPCCBankCheckPriceContract(CDPCCJsonDocument &header, CDPCCJsonDocument &dataset, string &reason) { static const string fields[8] = { "chart_price_basis", "execution_price_basis", "quote_adapter", "version", "max_quote_age_seconds", "max_last_age_seconds", "native_quote_cutoff", "future_offset_policy" }; const int header_contract = header.FindMember(header.Root(), "price_contract"); const int dataset_contract = dataset.FindMember(dataset.Root(), "price_contract"); if(header_contract < 0 || dataset_contract < 0 || header.Type(header_contract) != DPCC_JSON_OBJECT || dataset.Type(dataset_contract) != DPCC_JSON_OBJECT) { reason = "price_contract"; return(false); } for(int i = 0; i < ArraySize(fields); i++) { const int header_value = header.FindMember(header_contract, fields[i]); const int dataset_value = dataset.FindMember(dataset_contract, fields[i]); if(header_value < 0 || dataset_value < 0 || !DPCCBankJsonEqual(header, header_value, dataset, dataset_value, 0)) { reason = "price_contract." + fields[i]; return(false); } } return(true); } //--- Observed digest, record count, and expected bank ID are supplied by upstream //--- callers. This function validates those values against metadata; it does not //--- read or authenticate a DPCCBK02 container. bool DPCCValidateBankMetadata(const string bank_manifest_source, const uchar &header_bytes[], const string dataset_source, const string bank_manifest_schema, const string bank_header_schema, const string dataset_schema, const SDPCCDatasetManifest &accepted_dataset, const SDPCCDatasetLinkResult &accepted_links, const SDPCCDataProfile &accepted_profile, const string expected_bank_id, const string observed_bank_sha256, const long observed_record_count, const bool production, SDPCCBankMetadataResult &result, string &reason) { DPCCClearBankMetadataResult(result); reason = ""; if(expected_bank_id == "" || !DPCCResolverLowerSha256(observed_bank_sha256) || observed_record_count < 0) return(DPCCBankMetadataFail(result, reason, "observation.invalid")); string header_source = ""; if(!DPCCCentralBytesSource(header_bytes, header_source, reason, "header")) return(DPCCBankMetadataFail(result, reason, reason)); string header_sha256 = ""; if(!DPCCSHA256Lower(header_bytes, header_sha256, reason)) return(DPCCBankMetadataFail(result, reason, "header.sha256")); CDPCCJsonDocument bank_document, header_document, dataset_document; if(!DPCCCentralSchemaValidate(bank_document, bank_manifest_source, bank_manifest_schema, reason, "bank_manifest") || !DPCCCentralSchemaValidate(header_document, header_source, bank_header_schema, reason, "header") || !DPCCCentralSchemaValidate(dataset_document, dataset_source, dataset_schema, reason, "dataset")) return(DPCCBankMetadataFail(result, reason, reason)); SDPCCDatasetManifest decoded_dataset; if(!DPCCDecodeDatasetManifest(dataset_document, dataset_document.Root(), decoded_dataset, reason)) return(DPCCBankMetadataFail(result, reason, reason)); string dataset_digest = ""; if(!DPCCBankSourceHash(dataset_source, dataset_digest, reason, "dataset")) return(DPCCBankMetadataFail(result, reason, reason)); if(decoded_dataset.dataset_id != accepted_dataset.dataset_id || decoded_dataset.normalization_id != accepted_dataset.normalization_id || decoded_dataset.time_index.id != accepted_dataset.time_index.id || decoded_dataset.time_index.sha256 != accepted_dataset.time_index.sha256 || !DPCCBankShapeEqual(decoded_dataset.shapes, accepted_dataset.shapes)) return(DPCCBankMetadataFail(result, reason, "dataset.accepted_contract")); if(decoded_dataset.dataset_id != accepted_links.dataset_id || decoded_dataset.time_index.id != accepted_links.time_index_id || decoded_dataset.time_index.sha256 != accepted_links.time_index_sha256 || accepted_links.dataset_sha256 != dataset_digest || accepted_profile.dataset_manifest.id != accepted_links.dataset_id || accepted_profile.dataset_manifest.sha256 != accepted_links.dataset_sha256 || dataset_digest != accepted_links.dataset_sha256 || accepted_profile.forecast_id == "") return(DPCCBankMetadataFail(result, reason, "dataset.accepted_identity")); if(production && (decoded_dataset.example_only || accepted_profile.example_only)) return(DPCCBankMetadataFail(result, reason, "dataset.example_only")); const int bank_root = bank_document.Root(); const int header_root = header_document.Root(); bool bank_example = false, header_example = false; if(!DPCCCentralBool(bank_document, bank_root, "example_only", bank_example, reason, "bank_manifest.example_only") || !DPCCCentralBool(header_document, header_root, "example_only", header_example, reason, "header.example_only")) return(DPCCBankMetadataFail(result, reason, reason)); if(production && (bank_example || header_example)) return(DPCCBankMetadataFail(result, reason, "bank.example_only")); const int embedded_header = bank_document.FindMember(bank_root, "header"); if(embedded_header < 0 || !DPCCBankJsonEqual(bank_document, embedded_header, header_document, header_root, 0)) return(DPCCBankMetadataFail(result, reason, "bank_manifest.header.observed")); string manifest_bank_id = "", manifest_dataset_id = "", manifest_forecast_id = ""; string manifest_normalization_id = "", file_sha256 = ""; if(!DPCCBankReadString(bank_document, bank_root, "bank_id", manifest_bank_id, reason, "bank_manifest.bank_id") || !DPCCBankReadString(bank_document, bank_root, "forecast_id", manifest_forecast_id, reason, "bank_manifest.forecast_id") || !DPCCBankReadString(bank_document, bank_root, "normalization_id", manifest_normalization_id, reason, "bank_manifest.normalization_id")) return(DPCCBankMetadataFail(result, reason, reason)); SDPCCResourceReference dataset_reference, file_reference; if(!DPCCBankReadReference(bank_document, bank_root, "dataset_manifest", dataset_reference, reason, "bank_manifest.dataset_manifest") || !DPCCBankReadReference(bank_document, bank_root, "file", file_reference, reason, "bank_manifest.file")) return(DPCCBankMetadataFail(result, reason, reason)); manifest_dataset_id = dataset_reference.id; file_sha256 = file_reference.sha256; long declared_count = 0; if(!DPCCReadExactNonnegativeLong(bank_document, bank_document.FindMember(bank_root, "record_count"), declared_count)) return(DPCCBankMetadataFail(result, reason, "bank_manifest.record_count")); string header_bank_id = "", header_dataset_id = "", header_forecast_id = "", header_normalization_id = ""; if(!DPCCBankReadString(header_document, header_root, "bank_id", header_bank_id, reason, "header.bank_id") || !DPCCBankReadString(header_document, header_root, "dataset_id", header_dataset_id, reason, "header.dataset_id") || !DPCCBankReadString(header_document, header_root, "forecast_id", header_forecast_id, reason, "header.forecast_id") || !DPCCBankReadString(header_document, header_root, "normalization_id", header_normalization_id, reason, "header.normalization_id")) return(DPCCBankMetadataFail(result, reason, reason)); if(manifest_bank_id != expected_bank_id || header_bank_id != expected_bank_id) return(DPCCBankMetadataFail(result, reason, "bank_id.expected")); if(manifest_dataset_id != accepted_links.dataset_id || dataset_reference.sha256 != accepted_links.dataset_sha256) return(DPCCBankMetadataFail(result, reason, "bank_manifest.dataset_manifest")); if(header_dataset_id != decoded_dataset.dataset_id) return(DPCCBankMetadataFail(result, reason, "header.dataset_id")); if(manifest_forecast_id != accepted_profile.forecast_id || header_forecast_id != accepted_profile.forecast_id) return(DPCCBankMetadataFail(result, reason, "forecast_id")); if(manifest_normalization_id != decoded_dataset.normalization_id || header_normalization_id != decoded_dataset.normalization_id) return(DPCCBankMetadataFail(result, reason, "normalization_id")); if(!DPCCBankCheckShape(header_document, header_root, decoded_dataset.shapes, reason) || !DPCCBankCheckPriceContract(header_document, dataset_document, reason)) return(DPCCBankMetadataFail(result, reason, reason)); if(file_sha256 != observed_bank_sha256) return(DPCCBankMetadataFail(result, reason, "bank_manifest.file.sha256")); if(declared_count != observed_record_count) return(DPCCBankMetadataFail(result, reason, "bank_manifest.record_count.observed")); result.bank_id = expected_bank_id; result.dataset_id = decoded_dataset.dataset_id; result.time_index_id = accepted_links.time_index_id; result.header_sha256 = header_sha256; result.bank_sha256 = observed_bank_sha256; result.record_count = observed_record_count; return(true); } #endif #ifndef DPCC_CLOCK_SOURCE_SCHEMA_MQH #define DPCC_CLOCK_SOURCE_SCHEMA_MQH string DPCCClockSourceSchema(void) { return("{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"title\":\"Scoped broker UTC historical policy or account-observed mapping\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"schema_version\":{\"const\":1},\"mapping_id\":{\"type\":\"string\",\"minLength\":1},\"dataset_id\":{\"type\":\"string\",\"minLength\":1},\"time_index_id\":{\"type\":\"string\",\"minLength\":1},\"broker_server\":{\"type\":\"string\",\"minLength\":1},\"account_login\":{\"type\":\"integer\",\"minimum\":1},\"broker_domain\":{\"const\":\"BROKER_SERVER_SECONDS_V1\"},\"utc_domain\":{\"const\":\"UTC_SECONDS_V1\"},\"example_only\":{\"type\":\"boolean\"},\"origin\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"id\":{\"type\":\"string\",\"minLength\":1},\"path\":{\"type\":\"string\",\"minLength\":1},\"sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"}},\"required\":[\"id\",\"path\",\"sha256\"]},\"intervals\":{\"type\":\"array\",\"minItems\":1,\"items\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"id\":{\"type\":\"string\",\"minLength\":1},\"session_id\":{\"type\":\"string\",\"minLength\":1},\"server_start\":{\"type\":\"integer\",\"minimum\":0},\"server_end\":{\"type\":\"integer\",\"minimum\":0},\"utc_start\":{\"type\":\"integer\",\"minimum\":0},\"utc_end\":{\"type\":\"integer\",\"minimum\":0},\"offset_seconds\":{\"type\":\"integer\"}},\"required\":[\"id\",\"session_id\",\"server_start\",\"server_end\",\"utc_start\",\"utc_end\",\"offset_seconds\"]}},\"bars\":{\"type\":\"array\",\"minItems\":1,\"items\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"bar_index\":{\"type\":\"integer\",\"minimum\":0},\"server_open\":{\"type\":\"integer\",\"minimum\":0},\"server_close\":{\"type\":\"integer\",\"minimum\":0},\"interval_id\":{\"type\":\"string\",\"minLength\":1}},\"required\":[\"bar_index\",\"server_open\",\"server_close\",\"interval_id\"]}},\"observations\":{\"type\":\"array\",\"minItems\":0,\"items\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"id\":{\"type\":\"string\",\"minLength\":1},\"interval_id\":{\"type\":\"string\",\"minLength\":1},\"server_second\":{\"type\":\"integer\",\"minimum\":0},\"server_msc\":{\"type\":\"integer\",\"minimum\":0},\"utc_second\":{\"type\":\"integer\",\"minimum\":0},\"utc_msc\":{\"type\":\"integer\",\"minimum\":0}},\"required\":[\"id\",\"interval_id\",\"server_second\",\"server_msc\",\"utc_second\",\"utc_msc\"]}},\"mapping_scope\":{\"type\":\"string\",\"enum\":[\"ACCOUNT_OBSERVED_V1\",\"HISTORICAL_POLICY_V1\"]}},\"required\":[\"schema_version\",\"mapping_id\",\"dataset_id\",\"time_index_id\",\"broker_server\",\"broker_domain\",\"utc_domain\",\"example_only\",\"origin\",\"intervals\",\"bars\",\"observations\"],\"anyOf\":[{\"required\":[\"account_login\"]},{\"required\":[\"mapping_scope\",\"observations\"],\"properties\":{\"mapping_scope\":{\"const\":\"HISTORICAL_POLICY_V1\"},\"observations\":{\"maxItems\":0}}}]}"); } #endif struct SDPCCClockIdentity { string broker_server, broker_domain; long account_login; }; struct SDPCCClockInterval { string id, session_id; long server_start, server_end, utc_start, utc_end, offset_seconds; }; struct SDPCCClockBar { long ordinal, server_open, server_close; string interval_id; }; struct SDPCCClockObservation { string id, interval_id; long server_second, server_msc, utc_second, utc_msc; }; //--- Host time conversion only. Does not capture an account, reconstruct history, //--- authorize an order, or authenticate an externally reviewed mapping origin. class CDPCCClockSource { private: bool m_valid, m_synthetic, m_historical_policy; SDPCCClockIdentity m_identity; SDPCCDatasetTimeIndex m_index; SDPCCClockInterval m_intervals[]; SDPCCClockBar m_bars[]; SDPCCClockObservation m_observations[]; string m_map_path, m_map_sha, m_index_path, m_index_sha, m_origin_path, m_origin_sha; bool m_common; bool Fail(const string cause, string &reason) { reason = "BLOCKED.clock." + cause; return(false); } bool Signed(CDPCCJsonDocument &d, const int node, long &value, string &reason) { if(d.Type(node) != DPCC_JSON_NUMBER) return(Fail("signed_integer", reason)); string text = d.Text(node); bool negative = StringSubstr(text, 0, 1) == "-"; if(negative) text = StringSubstr(text, 1); ulong magnitude = 0; if(!DPCCCheckpointReadUnsigned(text, (ulong)LONG_MAX, magnitude)) return(Fail("signed_integer", reason)); value = (long)magnitude; if(negative) value = -value; return(true); } bool LongField(CDPCCJsonDocument &d, const int node, const string name, long &value, string &reason) { return(DPCCResourceReadLong(d, node, name, value, reason, "clock." + name)); } bool TextField(CDPCCJsonDocument &d, const int node, const string name, string &value, string &reason) { return(DPCCResourceReadString(d, node, name, value, reason, "clock." + name)); } bool AddOffset(const long value, const long offset, long &result, string &reason) { result = 0; if(value < 0 || (offset >= 0 ? value > LONG_MAX - offset : value < -offset)) return(Fail("offset_range", reason)); result = value + offset; return(true); } int Interval(const string id) { for(int i = 0; i < ArraySize(m_intervals); i++) if(m_intervals[i].id == id) return(i); return(-1); } bool ConvertInternal(const long server_second, long &utc_second, int &interval, string &reason) { utc_second = 0; interval = -1; for(int i = 0; i < ArraySize(m_intervals); i++) if(server_second >= m_intervals[i].server_start && server_second < m_intervals[i].server_end) { if(interval >= 0) return(Fail("ambiguous_server_or_DST_fold", reason)); interval = i; } if(interval < 0) return(Fail("uncovered_server_session_or_DST_gap", reason)); if(!AddOffset(server_second, -m_intervals[interval].offset_seconds, utc_second, reason)) return(false); if(utc_second < m_intervals[interval].utc_start || utc_second >= m_intervals[interval].utc_end) return(Fail("converted_interval", reason)); return(true); } public: CDPCCClockSource(void) {m_valid = false; m_synthetic = false; m_historical_policy = false; m_common = false;} void Reset(void) { m_valid = false; m_synthetic = false; m_historical_policy = false; ArrayFree(m_intervals); ArrayFree(m_bars); ArrayFree(m_observations); DPCCClearDatasetTimeIndex(m_index); ZeroMemory(m_identity); m_map_path = ""; m_map_sha = ""; m_index_path = ""; m_index_sha = ""; m_origin_path = ""; m_origin_sha = ""; } private: bool InitializeImpl(const string package_root, const string containing_json, const SDPCCResourceReference &map_ref, const SDPCCResourceReference &index_ref, const SDPCCResourceReference &reviewed_origin, const SDPCCClockIdentity &identity, const bool common_files, const bool directed_synthetic, const bool historical_policy, string &reason) { if(m_valid) return(Fail("explicit_reset_before_reinitialize", reason)); Reset(); reason = ""; if(StringLen(identity.broker_server) == 0 || identity.broker_domain != "BROKER_SERVER_SECONDS_V1" || (historical_policy ? identity.account_login != 0 : identity.account_login <= 0) || StringLen( reviewed_origin.id) == 0 || !DPCCResolverLowerSha256(reviewed_origin.sha256)) return(Fail("reviewed_origin_and_observed_identity", reason)); SDPCCArtifactResolveResult map_artifact, index_artifact, origin_artifact; string raw = "", index_raw = ""; CDPCCJsonDocument d, index_document; if(!DPCCResolveArtifact(package_root, containing_json, map_ref, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, map_artifact, reason) || !DPCCCentralBytesSource(map_artifact.bytes, raw, reason, "clock.map") || !DPCCCentralSchemaValidate(d, raw, DPCCClockSourceSchema(), reason, "clock.map") || !DPCCResolveArtifact(package_root, containing_json, index_ref, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, index_artifact, reason) || !DPCCCentralBytesSource(index_artifact.bytes, index_raw, reason, "clock.index") || !index_document.Parse(index_raw, reason)) return(false); const int root = d.Root(); string id = "", dataset_id = "", index_id = "", utc_domain = ""; const int scope_node = d.FindMember(root, "mapping_scope"); string mapping_scope = "ACCOUNT_OBSERVED_V1"; if(scope_node >= 0 && !TextField(d, root, "mapping_scope", mapping_scope, reason)) return(false); bool example = false; SDPCCClockIdentity decoded; ZeroMemory(decoded); SDPCCResourceReference origin, linked_map; if(!TextField(d, root, "mapping_id", id, reason) || !TextField(d, root, "dataset_id", dataset_id, reason) || !TextField(d, root, "time_index_id", index_id, reason) || !TextField(d, root, "broker_server", decoded.broker_server, reason) || !TextField(d, root, "broker_domain", decoded.broker_domain, reason) || !TextField(d, root, "utc_domain", utc_domain, reason) || !DPCCResourceReadBool(d, root, "example_only", example, reason, "clock.example_only") || !DPCCBankReadReference(d, root, "origin", origin, reason, "clock.origin") || !DPCCDecodeDatasetTimeIndex(index_document, index_document.Root(), dataset_id, m_index, reason)) return(false); if(historical_policy) { if(mapping_scope != "HISTORICAL_POLICY_V1" || identity.account_login != 0 || d.FindMember(root, "account_login") >= 0) return(Fail("historical_policy_scope_or_account_login", reason)); } else if(mapping_scope != "ACCOUNT_OBSERVED_V1" || identity.account_login <= 0 || !LongField(d, root, "account_login", decoded.account_login, reason)) return(Fail("account_observed_scope_and_login", reason)); if(decoded.broker_server != identity.broker_server || decoded.account_login != identity.account_login || decoded.broker_domain != identity.broker_domain || utc_domain != "UTC_SECONDS_V1" || id != map_ref.id || index_id != index_ref.id || index_id != m_index.time_index_id || id != m_index.timezone_mapping.id || map_ref.sha256 != m_index.timezone_mapping.sha256 || origin.id != reviewed_origin.id || origin.sha256 != reviewed_origin.sha256 || ((example || m_index.example_only) && !directed_synthetic)) return(Fail("identity_index_origin_or_example_scope", reason)); linked_map.id = m_index.timezone_mapping.id; linked_map.path = m_index.timezone_mapping.path; linked_map.sha256 = m_index.timezone_mapping.sha256; SDPCCArtifactResolveResult linked; string origin_review_path = "", root_entry = ""; if(!DPCCNormalizePackagePath(package_root, "root.json", "root.json", root_entry, reason)) return(false); const int prefix = StringLen(root_entry) - StringLen("root.json"); const string map_container = StringSubstr(map_artifact.canonical_path, prefix); const string index_container = StringSubstr(index_artifact.canonical_path, prefix); if(!DPCCResolveArtifact(package_root, index_container, linked_map, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, linked, reason) || linked.canonical_path != map_artifact.canonical_path || !DPCCResolveArtifact(package_root, map_container, origin, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, origin_artifact, reason) || !DPCCNormalizePackagePath(package_root, containing_json, reviewed_origin.path, origin_review_path, reason) || origin_artifact.canonical_path != origin_review_path || ArraySize(origin_artifact.bytes) == 0) return(Fail("actual_linked_source_and_origin_bytes", reason)); const int interval_node = d.FindMember(root, "intervals"), bar_node = d.FindMember(root, "bars"); const int observations = d.FindMember(root, "observations"); const int count = d.ChildCount(interval_node), bars = d.ChildCount(bar_node), pairs = d.ChildCount(observations); if(count < 1 || bars != ArraySize(m_index.bar_index) || pairs < 0 || (historical_policy && pairs != 0) || ArrayResize(m_intervals, count) != count || ArrayResize(m_bars, bars) != bars || ArrayResize(m_observations, pairs) != pairs) return(Fail("source_geometry", reason)); for(int i = 0; i < count; i++) { const int node = d.ChildAt(interval_node, i); SDPCCClockInterval v; if(!TextField(d, node, "id", v.id, reason) || !TextField(d, node, "session_id", v.session_id, reason) || !LongField(d, node, "server_start", v.server_start, reason) || !LongField(d, node, "server_end", v.server_end, reason) || !LongField(d, node, "utc_start", v.utc_start, reason) || !LongField(d, node, "utc_end", v.utc_end, reason) || !Signed(d, d.FindMember(node, "offset_seconds"), v.offset_seconds, reason)) return(false); long start = 0, end = 0; if(v.server_start >= v.server_end || v.utc_start >= v.utc_end || !AddOffset(v.utc_start, v.offset_seconds, start, reason) || !AddOffset(v.utc_end, v.offset_seconds, end, reason) || start != v.server_start || end != v.server_end) return(Fail("dated_offset_arithmetic", reason)); for(int j = 0; j < i; j++) if(v.id == m_intervals[j].id || (v.server_start < m_intervals[j].server_end && m_intervals[j].server_start < v.server_end) || (v.utc_start < m_intervals[j].utc_end && m_intervals[j].utc_start < v.utc_end)) return(Fail("overlap_or_ambiguous_DST", reason)); m_intervals[i] = v; } for(int i = 0; i < bars; i++) { const int node = d.ChildAt(bar_node, i); SDPCCClockBar bar; if(!LongField(d, node, "bar_index", bar.ordinal, reason) || !LongField(d, node, "server_open", bar.server_open, reason) || !LongField(d, node, "server_close", bar.server_close, reason) || !TextField(d, node, "interval_id", bar.interval_id, reason)) return(false); const int interval = Interval(bar.interval_id); long open = 0, close = 0; if(bar.ordinal != i || interval < 0 || bar.server_open >= bar.server_close || bar.server_open < m_intervals[interval].server_start || bar.server_close > m_intervals[interval].server_end || !AddOffset(bar.server_open, -m_intervals[interval].offset_seconds, open, reason) || !AddOffset(bar.server_close, -m_intervals[interval].offset_seconds, close, reason) || open != m_index.start_utc[i] || close != m_index.end_utc[i]) return(Fail("exact_indexed_bar_or_cross_interval", reason)); m_bars[i] = bar; } for(int i = 0; i < pairs; i++) { const int node = d.ChildAt(observations, i); SDPCCClockObservation pair; if(!TextField(d, node, "id", pair.id, reason) || !TextField(d, node, "interval_id", pair.interval_id, reason) || !LongField(d, node, "server_second", pair.server_second, reason) || !LongField(d, node, "server_msc", pair.server_msc, reason) || !LongField(d, node, "utc_second", pair.utc_second, reason) || !LongField(d, node, "utc_msc", pair.utc_msc, reason)) return(false); long utc = 0; int interval = -1; if(!ConvertInternal(pair.server_second, utc, interval, reason) || utc != pair.utc_second || m_intervals[interval].id != pair.interval_id || pair.server_msc / 1000 != pair.server_second || pair.utc_msc / 1000 != pair.utc_second || pair.server_msc % 1000 != pair.utc_msc % 1000) return(Fail("paired_rawserver_UTC_observation", reason)); for(int j = 0; j < i; j++) if(pair.id == m_observations[j].id) return(Fail("duplicate_observation", reason)); m_observations[i] = pair; } m_identity = identity; m_synthetic = directed_synthetic; m_historical_policy = historical_policy; m_common = common_files; m_map_path = map_artifact.canonical_path; m_map_sha = map_ref.sha256; m_index_path = index_artifact.canonical_path; m_index_sha = index_ref.sha256; m_origin_path = origin_artifact.canonical_path; m_origin_sha = origin.sha256; if(!VerifySource(reason)) return(false); m_valid = true; reason = ""; return(true); } public: bool Initialize(const string package_root, const string containing_json, const SDPCCResourceReference &map_ref, const SDPCCResourceReference &index_ref, const SDPCCResourceReference &reviewed_origin, const SDPCCClockIdentity &identity, const bool common_files, const bool directed_synthetic, string &reason) { return(InitializeImpl(package_root, containing_json, map_ref, index_ref, reviewed_origin, identity, common_files, directed_synthetic, false, reason)); } bool InitializeHistoricalPolicy(const string package_root, const string containing_json, const SDPCCResourceReference &map_ref, const SDPCCResourceReference &index_ref, const SDPCCResourceReference &reviewed_origin, const SDPCCClockIdentity &identity, const bool common_files, string &reason) { return(InitializeImpl(package_root, containing_json, map_ref, index_ref, reviewed_origin, identity, common_files, false, true, reason)); } bool VerifySource(string &reason) { string observed = ""; if(StringLen(m_map_sha) == 0 || !DPCCCheckpointFileSHA(m_map_path, m_common, DPCC_RESOLVER_MAX_JSON_BYTES, observed, reason) || observed != m_map_sha || !DPCCCheckpointFileSHA(m_index_path, m_common, DPCC_RESOLVER_MAX_JSON_BYTES, observed, reason) || observed != m_index_sha || !DPCCCheckpointFileSHA(m_origin_path, m_common, DPCC_RESOLVER_MAX_JSON_BYTES, observed, reason) || observed != m_origin_sha) {m_valid = false; return(Fail("retained_source_changed_or_missing", reason));} return(true); } bool Convert(const SDPCCClockIdentity &identity, const long server_second, long &utc_second, string &reason) { utc_second = 0; reason = ""; if(!m_valid || identity.broker_server != m_identity.broker_server || identity.account_login != m_identity.account_login || identity.broker_domain != m_identity.broker_domain) return(Fail("observed_identity_or_not_admitted", reason)); if(!VerifySource(reason)) return(false); int interval = -1; return(ConvertInternal(server_second, utc_second, interval, reason)); } bool Observation(const SDPCCClockIdentity &identity, const string observation_id, const long raw_second, const long raw_msc, const long observed_utc, const long observed_utc_msc, string &reason) { if(m_historical_policy) return(Fail("historical_policy_has_no_observed_pairs", reason)); long converted = 0; if(!Convert(identity, raw_second, converted, reason)) return(false); for(int i = 0; i < ArraySize(m_observations); i++) if(m_observations[i].id == observation_id) { if(m_observations[i].server_second != raw_second || m_observations[i].server_msc != raw_msc || m_observations[i].utc_second != observed_utc || m_observations[i].utc_msc != observed_utc_msc || converted != observed_utc) return(Fail("actual_observation_pair_mismatch", reason)); return(true); } return(Fail("actual_observation_pair_missing", reason)); } bool Bar(const SDPCCClockIdentity &identity, const long ordinal, const long raw_open, const long raw_close, long &open_utc, long &close_utc, string &reason) { open_utc = 0; close_utc = 0; long converted = 0; if(!Convert(identity, raw_open, converted, reason)) return(false); if(ordinal < 0 || ordinal >= ArraySize(m_bars) || m_bars[(int)ordinal].server_open != raw_open || m_bars[(int)ordinal].server_close != raw_close || converted != m_index.start_utc[(int)ordinal]) return(Fail("actual_indexed_bar_pair", reason)); open_utc = m_index.start_utc[(int)ordinal]; close_utc = m_index.end_utc[(int)ordinal]; return(true); } bool Decision(const SDPCCClockIdentity &identity, const string observation_id, const long raw_second, const long raw_msc, const long observed_utc, const long observed_utc_msc, const long ordinal, const long raw_bar_open, const long raw_bar_close, long &decision_utc, string &reason) { decision_utc = 0; if(!Observation(identity, observation_id, raw_second, raw_msc, observed_utc, observed_utc_msc, reason)) return(false); if(ordinal < 0 || ordinal >= ArraySize(m_bars) || m_bars[(int)ordinal].server_open != raw_bar_open || m_bars[(int)ordinal].server_close != raw_bar_close) return(Fail("actual_indexed_bar_pair", reason)); if(observed_utc != m_index.start_utc[(int)ordinal]) return(Fail("missing_exact_bar_open_anchor", reason)); decision_utc = observed_utc; return(true); } bool ReadIndexedBar(const SDPCCClockIdentity &identity, const long ordinal, SDPCCClockBar &bar, long &open_utc, long &close_utc, string &reason) { ZeroMemory(bar); open_utc = 0; close_utc = 0; if(ordinal < 0 || ordinal >= ArraySize(m_bars)) return(Fail("indexed_bar_ordinal", reason)); long converted = 0; if(!Convert(identity, m_bars[(int)ordinal].server_open, converted, reason)) return(false); bar = m_bars[(int)ordinal]; open_utc = m_index.start_utc[(int)ordinal]; close_utc = m_index.end_utc[(int)ordinal]; return(true); } bool ReadMapBinding(string &path, string &sha256, string &id, string &reason) { path = ""; sha256 = ""; id = ""; if(!m_valid || !VerifySource(reason)) return(false); path = m_map_path; sha256 = m_map_sha; id = m_index.timezone_mapping.id; return(true); } bool DirectedSynthetic(void) const {return(m_valid && m_synthetic);} bool HistoricalPolicyOnly(void) const {return(m_valid && m_historical_policy);} }; #endif //--- UTC is derived from the reviewed map; new ticks are not predeclared in that map. struct SDPCCMarketClockCapture { bool native_observed; string symbol, map_path, map_sha256, map_id; ENUM_TIMEFRAMES timeframe; long broker_second, broker_tick_msc, utc_second, utc_tick_msc, decision_bar_index; long broker_bar_open, broker_bar_close; }; //+------------------------------------------------------------------+ //| Declaration CDPCCMarketClock | //+------------------------------------------------------------------+ class CDPCCMarketClock { private: CDPCCClockSource m_source; SDPCCClockIdentity m_identity; SDPCCMarketClockCapture m_capture; bool m_captured; bool m_server_calendar; long m_server_period_seconds; long m_server_opens[]; public: CDPCCMarketClock(void) { m_captured = false; m_server_calendar = false; m_server_period_seconds = 0; ZeroMemory(m_capture); } void Reset(void) { m_captured = false; m_server_calendar = false; m_server_period_seconds = 0; ArrayFree(m_server_opens); ZeroMemory(m_capture); m_source.Reset(); } bool Initialize(const string root, const string container, const SDPCCResourceReference &map_ref, const SDPCCResourceReference &index_ref, const SDPCCResourceReference &origin, const SDPCCClockIdentity &identity, const bool common_files, const bool synthetic, string &reason) { Reset(); if(!m_source.Initialize(root, container, map_ref, index_ref, origin, identity, common_files, synthetic, reason)) { m_source.Reset(); return(false); } m_identity = identity; return(true); } bool InitializeHistoricalPolicy(const string root, const string container, const SDPCCResourceReference &map_ref, const SDPCCResourceReference &index_ref, const SDPCCResourceReference &origin, const SDPCCClockIdentity &identity, const bool common_files, string &reason) { Reset(); if(!m_source.InitializeHistoricalPolicy(root, container, map_ref, index_ref, origin, identity, common_files, reason)) { m_source.Reset(); return(false); } m_identity = identity; return(true); } //--- Raw source authentication belongs to the loader; this binds its ordered H1 calendar. bool InitializeHistoricalServerCalendar(const long &opens[], string &reason, const ENUM_TIMEFRAMES timeframe = PERIOD_H1) { Reset(); reason = ""; const int count = ArraySize(opens); const long period_seconds = PeriodSeconds(timeframe); if(count < 1 || timeframe <= PERIOD_CURRENT || period_seconds <= 0) { reason = "BLOCKED.clock_binding.server_calendar_empty"; return(false); } for(int i = 0; i < count; i++) if(opens[i] <= 0 || opens[i] > LONG_MAX - period_seconds || (i > 0 && opens[i] < opens[i - 1] + period_seconds)) { reason = "BLOCKED.clock_binding.server_calendar_order"; return(false); } if(ArrayCopy(m_server_opens, opens) != count) { ArrayFree(m_server_opens); reason = "BLOCKED.clock_binding.server_calendar_capacity"; return(false); } m_server_calendar = true; m_server_period_seconds = period_seconds; return(true); } long ServerPeriodSeconds(void) const { return(m_server_calendar ? m_server_period_seconds : 0); } string TimeDomain(void) { return(m_server_calendar ? "SERVER_CALENDAR_SECONDS_V1" : "UTC_SECONDS_V1"); } bool TimelineSecond(const long raw, long &time, string &reason) { if(!m_server_calendar) return(Convert(raw, time, reason)); time = 0; const int count = ArraySize(m_server_opens); if(count < 1 || m_server_period_seconds <= 0 || raw < m_server_opens[0] || raw > m_server_opens[count - 1] + m_server_period_seconds) { reason = "BLOCKED.clock_binding.server_calendar_range"; return(false); } time = raw; return(true); } bool IndexedTimelineBar(const long ordinal, SDPCCClockBar &bar, long &open_time, long &close_time, string &reason) { if(!m_server_calendar) return(IndexedBar(ordinal, bar, open_time, close_time, reason)); ZeroMemory(bar); open_time = 0; close_time = 0; if(ordinal < 0 || ordinal >= ArraySize(m_server_opens)) { reason = "BLOCKED.clock_binding.server_calendar_ordinal"; return(false); } bar.ordinal = ordinal; bar.server_open = m_server_opens[(int)ordinal]; bar.server_close = bar.server_open + m_server_period_seconds; bar.interval_id = "HISTORICAL_SERVER_CALENDAR_V2"; open_time = bar.server_open; close_time = bar.server_close; return(true); } bool Convert(const long raw, long &utc, string &reason) { if(m_server_calendar) { utc = 0; reason = "BLOCKED.clock_binding.server_calendar_has_no_UTC_authority"; return(false); } return(m_source.Convert(m_identity, raw, utc, reason)); } bool ConvertMilliseconds(const long raw, long &utc, string &reason) { utc = 0; long second = 0; if(raw < 0) { reason = "BLOCKED.clock_binding.negative_millisecond"; return(false); } if(!Convert(raw / 1000, second, reason)) return(false); const long remainder = raw % 1000; if(second > (LONG_MAX - remainder) / 1000) { reason = "BLOCKED.clock_binding.millisecond_overflow"; return(false); } utc = second * 1000 + remainder; return(true); } bool Elapsed(const long start, const long end, long &seconds, string &reason) { seconds = 0; long first = 0, last = 0; if(!Convert(start, first, reason) || !Convert(end, last, reason)) return(false); if(last < first) { reason = "BLOCKED.clock_binding.UTC_endpoint_order"; return(false); } seconds = last - first; return(true); } bool IndexedBar(const long ordinal, SDPCCClockBar &bar, long &open, long &close, string &reason) { if(m_server_calendar) { ZeroMemory(bar); open = 0; close = 0; reason = "BLOCKED.clock_binding.server_calendar_has_no_UTC_authority"; return(false); } return(m_source.ReadIndexedBar(m_identity, ordinal, bar, open, close, reason)); } //--- Pure diagnostic/retained receipt derivation grants no native capture authority. bool Derive(const long raw, const long raw_msc, const long ordinal, const long actual_bar_open, SDPCCMarketClockCapture &out, string &reason) { ZeroMemory(out); if(m_server_calendar || m_source.HistoricalPolicyOnly()) { reason = "BLOCKED.clock_binding.historical_policy_offline_only"; return(false); } long utc = 0, utc_msc = 0, open = 0, close = 0; SDPCCClockBar bar; if(raw <= 0 || raw_msc / 1000 != raw) { reason = "BLOCKED.clock_binding.raw_second_millisecond"; return(false); } if(!Convert(raw, utc, reason) || !ConvertMilliseconds(raw_msc, utc_msc, reason) || !IndexedBar(ordinal, bar, open, close, reason)) return(false); if(raw != actual_bar_open || actual_bar_open != bar.server_open || utc != open) { reason = "BLOCKED.clock_binding.missing_exact_bar_open_anchor"; return(false); } SDPCCMarketClockCapture candidate; ZeroMemory(candidate); candidate.broker_second = raw; candidate.broker_tick_msc = raw_msc; candidate.utc_second = utc; candidate.utc_tick_msc = utc_msc; candidate.decision_bar_index = ordinal; candidate.broker_bar_open = bar.server_open; candidate.broker_bar_close = bar.server_close; if(!m_source.ReadMapBinding(candidate.map_path, candidate.map_sha256, candidate.map_id, reason)) return(false); out = candidate; return(true); } bool Capture(const string symbol, const ENUM_TIMEFRAMES timeframe, const long ordinal, SDPCCMarketClockCapture &out, string &reason) { m_captured = false; ZeroMemory(out); if(m_server_calendar || m_source.HistoricalPolicyOnly() || m_source.DirectedSynthetic() || !TerminalInfoInteger( TERMINAL_CONNECTED) || AccountInfoInteger(ACCOUNT_LOGIN) != m_identity.account_login || AccountInfoString(ACCOUNT_SERVER) != m_identity.broker_server) { reason = "BLOCKED.clock_binding.native_identity_or_synthetic_source"; return(false); } const long raw = (long)TimeCurrent(); const long bar = (long)iTime(symbol, timeframe, 0); MqlTick tick; if(!SymbolInfoTick(symbol, tick)) { reason = "BLOCKED.clock_binding.native_tick_missing"; return(false); } if((long)tick.time != raw || tick.time_msc / 1000 != raw) { reason = "BLOCKED.clock_binding.native_tick_asof"; return(false); } if(!Derive(raw, tick.time_msc, ordinal, bar, out, reason)) return(false); out.symbol = symbol; out.timeframe = timeframe; out.native_observed = true; m_capture = out; m_captured = true; if(!VerifyCurrent(reason)) { ZeroMemory(out); return(false); } return(true); } bool VerifyCurrent(string &reason) { if(m_server_calendar || m_source.HistoricalPolicyOnly()) { m_captured = false; reason = "BLOCKED.clock_binding.historical_policy_offline_only"; return(false); } if(!m_source.VerifySource(reason)) { m_captured = false; return(false); } MqlTick tick; if(!m_captured || !m_capture.native_observed || !TerminalInfoInteger(TERMINAL_CONNECTED) || (long)TimeCurrent() != m_capture.broker_second || AccountInfoInteger(ACCOUNT_LOGIN) != m_identity.account_login || AccountInfoString(ACCOUNT_SERVER) != m_identity.broker_server || (long)iTime(m_capture.symbol, m_capture.timeframe, 0) != m_capture.broker_bar_open || !SymbolInfoTick(m_capture.symbol, tick) || tick.time_msc != m_capture.broker_tick_msc) { m_captured = false; reason = "BLOCKED.clock_binding.capture_changed_or_not_native"; return(false); } return(true); } bool ReadCapture(SDPCCMarketClockCapture &out, string &reason) { ZeroMemory(out); if(!VerifyCurrent(reason)) return(false); out = m_capture; return(true); } bool ClosedBar(const int shift, const long actual_open, long &open_utc, string &reason) { SDPCCMarketClockCapture current; SDPCCClockBar bar; long close = 0; open_utc = 0; if(shift < 1 || !ReadCapture(current, reason) || !IndexedBar(current.decision_bar_index - shift, bar, open_utc, close, reason)) return(false); if(bar.server_open != actual_open || close > current.utc_second) { open_utc = 0; reason = "BLOCKED.clock_binding.closed_bar_index_or_causality"; return(false); } return(true); } }; #endif bool DPCCFinancialContextBlocked(string &reason, const string prerequisite) { reason = "BLOCKED:" + prerequisite; return(false); } //+------------------------------------------------------------------+ //| Advance one source-server civil date to its checked midnight | //+------------------------------------------------------------------+ bool DPCCServerCalendarNextMidnight(const long raw, long &midnight, string &reason) { midnight = 0; MqlDateTime date; if(raw <= 0 || !TimeToStruct((datetime)raw, date) || (long)StructToTime(date) != raw) return(DPCCFinancialContextBlocked(reason, "history.calendar.raw_civil_time")); int days = 31; if(date.mon == 4 || date.mon == 6 || date.mon == 9 || date.mon == 11) days = 30; else if(date.mon == 2) days = ((date.year % 4 == 0 && date.year % 100 != 0) || date.year % 400 == 0 ? 29 : 28); if(date.mon < 1 || date.mon > 12 || date.day < 1 || date.day > days || date.year == INT_MAX) return(DPCCFinancialContextBlocked(reason, "history.calendar.raw_civil_date")); date.hour = 0; date.min = 0; date.sec = 0; if(date.day < days) date.day++; else { date.day = 1; if(date.mon < 12) date.mon++; else { date.mon = 1; date.year++; } } const long candidate = (long)StructToTime(date); MqlDateTime checked; if(candidate <= raw || !TimeToStruct((datetime)candidate, checked) || checked.year != date.year || checked.mon != date.mon || checked.day != date.day || checked.hour != 0 || checked.min != 0 || checked.sec != 0) return(DPCCFinancialContextBlocked(reason, "history.calendar.next_civil_midnight")); midnight = candidate; return(true); } //+------------------------------------------------------------------+ //| Append one bounded cash/reset event in chronological order | //+------------------------------------------------------------------+ bool DPCCAppendHistoricalCalendarEvent(const int h, const int events, const int type, const long event_time, const long from_time, const long open_time, const long close_time, const double buy_cash, const double sell_cash, const double factor, float &counts[], float &calendar[], string &reason) { if(event_time <= from_time || event_time > close_time) return(true); if(h < 0 || h >= ArraySize(counts) || events < 1 || events > INT_MAX / 6 || h > INT_MAX / (events * 6) - 1 || ArraySize(calendar) < (h + 1)*events * 6 || close_time <= open_time || from_time > open_time || (type != 1 && type != 2) || !MathIsValidNumber(counts[h]) || counts[h] < 0 || counts[h] > events || MathFloor(counts[h]) != counts[h] || !MathIsValidNumber(buy_cash) || !MathIsValidNumber(sell_cash) || !MathIsValidNumber(factor) || factor < 0) return(DPCCFinancialContextBlocked(reason, "history.calendar.event_shape_or_numeric")); const int count = (int)counts[h]; if(count >= events) return(DPCCFinancialContextBlocked(reason, "history.calendar.E_exceeded")); const int index = (h * events + count) * 6; const float offset = (float)(event_time - open_time); if(count > 0 && (calendar[index - 6] > offset || (calendar[index - 6] == offset && calendar[index - 5] > (float)type))) return(DPCCFinancialContextBlocked(reason, "history.calendar.event_order")); if(type == 2 && (!MathIsValidNumber((double)(float)buy_cash) || !MathIsValidNumber((double)(float)sell_cash) || !MathIsValidNumber((double)(float)factor))) return(DPCCFinancialContextBlocked(reason, "history.calendar.event_float")); calendar[index] = offset; calendar[index + 1] = (float)type; calendar[index + 2] = (type == 2 ? (float)buy_cash : 0.0f); calendar[index + 3] = (type == 2 ? (float)sell_cash : 0.0f); calendar[index + 4] = (type == 2 ? (float)factor : 0.0f); calendar[index + 5] = 0; counts[h] += 1; return(true); } //+------------------------------------------------------------------+ //| Shared factual clock join and bounded midnight-event packing | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalFinancialRow(const SDPCCConfig &config, const SDPCCHistoricalFinancialProfile &profile, const SDPCCStudyForecastBar &bars[], const int raw, const long decision_bar, const bool episode_first, CDPCCMarketClock &clock, float &context[], float &native_ohlc[], long &timeline_opens[], long &timeline_closes[], string &reason) { if(config.shape.E < DPCC_HISTORY_CALENDAR_MIN_EVENTS) return(DPCCFinancialContextBlocked(reason, "history.calendar.E_below_global_bound")); if((profile.time_domain == "" ? "UTC_SECONDS_V1" : profile.time_domain) != clock.TimeDomain()) return(DPCCFinancialContextBlocked(reason, "history.row.time_domain_mismatch")); const int horizon = (int)config.shape.H; const int events = (int)config.shape.E; const long period_seconds = clock.ServerPeriodSeconds(); if(horizon != 12 || events < 1 || events > INT_MAX / (horizon * 6) || period_seconds <= 0 || raw < 1 || raw > ArraySize(bars) - horizon || !profile.server_midnight) return(DPCCFinancialContextBlocked(reason, "history.calendar.rollover_hour_unconfirmed_or_shape")); float counts[], calendar[]; if(ArrayResize(timeline_opens, horizon) != horizon || ArrayResize(timeline_closes, horizon) != horizon || ArrayResize(counts, horizon) != horizon || ArrayResize(calendar, horizon * events * 6) != horizon * events * 6) return(DPCCFinancialContextBlocked(reason, "history.row.capacity")); ArrayInitialize(counts, 0.0f); ArrayInitialize(calendar, 0.0f); long prior_close = 0, prior_server_close = 0; for(int h = 0; h < horizon; h++) { SDPCCClockBar bar, previous; long previous_open = 0, previous_close = 0; if(!clock.IndexedTimelineBar(decision_bar + h, bar, timeline_opens[h], timeline_closes[h], reason) || bar.ordinal != decision_bar + h || raw + h >= ArraySize(bars) || bar.server_open != bars[raw + h].server_open || bar.server_close - bar.server_open != period_seconds) return(DPCCFinancialContextBlocked(reason, "history.row.raw_clock_period_join")); if(h == 0) { prior_close = timeline_opens[0]; prior_server_close = bar.server_open; if(!episode_first) { if(!clock.IndexedTimelineBar(decision_bar - 1, previous, previous_open, previous_close, reason) || previous.server_open != bars[raw - 1].server_open) return(DPCCFinancialContextBlocked(reason, "history.row.prior_clock_join")); prior_close = previous_close; prior_server_close = previous.server_close; } } const long from_server = (h == 0 ? prior_server_close : bars[raw + h - 1].server_open + period_seconds); const long from_time = (h == 0 ? prior_close : timeline_closes[h - 1]); if(from_server > bar.server_open || from_time > timeline_opens[h] || bar.server_close - from_server > DPCC_HISTORY_CALENDAR_MAX_SPAN_SECONDS) return(DPCCFinancialContextBlocked(reason, "history.calendar.gap_order_or_capacity")); //--- Boundaries belong to the outgoing bar: (previous close,current close]. long midnight = 0; if(!DPCCServerCalendarNextMidnight(from_server, midnight, reason)) return(false); while(midnight <= bar.server_close) { long reset_time = 0, swap_time = 0; if(!clock.TimelineSecond(midnight, reset_time, reason) || !clock.TimelineSecond(midnight - 15, swap_time, reason) || swap_time >= reset_time || reset_time <= from_time || reset_time > timeline_closes[h]) return(DPCCFinancialContextBlocked(reason, "history.calendar.clock_edge")); MqlDateTime day; if(!TimeToStruct((datetime)(midnight - 1), day)) return(DPCCFinancialContextBlocked(reason, "history.calendar.weekday")); const double factor = profile.swap_factors[day.day_of_week]; const bool swap = (factor > 0 && (profile.swap_cash_buy != 0 || profile.swap_cash_sell != 0)); if(swap) { double buy_cash = profile.swap_cash_buy, sell_cash = profile.swap_cash_sell; if(profile.swap_profit_currency) { const int index = raw + h; if(index < 1 || index >= ArraySize(bars)) return(DPCCFinancialContextBlocked(reason, "history.fx.swap_previous_source_bar_required")); if(config.margin_fx_profile != DPCC_MARGIN_HISTORICAL_CLOSE_V1 || ArraySize(profile.fx_source_open) != ArraySize(bars) || ArraySize(profile.fx_frozen) != (long)ArraySize(bars) * 4 || profile.fx_source_open[index] != bars[index - 1].server_open) return(DPCCFinancialContextBlocked(reason, "history.fx.swap_source_join")); const double gain = profile.fx_frozen[index * 4], loss = profile.fx_frozen[index * 4 + 1]; if(!MathIsValidNumber(gain) || gain <= 0 || !MathIsValidNumber(loss) || loss < gain) return(DPCCFinancialContextBlocked(reason, "history.fx.swap_rate_domain")); buy_cash *= buy_cash >= 0 ? gain : loss; sell_cash *= sell_cash >= 0 ? gain : loss; } if(!DPCCAppendHistoricalCalendarEvent(h, events, 2, swap_time, from_time, timeline_opens[h], timeline_closes[h], buy_cash, sell_cash, factor, counts, calendar, reason)) return(false); } if(!DPCCAppendHistoricalCalendarEvent(h, events, 1, reset_time, from_time, timeline_opens[h], timeline_closes[h], 0, 0, 0, counts, calendar, reason)) return(false); if(midnight == bar.server_close) break; long next_midnight = 0; if(!DPCCServerCalendarNextMidnight(midnight, next_midnight, reason)) return(false); midnight = next_midnight; } } if(!DPCCBuildRawFinancialContext(config, profile, bars, raw, timeline_opens, timeline_closes, counts, calendar, prior_close, context, native_ohlc, reason)) return(false); return(true); } //--- A physical single-path rollout is a native CNet, with no host transition. bool DPCCCreateHistoricalRolloutNet(CNet &net, const SDPCCConfig &config, COpenCLMy *context, string &reason) { reason = ""; if(net.Layer(0) != NULL || (context != NULL && CheckPointer(context) == POINTER_INVALID) || config.shape.K != 1 || config.shape.M != 1 || config.shape.Depth != 0 || config.shape.H == 0 || config.shape.H > INT_MAX / 6) {reason = "offline.rollout_factory.destination_or_shape"; return(false);} CArrayObj descriptions; CLayerDescription *input_desc = new CLayerDescription, *layer = new CLayerDescription; if(!input_desc || !layer) { delete input_desc; delete layer; reason = "offline.rollout_factory.allocation"; return(false); } DPCCSetLayerBaseDefaults(input_desc, defNeuronBaseOCL, config.shape.H * 6, 1); DPCCSetLayerBaseDefaults(layer, defNeuronScenarioRolloutOCL, 1, 1); layer.window = 6; layer.window_out = 13; layer.variables = config.shape.V; layer.step = config.shape.H; layer.layers = 0; if(ArrayResize(layer.windows, 9) != 9 || ArrayResize(layer.units, 5) != 5 || ArrayResize(layer.heads, 1) != 1) {delete input_desc; delete layer; reason = "offline.rollout_factory.capacity"; return(false);} const uint windows[] = {2, config.shape.K, config.shape.V, config.shape.H, config.shape.D, config.shape.M, config.shape.J, config.shape.E, config.shape.PlanBatch }; const uint units[] = {config.shape.Embedding, config.shape.Heads, config.shape.KeyDimension, config.shape.NActions, config.shape.StateSize }; for(int i = 0; i < 9; i++) layer.windows[i] = windows[i]; for(int i = 0; i < 5; i++) layer.units[i] = units[i]; layer.heads[0] = config.shape.Heads; if(!descriptions.Add(input_desc)) { delete input_desc; delete layer; reason = "offline.rollout_factory.input"; return(false); } if(!descriptions.Add(layer)) { delete layer; reason = "offline.rollout_factory.layer"; return(false); } if(!net.Create(GetPointer(descriptions)) || (context != NULL && !net.SetOpenCLChecked(context))) {net.Clear(); reason = "offline.rollout_factory.create"; return(false);} CNeuronScenarioRolloutOCL *rollout = dynamic_cast(net.Layer(1)); if(!rollout || !rollout.Configure(config)) { net.Clear(); reason = "offline.rollout_factory.configure"; return(false); } return(true); } struct SDPCCOfflineRowResult { bool committed, floor_reached; ulong next_row; long decision_bar; float state[], account13[], pre_update_plan[], executed_action[], teacher_plan[]; uchar valid_horizons[]; double teacher_score, reward; int teacher_source; SDPCCSimulatedPositionMetadata position; }; //+------------------------------------------------------------------+ //| Function DPCCClearOfflineRowResult | //+------------------------------------------------------------------+ void DPCCClearOfflineRowResult(SDPCCOfflineRowResult &out) { ZeroMemory(out); out.teacher_source = -1; } //+------------------------------------------------------------------+ //| Encodes a float without losing its binary32 identity | //+------------------------------------------------------------------+ string DPCCOfflineCheckpointFloat(const float value) { UDPCCObservedFloatBits bits; bits.value = value; return(StringFormat("%08X", bits.bits)); } //+------------------------------------------------------------------+ //| Strictly decodes one canonical binary32 checkpoint field | //+------------------------------------------------------------------+ bool DPCCOfflineCheckpointReadFloat(const string text, float &value) { if(StringLen(text) != 8) return(false); uint parsed = 0; for(int i = 0; i < 8; i++) { const ushort ch = StringGetCharacter(text, i); uint digit = 0; if(ch >= '0' && ch <= '9') digit = (uint)(ch - '0'); else if(ch >= 'A' && ch <= 'F') digit = (uint)(ch - 'A' + 10); else return(false); parsed = (parsed << 4) | digit; } UDPCCObservedFloatBits bits; bits.bits = parsed; value = bits.value; return(MathIsValidNumber((double)value)); } //+------------------------------------------------------------------+ //| Strictly decodes one canonical uppercase hexadecimal uint64 | //| field | //+------------------------------------------------------------------+ bool DPCCOfflineCheckpointReadHex64(const string text, ulong &value) { value = 0; if(StringLen(text) != 16) return(false); for(int i = 0; i < 16; i++) { const ushort ch = StringGetCharacter(text, i); uint digit = 0; if(ch >= '0' && ch <= '9') digit = (uint)(ch - '0'); else if(ch >= 'A' && ch <= 'F') digit = (uint)(ch - 'A' + 10); else return(false); value = (value << 4) | digit; } return(true); } //+------------------------------------------------------------------+ //| Appends one exact field to the versioned offline boundary wire | //+------------------------------------------------------------------+ void DPCCOfflineCheckpointField(string &wire, const string value) { wire += value + "\n"; } //+------------------------------------------------------------------+ //| Shared buffers for the isolated historical financial transition | //+------------------------------------------------------------------+ bool DPCCUploadHistoricalBuffer(CBufferFloat &buffer, const float &values[], COpenCLMy *context) { if(ArraySize(values) <= 0 || !buffer.BufferInit((uint)ArraySize(values), 0)) return(false); for(int i = 0; i < ArraySize(values); i++) if(!buffer.Update(i, values[i])) return(false); return(buffer.BufferCreate(context)); } //+------------------------------------------------------------------+ //| Read one exact native continuation in its published field order | //+------------------------------------------------------------------+ bool DPCCReadHistoricalState(CBufferFloat &buffer, float &values[]) { if(buffer.Total() != 33 || !buffer.BufferRead() || ArrayResize(values, 33) != 33) return(false); for(int i = 0; i < 33; i++) { values[i] = buffer[i]; if(!MathIsValidNumber((double)values[i])) return(false); } return(values[24] == 0 && values[32] == 0); } //+------------------------------------------------------------------+ //| Prepared factual open; the native owner retains its commit token | //+------------------------------------------------------------------+ struct SDPCCHistoricalPreparedBar { CNeuronScenarioRolloutOCL *owner; ulong token, signature; ulong decision_identity; bool pending; long ordinal, server_open, server_close, open_time, close_time; string time_domain; ENUM_SYMBOL_CALC_MODE calculation_mode; double preparation_reward; float state[], account13[]; SDPCCSimulatedPositionMetadata position; }; struct SDPCCHistoricalCommittedBar { double reward; bool committed, floor_reached; ulong decision_identity; float state[], account13[]; SDPCCSimulatedPositionMetadata position; }; //--- Deterministic one-bar critic sample bound to a frozen sampler and transition identity. bool DPCCOnlineCriticSampleU(const string sampler_sha256, const ulong sampler_cursor, const int critic_id, float &u, string &reason) { u = 0.0f; reason = ""; if(!DPCCResolverLowerSha256(sampler_sha256) || sampler_cursor == 0 || (critic_id != 1 && critic_id != 2)) {reason = "online.sampler.arguments"; return(false);} string wire = "DPCC_ONLINE_CRITIC_SAMPLE_V1\n"; DPCCOfflineCheckpointField(wire, sampler_sha256); DPCCOfflineCheckpointField(wire, (string)sampler_cursor); DPCCOfflineCheckpointField(wire, (critic_id == 1 ? "Q1" : "Q2")); uchar bytes[]; if(StringToCharArray(wire, bytes, 0, WHOLE_ARRAY, CP_UTF8) <= 1) {reason = "online.sampler.encoding"; return(false);} ArrayResize(bytes, ArraySize(bytes) - 1); string digest = ""; if(!DPCCSHA256Lower(bytes, digest, reason)) return(false); uint numerator = 0; for(int i = 0; i < 6; i++) { const ushort code = StringGetCharacter(digest, i); const uint nibble = (code >= '0' && code <= '9' ? (uint)(code - '0') : (uint)(code - 'a' + 10)); numerator = (numerator << 4) | nibble; } if(numerator == 0) numerator = 1; u = (float)numerator / 16777216.0f; return(true); } //+------------------------------------------------------------------+ //| Validate prepared values used by the close features and reward | //+------------------------------------------------------------------+ bool DPCCValidateHistoricalPrepared(const SDPCCHistoricalPreparedBar &prepared, string &reason) { reason = ""; if(prepared.decision_identity == 0 || prepared.decision_identity > (ulong)LONG_MAX || prepared.owner == NULL || prepared.signature == 0 || ArraySize(prepared.state) != 33 || ArraySize(prepared.account13) != 13 || !MathIsValidNumber(prepared.preparation_reward) || !MathIsValidNumber((double)prepared.position.signed_lots)) {reason = "historical_bar.prepared_shape_or_reward"; return(false);} if((prepared.time_domain != "UTC_SECONDS_V1" && prepared.time_domain != "SERVER_CALENDAR_SECONDS_V1") || prepared.position.time_domain != prepared.time_domain) {reason = "historical_bar.prepared_time_domain"; return(false);} for(int i = 0; i < 33; i++) if(!MathIsValidNumber((double)prepared.state[i])) {reason = "historical_bar.prepared_state"; return(false);} for(int i = 0; i < 13; i++) if(!MathIsValidNumber((double)prepared.account13[i])) {reason = "historical_bar.prepared_account13"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Pure factual context preflight preserves retryable caller state | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalBarStepValues(const SDPCCShape &source_shape, const float &context[], const float &target_native_ohlc[], const long ordinal, const long &opens[], const long &closes[], const long &ordinals[], const long open_time, const long close_time, float &step_values[], ENUM_SYMBOL_CALC_MODE &calculation_mode, string &reason) { ArrayResize(step_values, 0); reason = ""; SDPCCLayout layout, unused; uint tokens = 0; if(!DPCCCompositeLayouts(source_shape, layout, unused, tokens) || ArraySize(context) != (int)layout.offsets[10] || source_shape.H == 0 || ArraySize(opens) != (int)source_shape.H || ArraySize(closes) != (int)source_shape.H || ArraySize(ordinals) != (int)source_shape.H || ordinal < 0 || ordinal != ordinals[0] || open_time <= 0 || close_time <= open_time) {reason = "historical_bar.prepare_shape"; return(false);} const float calc = context[(int)layout.offsets[5] + 11]; if(!DPCCSupportedHistoricalCalculationModeValue(calc)) {reason = "historical_bar.native_calculation_mode_required"; return(false);} float historical[]; SDPCCShape history_shape, step_shape; if(!DPCCBuildHistoricalRolloutContext(source_shape, context, target_native_ohlc, opens[0], ordinal, opens, closes, ordinals, history_shape, historical, reason)) return(false); if(!DPCCBuildHistoricalRolloutStepContext(history_shape, historical, 0, step_shape, step_values, reason)) return(false); calculation_mode = (ENUM_SYMBOL_CALC_MODE)(int)calc; return(true); } //+------------------------------------------------------------------+ //| Build a flat simulated seed from a caller-captured balance | //+------------------------------------------------------------------+ bool DPCCBuildHistoricalFlatSeed(const double starting_balance, const string captured_currency, const uint captured_currency_digits, const SDPCCSnapshotMeta &profile_meta, const SDPCCConfig &config, const SDPCCShape &source_shape, float &context[], const float &target_native_ohlc[], float &previous_state[], SDPCCSimulatedPositionMetadata &position, string &reason) { reason = ""; SDPCCLayout layout, unused; uint tokens = 0; if(!MathIsValidNumber(starting_balance) || starting_balance <= 0) {reason = "historical_seed.balance"; return(false);} const float balance = (float)starting_balance; if(!MathIsValidNumber((double)balance) || balance <= 0 || DPCCTrainingFloorReached(balance, balance) || captured_currency == "" || profile_meta.account_currency == "" || captured_currency != profile_meta.account_currency || captured_currency_digits > 8 || (config.margin_fx_profile != DPCC_MARGIN_EXTERNAL_V2 && captured_currency_digits != config.account_currency_digits) || (config.margin_fx_profile == DPCC_MARGIN_EXTERNAL_V2 && config.account_currency_digits != 0) || (config.margin_fx_profile != DPCC_MARGIN_FOREX_SPOT_V1 && config.margin_fx_profile != DPCC_MARGIN_EXTERNAL_V2 && config.margin_fx_profile != DPCC_MARGIN_HISTORICAL_CLOSE_V1) || config.price_basis != DPCC_PRICE_BID || config.quote_adapter != DPCC_QUOTE_BID_IDENTITY_V1 || !DPCCValidateFixedConfig(source_shape, config) || !DPCCCompositeLayouts(source_shape, layout, unused, tokens) || source_shape.H == 0 || source_shape.H > INT_MAX / 4 || ArraySize(context) != (int)layout.offsets[10] || ArraySize(target_native_ohlc) != (int)(source_shape.H * 4)) {reason = "historical_seed.profile_or_shape"; return(false);} const int account_offset = (int)layout.offsets[4]; const int instrument_offset = (int)layout.offsets[5]; const int limits_offset = (int)layout.offsets[6]; const float first_bid = context[account_offset + 15]; const float first_ask = context[account_offset + 16]; const float calculation_mode = context[instrument_offset + 11]; const float initial_interest = context[instrument_offset + 23]; const float open_tolerance = context[limits_offset + 15]; const float first_open = target_native_ohlc[0]; if(!MathIsValidNumber((double)first_bid) || !MathIsValidNumber((double)first_ask) || !MathIsValidNumber((double)calculation_mode) || !MathIsValidNumber((double)initial_interest) || !MathIsValidNumber((double)open_tolerance) || !MathIsValidNumber((double)first_open) || first_bid <= 0 || first_ask < first_bid || open_tolerance < 0 || first_open <= 0 || !DPCCSupportedHistoricalCalculationModeValue(calculation_mode) || MathAbs((double)first_open - (double)first_bid) > (double)open_tolerance) {reason = "historical_seed.first_quote_or_mode"; return(false);} float candidate_context[]; float candidate_state[]; if(ArrayCopy(candidate_context, context) != ArraySize(context) || ArrayResize(candidate_state, 33) != 33) {reason = "historical_seed.capacity"; return(false);} ArrayInitialize(candidate_state, 0.0f); candidate_state[0] = balance; candidate_state[1] = balance; candidate_state[7] = balance; candidate_state[8] = balance; candidate_state[9] = balance; candidate_state[12] = balance; candidate_state[15] = first_bid; candidate_state[16] = first_ask; candidate_state[20] = 1.0f; candidate_state[24] = 0.0f; candidate_state[25] = 0.0f; candidate_state[26] = first_bid; candidate_state[27] = initial_interest; candidate_state[28] = 0.0f; candidate_state[29] = (float)config.account_currency_digits; candidate_state[30] = -1.0f; candidate_state[31] = 0.0f; candidate_state[32] = 0.0f; for(int i = 0; i < 24; i++) candidate_context[account_offset + i] = candidate_state[i]; SDPCCSimulatedPositionMetadata candidate_position; ZeroMemory(candidate_position); if(!ArraySwap(context, candidate_context)) {reason = "historical_seed.context_publish"; return(false);} if(!ArraySwap(previous_state, candidate_state)) { ArraySwap(context, candidate_context); reason = "historical_seed.state_publish"; return(false); } position = candidate_position; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Prepare factual Account13 before a caller chooses its action | //+------------------------------------------------------------------+ //--- A false return with pending=true requires destruction of the native owner; //--- there is no native abort API, so that layer must never be retried or reused. bool DPCCPrepareHistoricalBar(CNeuronScenarioRolloutOCL &native, const float &step_values[], const long ordinal, const ulong decision_identity, const long server_open, const long server_close, const long open_time, const long close_time, const ENUM_SYMBOL_CALC_MODE calculation_mode, const float &previous_state[], const SDPCCSimulatedPositionMetadata &previous_position, const uint episode_step, const uint episode_horizon, const bool initialize, CBufferFloat &prior, SDPCCHistoricalPreparedBar &prepared, string &reason, const bool observed_open = false, const string time_domain = "UTC_SECONDS_V1") { ZeroMemory(prepared); reason = ""; if((time_domain != "UTC_SECONDS_V1" && time_domain != "SERVER_CALENDAR_SECONDS_V1") || (time_domain == "SERVER_CALENDAR_SECONDS_V1" && (open_time != server_open || close_time != server_close))) {reason = "historical_bar.prepare_time_domain"; return(false);} COpenCLMy *device = native.GetOpenCL(); if(CheckPointer(device) == POINTER_INVALID || ArraySize(previous_state) != 33 || ArraySize(step_values) == 0 || ordinal < 0 || decision_identity == 0 || decision_identity > (ulong)LONG_MAX || server_open <= 0 || server_close <= server_open || open_time <= 0 || close_time <= open_time || !DPCCSupportedHistoricalCalculationMode(calculation_mode)) {reason = "historical_bar.prepare_boundary"; return(false);} CBufferFloat step, open_state, rewards, status, diagnostics; if(!DPCCUploadHistoricalBuffer(step, step_values, device) || !DPCCUploadHistoricalBuffer(prior, previous_state, device) || !open_state.BufferInit(33, 0) || !rewards.BufferInit(1, 0) || !status.BufferInit(1, 0) || !diagnostics.BufferInit(4, 0) || !open_state.BufferCreate(device) || !rewards.BufferCreate(device) || !status.BufferCreate(device) || !diagnostics.BufferCreate(device)) {reason = "historical_bar.prepare_buffers"; return(false);} ulong token = 0; const bool success = (observed_open ? native.PrepareObservedOpen(GetPointer(step), episode_step, episode_horizon, initialize, (initialize ? NULL : GetPointer(prior)), GetPointer(open_state), GetPointer(rewards), GetPointer(status), GetPointer(diagnostics), token) : native.PrepareOpen(GetPointer(step), episode_step, episode_horizon, initialize, (initialize ? NULL : GetPointer(prior)), GetPointer(open_state), GetPointer(rewards), GetPointer(status), GetPointer(diagnostics), token)); if(!success || (!observed_open && token == 0)) { //--- Retain the native verdict before attempting diagnostic readback. const int native_status = (int)native.LastStatus(); const bool status_read = status.BufferRead(); const string device_status = (status_read ? StringFormat("%.9g", (double)status[0]) : "UNREAD"); int scenario_count = -1; CLayerDescription *descriptor = native.GetLayerInfo(); if(descriptor != NULL) { if(ArraySize(descriptor.windows) > 1) scenario_count = (int)descriptor.windows[1]; //--- Dump the failed CPU context in bounded section chunks. SDPCCShape diagnostic_shape; SDPCCLayout diagnostic_layout; bool layout_ready = DPCCReadFinancialDescriptor(descriptor, diagnostic_shape, defNeuronScenarioRolloutOCL, false); if(layout_ready) { diagnostic_shape.H = 1; diagnostic_shape.M = 1; layout_ready = DPCCBuildRolloutLayout(diagnostic_shape, diagnostic_layout) && diagnostic_layout.total == (ulong)ArraySize(step_values); } if(layout_ready) { const string sections[] = {"OHLC", "ACCOUNT24", "INSTRUMENT48", "LIMITS24", "EXECUTION24", "CALENDAR_COUNT", "CALENDAR", "ACTIVE" }; for(int section = 0; section < DPCC_CONTEXT_SECTIONS; section++) for(int start = 0; start < (int)diagnostic_layout.lengths[section]; start += 12) { string fields = ""; const int end = (int)MathMin(start + 12, (int)diagnostic_layout.lengths[section]); for(int field = start; field < end; field++) { const int index = (int)diagnostic_layout.offsets[section] + field; fields += StringFormat(" %d@%d=%.9g", field, index, (double)step_values[index]); } PrintFormat("DPCC_PREPARE_CONTEXT row=%I64d section=%s%s", ordinal, sections[section], fields); } } else PrintFormat("DPCC_PREPARE_CONTEXT row=%I64d layout=UNAVAILABLE step_values=%d", ordinal, ArraySize(step_values)); delete descriptor; } reason = StringFormat("historical_bar.prepare_open:native_status=%d;success=%d;token=%I64u;", native_status, (int)success, token); reason += StringFormat("status_read=%d;device_status=%s;K=%d;step_values=%d;", (int)status_read, device_status, scenario_count, ArraySize(step_values)); reason += StringFormat("initialize=%d;episode_step=%u;episode_horizon=%u;observed_open=%d", (int)initialize, episode_step, episode_horizon, (int)observed_open); return(false); } //--- Expose the native pending handle before any later operation can fail. prepared.owner = GetPointer(native); prepared.token = token; prepared.signature = native.ContractSignature(); prepared.decision_identity = decision_identity; prepared.ordinal = ordinal; prepared.server_open = server_open; prepared.server_close = server_close; prepared.time_domain = time_domain; prepared.open_time = open_time; prepared.close_time = close_time; prepared.calculation_mode = calculation_mode; prepared.pending = (token != 0); if(native.LastStatus() != DPCC_OK || !status.BufferRead() || status[0] != 0 || !rewards.BufferRead() || !MathIsValidNumber((double)rewards[0]) || !DPCCReadHistoricalState(open_state, prepared.state)) {reason = "historical_bar.prepare_readback"; return(false);} if(initialize && (prepared.state[0] != previous_state[0] || prepared.state[1] != previous_state[1] || prepared.state[2] != 0 || prepared.state[3] != 0 || prepared.state[4] != 0 || prepared.state[5] != 0 || prepared.state[6] != 0 || prepared.state[10] != 0 || prepared.state[11] != 0 || prepared.state[13] != 0 || prepared.state[14] != 0 || prepared.state[17] != 0 || prepared.state[18] != 0 || prepared.state[19] != 0 || prepared.state[20] != 1 || prepared.state[23] != 0 || prepared.state[24] != 0 || prepared.state[25] != 0 || prepared.state[28] != 0 || prepared.state[29] != previous_state[29] || prepared.state[30] != -1 || prepared.state[31] != 0 || prepared.state[32] != 0)) {reason = "historical_bar.native_flat_seed_mismatch"; return(false);} if(!DPCCBuildHistoricalAccount13(ordinal, server_open, server_close, open_time, close_time, server_open, calculation_mode, previous_state, prepared.state, previous_position, prepared.position, prepared.account13, reason, time_domain)) return(false); const bool floor_reached = DPCCTrainingFloorReached(prepared.state[0], prepared.state[1]); if((floor_reached && !observed_open) || (!floor_reached && token == 0)) {reason = "historical_bar.training_floor"; return(false);} prepared.preparation_reward = rewards[0]; return(DPCCValidateHistoricalPrepared(prepared, reason)); } //+------------------------------------------------------------------+ //| Commit the supplied six actions through the same native ledger | //+------------------------------------------------------------------+ bool DPCCCommitHistoricalBar(CNeuronScenarioRolloutOCL &native, SDPCCHistoricalPreparedBar &prepared, const float &actions[], SDPCCHistoricalCommittedBar &closed, string &reason, CBufferFloat *factual_context = NULL) { ZeroMemory(closed); reason = ""; if(!prepared.pending || prepared.owner != GetPointer(native) || prepared.token == 0 || prepared.signature != native.ContractSignature() || ArraySize(actions) != 6) {reason = "historical_bar.commit_boundary"; return(false);} if(!DPCCValidateHistoricalPrepared(prepared, reason)) return(false); if(prepared.owner != GetPointer(native) || prepared.signature != native.ContractSignature()) {reason = "historical_bar.prepared_owner_or_contract"; return(false);} for(int i = 0; i < 6; i++) if(!MathIsValidNumber((double)actions[i]) || actions[i] < 0 || actions[i] > 1) {reason = "historical_bar.action_range"; return(false);} COpenCLMy *device = native.GetOpenCL(); CBufferFloat action, state, rewards, status, diagnostics; if(!DPCCUploadHistoricalBuffer(action, actions, device) || !state.BufferInit(33, 0) || !rewards.BufferInit(1, 0) || !status.BufferInit(1, 0) || !diagnostics.BufferInit(4, 0) || !state.BufferCreate(device) || !rewards.BufferCreate(device) || !status.BufferCreate(device) || !diagnostics.BufferCreate(device)) {reason = "historical_bar.commit_buffers"; return(false);} //--- A failed native commit must never be retried with another action. prepared.pending = false; const bool success = (factual_context != NULL ? native.CommitObservedAction(GetPointer(action), 0, factual_context, prepared.token, GetPointer( state), GetPointer(rewards), GetPointer(status), GetPointer(diagnostics)) : native.CommitPreparedAction(GetPointer(action), 0, prepared.token, GetPointer(state), GetPointer(rewards), GetPointer(status), GetPointer(diagnostics))); if(!success || native.LastStatus() != DPCC_OK || !status.BufferRead() || status[0] != 0 || !rewards.BufferRead() || !MathIsValidNumber((double)rewards[0]) || !DPCCReadHistoricalState(state, closed.state)) {reason = "historical_bar.commit"; return(false);} if(!DPCCBuildHistoricalAccount13(prepared.ordinal, prepared.server_open, prepared.server_close, prepared.open_time, prepared.close_time, prepared.server_close, prepared.calculation_mode, prepared.state, closed.state, prepared.position, closed.position, closed.account13, reason, prepared.time_domain)) return(false); closed.reward = prepared.preparation_reward + rewards[0]; closed.floor_reached = DPCCTrainingFloorReached(closed.state[0], closed.state[1]); closed.decision_identity = prepared.decision_identity; closed.committed = true; return(true); } //+------------------------------------------------------------------+ //| Build TD reward across the decision and next open observation | //+------------------------------------------------------------------+ bool DPCCBuildOpenToOpenTDReward(const SDPCCHistoricalPreparedBar &previous_prepared, const SDPCCHistoricalCommittedBar &previous_closed, const SDPCCHistoricalPreparedBar &next_prepared, double &td_reward, string &reason, const bool terminal_preparation = false) { td_reward = 0.0; reason = ""; //--- Require a successful prior commit and an active next-open preparation. if(!previous_closed.committed || previous_prepared.pending || (!next_prepared.pending && !terminal_preparation) || previous_prepared.decision_identity == 0 || previous_prepared.decision_identity >= (ulong)LONG_MAX || previous_closed.decision_identity != previous_prepared.decision_identity || next_prepared.decision_identity != previous_prepared.decision_identity + 1 || previous_prepared.ordinal < 0 || previous_prepared.ordinal >= LONG_MAX || next_prepared.ordinal != previous_prepared.ordinal + 1) {reason = "online.td_reward.evidence"; return(false);} //--- Compute the prior commit suffix plus preparation reward at the next open. if(!MathIsValidNumber(previous_closed.reward) || !MathIsValidNumber(previous_prepared.preparation_reward) || !MathIsValidNumber(next_prepared.preparation_reward)) {reason = "online.td_reward.nonfinite_input"; return(false);} const double candidate = (previous_closed.reward - previous_prepared.preparation_reward) + next_prepared.preparation_reward; if(!MathIsValidNumber(candidate)) {reason = "online.td_reward.nonfinite_result"; return(false);} td_reward = candidate; return(true); } //+------------------------------------------------------------------+ //| Owns only a chronological row boundary, never a second simulator | //+------------------------------------------------------------------+ class CDPCCOfflineRowOwner { private: bool m_initialized, m_failed, m_inflight, m_durable, m_floor_reached; ulong m_next; SDPCCOfflinePolicyProfile m_profile; SDPCCOfflineFitKey m_keys[]; SDPCCConfig m_config; float m_state[]; SDPCCSimulatedPositionMetadata m_position; CNet *m_actor, *m_factual, *m_scoring; CDPCCMarketClock *m_clock; string m_forecast_receipt_sha, m_forecast_raw_sha, m_forecast_raw_path; string m_restored_actor_path, m_restored_actor_sha; ulong m_restored_actor_fingerprint; bool Fail(const string cause, string &reason) { if(m_inflight) m_failed = true; reason = cause; return(false); } bool ValidateCheckpointActor(CNet &actor, const ulong next_row, string &reason) { if(!DPCCValidatePolicyNet(actor, m_config.shape, false, reason)) return(false); for(int i = 0; i < 3; i++) { CNeuronBaseOCL *layer = actor.Layer(i); if(CheckPointer(layer) == POINTER_INVALID || !layer.TrainMode()) { reason = "offline.checkpoint.actor_train_mode"; return(false); } } CNeuronConvOCL *hidden = dynamic_cast(actor.Layer(1)); CNeuronConvOCL *head = dynamic_cast(actor.Layer(2)); if(!hidden || !head || hidden.Optimization() != ADAM || head.Optimization() != ADAM || m_profile.initial_step < 0 || (ulong)m_profile.initial_step > UINT_MAX - next_row || hidden.NativeOptimizerStep() != (uint)m_profile.initial_step + (uint)next_row || head.NativeOptimizerStep() != (uint)m_profile.initial_step + (uint)next_row) { reason = "offline.checkpoint.actor_optimizer_step"; return(false); } ulong before = 0, after = 0; if(!actor.ParameterFingerprint(before)) { reason = "offline.checkpoint.actor_fingerprint"; return(false); } SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(m_config.shape, layout, reason)) return(false); CBufferFloat scenarios, probe_input; float account13[13]; ArrayInitialize(account13, 0.0f); if(layout.actor_count <= layout.scenario_count || !scenarios.BufferInit(layout.scenario_count, 0.0f) || !DPCCPackPolicyInput(m_config.shape, GetPointer(scenarios), account13, NULL, GetPointer(probe_input), reason) || probe_input.Total() != (int)layout.actor_count) { if(reason == "") reason = "offline.checkpoint.probe_input_shape"; return(false); } if(!actor.feedForward(GetPointer(probe_input), 1, false, (CBufferFloat*)NULL)) { reason = "offline.checkpoint.probe_forward"; return(false); } float output[]; actor.getResults(output); if(ArraySize(output) != (int)(m_config.shape.H * 6)) { reason = "offline.checkpoint.probe_output_shape"; return(false); } for(int i = 0; i < ArraySize(output); i++) if(!MathIsValidNumber((double)output[i]) || output[i] < 0 || output[i] > 1) { reason = "offline.checkpoint.probe_output_numeric"; return(false); } if(!actor.ParameterFingerprint(after) || before != after) { reason = "offline.checkpoint.probe_mutated_actor"; return(false); } return(true); } bool BuildCheckpointLedger(string &wire, string &reason) { if(ArraySize(m_state) != 33 || ArraySize(m_profile.seed.risk_limits) != 24 || m_next > (ulong)ArraySize(m_keys)) { reason = "offline.checkpoint.ledger_shape"; return(false); } const bool forecast_bound = (StringLen(m_forecast_receipt_sha) > 0 && StringLen(m_forecast_raw_sha) > 0); wire = (forecast_bound ? "DPCC_OFFLINE_LEDGER_V2\n" : "DPCC_OFFLINE_LEDGER_V1\n"); const string bindings[] = {m_profile.seed_sha256, m_profile.profile_sha256, m_profile.fit_plan_sha256, m_profile.model_sha256, m_profile.training_sha256, m_profile.bank_sha256, m_profile.time_index_sha256, m_profile.decoder_sha256 }; for(int i = 0; i < ArraySize(bindings); i++) DPCCOfflineCheckpointField(wire, bindings[i]); DPCCOfflineCheckpointField(wire, (string)m_next); DPCCOfflineCheckpointField(wire, (string)ArraySize(m_keys)); for(int side = 0; side < 2; side++) { const long index = (side == 0 ? (long)m_next - 1 : (long)m_next); if(index < 0 || index >= ArraySize(m_keys)) { const string empty[] = {"-1", "", "", "0", "0", "0", "0000000000000000", "0000000000000000", "0000000000000000" }; for(int i = 0; i < ArraySize(empty); i++) DPCCOfflineCheckpointField(wire, empty[i]); continue; } const SDPCCOfflineFitKey key = m_keys[(int)index]; DPCCOfflineCheckpointField(wire, (string)key.record_index); DPCCOfflineCheckpointField(wire, key.window_id); DPCCOfflineCheckpointField(wire, key.record_sha256); DPCCOfflineCheckpointField(wire, (string)key.episode_id); DPCCOfflineCheckpointField(wire, (string)key.decision_bar); DPCCOfflineCheckpointField(wire, (string)key.decision_time); DPCCOfflineCheckpointField(wire, StringFormat("%016I64X", key.decision_identity)); DPCCOfflineCheckpointField(wire, StringFormat("%016I64X", key.context_identity)); DPCCOfflineCheckpointField(wire, StringFormat("%016I64X", key.owner_identity)); } DPCCOfflineCheckpointField(wire, (string)(m_next == (ulong)ArraySize(m_keys))); DPCCOfflineCheckpointField(wire, (string)(m_state[24] != 0)); DPCCOfflineCheckpointField(wire, (string)m_floor_reached); for(int i = 0; i < ArraySize(m_state); i++) { if(!MathIsValidNumber((double)m_state[i])) { reason = "offline.checkpoint.state_nonfinite"; return(false); } DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(m_state[i])); } for(int i = 0; i < ArraySize(m_profile.seed.risk_limits); i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(m_profile.seed.risk_limits[i])); DPCCOfflineCheckpointField(wire, (m_position.initialized ? "1" : "0")); DPCCOfflineCheckpointField(wire, (string)m_position.snapshot_bar); DPCCOfflineCheckpointField(wire, (string)m_position.snapshot_server); DPCCOfflineCheckpointField(wire, (string)m_position.snapshot_time); DPCCOfflineCheckpointField(wire, (string)m_position.bar_server_open); DPCCOfflineCheckpointField(wire, (string)m_position.bar_server_close); DPCCOfflineCheckpointField(wire, (string)m_position.bar_open_time); DPCCOfflineCheckpointField(wire, (string)m_position.bar_close_time); DPCCOfflineCheckpointField(wire, (string)m_position.first_open_bar); DPCCOfflineCheckpointField(wire, (string)m_position.first_open_server); DPCCOfflineCheckpointField(wire, (string)m_position.first_open_time); DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(m_position.signed_lots)); if(forecast_bound) { DPCCOfflineCheckpointField(wire, m_forecast_receipt_sha); DPCCOfflineCheckpointField(wire, m_forecast_raw_sha); DPCCOfflineCheckpointField(wire, m_forecast_raw_path); } DPCCOfflineCheckpointField(wire, (forecast_bound ? "END_DPCC_OFFLINE_LEDGER_V2" : "END_DPCC_OFFLINE_LEDGER_V1")); reason = ""; return(true); } bool ReadCheckpointText(const string path, const bool common, string &text, string &reason) { const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "offline.checkpoint.text_open"; return(false); } const ulong size = FileSize(handle); if(size == 0 || size > 1048576) { FileClose(handle); reason = "offline.checkpoint.text_size"; return(false); } uchar bytes[]; if(ArrayResize(bytes, (int)size) != (int)size) { FileClose(handle); reason = "offline.checkpoint.text_capacity"; return(false); } const uint read = FileReadArray(handle, bytes, 0, (uint)size); const bool complete = (read == (uint)size && FileTell(handle) == size && FileSize(handle) == size); FileClose(handle); if(!complete) { reason = "offline.checkpoint.text_read"; return(false); } text = CharArrayToString(bytes, 0, (int)size, CP_UTF8); return(StringLen(text) > 0); } bool ValidateCheckpointPosition(const SDPCCOfflinePolicyProfile &profile, const ulong next, const SDPCCOfflineFitKey &keys[], const float &state[], const SDPCCSimulatedPositionMetadata &position, CDPCCMarketClock &clock, string &reason) { if(ArraySize(state) != 33 || next > (ulong)ArraySize(keys)) {reason = "offline.checkpoint.position_shape"; return(false);} if(next == 0) { const SDPCCSimulatedPositionMetadata seed = profile.seed.position; UDPCCObservedFloatBits actual_lots, seed_lots; actual_lots.value = position.signed_lots; seed_lots.value = seed.signed_lots; if(position.initialized != seed.initialized || position.snapshot_bar != seed.snapshot_bar || position.snapshot_server != seed.snapshot_server || position.snapshot_time != seed.snapshot_time || position.bar_server_open != seed.bar_server_open || position.bar_server_close != seed.bar_server_close || position.bar_open_time != seed.bar_open_time || position.bar_close_time != seed.bar_close_time || position.first_open_bar != seed.first_open_bar || position.first_open_server != seed.first_open_server || position.first_open_time != seed.first_open_time || actual_lots.bits != seed_lots.bits) {reason = "offline.restore.cursor_position_join"; return(false);} return(true); } if(!position.initialized || position.snapshot_bar != keys[(int)next - 1].decision_bar) {reason = "offline.restore.cursor_position_join"; return(false);} UDPCCObservedFloatBits state_lots, position_lots; state_lots.value = state[2]; position_lots.value = position.signed_lots; if(state_lots.bits != position_lots.bits) {reason = "offline.restore.cursor_position_join"; return(false);} SDPCCClockBar bar; long open = 0, close = 0; if(!clock.IndexedBar(keys[(int)next - 1].decision_bar, bar, open, close, reason)) {reason = "offline.restore.position_clock"; return(false);} if(bar.ordinal != keys[(int)next - 1].decision_bar || open != keys[(int)next - 1].decision_time || position.snapshot_server != bar.server_close || position.snapshot_time != close || position.bar_server_open != bar.server_open || position.bar_server_close != bar.server_close || position.bar_open_time != open || position.bar_close_time != close) {reason = "offline.restore.cursor_position_join"; return(false);} if(position.signed_lots == 0) { if(position.first_open_bar != 0 || position.first_open_server != 0 || position.first_open_time != 0) {reason = "offline.restore.cursor_position_join"; return(false);} return(true); } SDPCCClockBar first; long first_open = 0, first_close = 0; if(position.first_open_bar < 0 || position.first_open_bar > position.snapshot_bar || !clock.IndexedBar(position.first_open_bar, first, first_open, first_close, reason)) {reason = "offline.restore.position_first_open_clock"; return(false);} if(first.ordinal != position.first_open_bar || position.first_open_server != first.server_open || position.first_open_time != first_open || first_open > close) {reason = "offline.restore.cursor_position_join"; return(false);} return(true); } public: CDPCCOfflineRowOwner(void) { m_initialized = false; m_failed = false; m_inflight = false; m_durable = false; m_floor_reached = false; m_next = 0; m_actor = NULL; m_factual = NULL; m_scoring = NULL; m_clock = NULL; m_restored_actor_path = ""; m_restored_actor_sha = ""; m_restored_actor_fingerprint = 0; ZeroMemory(m_position); } bool Failed(void)const {return(m_failed);} ulong NextRow(void)const {return(m_next);} bool Durable(void)const {return(m_durable);} bool FloorReached(void)const {return(m_floor_reached);} //+-------------------------------------------------------------------+ //| Return Actor artifact identity after authenticated native restore | //+-------------------------------------------------------------------+ bool RestoredActorArtifact(CNet &actor, string &path, string &sha, string &reason) { path = ""; sha = ""; ulong fingerprint = 0; if(!m_initialized || m_failed || !m_durable || m_actor != GetPointer(actor) || m_restored_actor_path == "" || !DPCCResolverLowerSha256(m_restored_actor_sha) || !actor.ParameterFingerprint(fingerprint) || fingerprint != m_restored_actor_fingerprint) { reason = "offline.restore.actor_artifact_unavailable"; return(false); } path = m_restored_actor_path; sha = m_restored_actor_sha; reason = ""; return(true); } //--- Admission is issued by the existing Study loader, not by this owner. bool Initialize(const SDPCCOfflinePolicyProfile &profile, const SDPCCOfflineFitKey &keys[], const SDPCCOfflinePolicyAdmission &admission, const SDPCCConfig &config, string &reason, const string forecast_receipt_sha = "", const string forecast_raw_sha = "", const string forecast_raw_path = "") { string seed = "", policy = "", plan = ""; if(m_initialized || m_failed || !admission.admitted || admission.rows != (ulong)ArraySize(keys) || ArraySize(keys) == 0 || profile.horizon != config.shape.H || profile.gamma != config.gamma || !DPCCOfflinePolicyDigests(profile, keys, seed, policy, plan, reason) || seed != admission.seed_sha256 || seed != profile.seed_sha256 || policy != admission.profile_sha256 || policy != profile.profile_sha256 || plan != admission.fit_plan_sha256 || plan != profile.fit_plan_sha256 || admission.candidate_bound != 1 + (ulong)profile.horizon * 12 || profile.last_decision_bar - profile.first_decision_bar + profile.horizon > 16777216 || ArraySize(profile.seed.state) != 33 || ArraySize(profile.seed.risk_limits) != 24 || ((StringLen(forecast_receipt_sha) == 0) != (StringLen(forecast_raw_sha) == 0)) || (StringLen(forecast_receipt_sha) > 0 && (!DPCCArchitectureSHA256Encoding(forecast_receipt_sha) || !DPCCArchitectureSHA256Encoding(forecast_raw_sha) || StringLen(forecast_raw_path) == 0)) || (StringLen(forecast_receipt_sha) == 0 && StringLen(forecast_raw_path) > 0) || profile.seed.state[2] != 0 || profile.seed.position.initialized) return(Fail("offline.row.admission", reason)); if(ArrayResize(m_keys, ArraySize(keys)) != ArraySize(keys) || ArrayCopy(m_state, profile.seed.state) != 33) return(Fail("offline.row.capacity", reason)); for(int i = 0; i < ArraySize(keys); i++) m_keys[i] = keys[i]; m_profile = profile; m_config = config; m_position = profile.seed.position; m_forecast_receipt_sha = forecast_receipt_sha; m_forecast_raw_sha = forecast_raw_sha; m_forecast_raw_path = forecast_raw_path; m_initialized = true; m_durable = false; m_floor_reached = false; reason = ""; return(true); } //--- Save one immutable Actor+ledger boundary. Call after Initialize before row zero, //--- then after each successful Step and before requesting the next row. bool SaveCheckpoint(CNet &actor, const string actor_path, const string ledger_path, const string manifest_path, const bool common, string &manifest_sha, string &reason, const ulong maximum_bytes = 536870912) { manifest_sha = ""; const ulong actor_limit = (maximum_bytes < 536870912 ? maximum_bytes : (ulong)536870912); const ulong text_limit = (maximum_bytes < 1048576 ? maximum_bytes : (ulong)1048576); if(!m_initialized || m_failed || m_inflight || (m_actor != NULL && m_actor != GetPointer(actor))) { m_failed = true; reason = "offline.checkpoint.owner_boundary"; return(false); } if(!ValidateCheckpointActor(actor, m_next, reason)) { m_failed = true; return(false); } if(m_next > 0 && (m_clock == NULL || !ValidateCheckpointPosition(m_profile, m_next, m_keys, m_state, m_position, *m_clock, reason))) { m_failed = true; return(false); } string canonical_actor = "", canonical_ledger = "", canonical_manifest = ""; if(!DPCCCheckpointCanonicalPath(actor_path, canonical_actor, reason) || canonical_actor != actor_path || !DPCCCheckpointCanonicalPath(ledger_path, canonical_ledger, reason) || canonical_ledger != ledger_path || !DPCCCheckpointCanonicalPath(manifest_path, canonical_manifest, reason) || canonical_manifest != manifest_path || actor_path == ledger_path || actor_path == manifest_path || ledger_path == manifest_path || FileIsExist(actor_path, common ? FILE_COMMON : 0) || FileIsExist(ledger_path, common ? FILE_COMMON : 0) || FileIsExist(manifest_path, common ? FILE_COMMON : 0)) { m_failed = true; if(reason == "") reason = "offline.checkpoint.immutable_destination"; return(false); } string ledger = "", ledger_sha = ""; uchar ledger_bytes[]; if(maximum_bytes == 0 || !BuildCheckpointLedger(ledger, reason) || StringToCharArray(ledger, ledger_bytes, 0, WHOLE_ARRAY, CP_UTF8) <= 1 || (ulong)(ArraySize(ledger_bytes) - 1) > text_limit || !DPCCCheckpointTextSHA(ledger, ledger_sha, reason)) { m_failed = true; return(false); } const int flags = FILE_WRITE | FILE_BIN | (common ? FILE_COMMON : 0); const int handle = FileOpen(actor_path, flags); if(handle == INVALID_HANDLE) { m_failed = true; reason = "offline.checkpoint.actor_open"; return(false); } ResetLastError(); bool saved = actor.Save(handle); FileFlush(handle); saved = saved && GetLastError() == 0; FileClose(handle); if(!saved || !DPCCCheckpointWriteText(ledger_path, common, ledger, reason)) { m_failed = true; if(reason == "") reason = "offline.checkpoint.actor_save_or_flush"; return(false); } string actor_sha = ""; if(!DPCCCheckpointFileSHA(actor_path, common, actor_limit, actor_sha, reason)) { m_failed = true; return(false); } CNet *candidate = new CNet; if(CheckPointer(candidate) == POINTER_INVALID || !DPCCCreatePolicyNet(candidate, m_config.shape, false, actor.GetOpenCL(), reason)) { if(CheckPointer(candidate) != POINTER_INVALID) delete candidate; m_failed = true; if(reason == "") reason = "offline.checkpoint.candidate_create"; return(false); } float candidate_error = 0, candidate_undefined = 0, candidate_forecast = 0; datetime candidate_time = 0; const bool loaded = candidate.Load(actor_path, candidate_error, candidate_undefined, candidate_forecast, candidate_time, common); if(!loaded || !ValidateCheckpointActor(candidate, m_next, reason)) { delete candidate; m_failed = true; if(reason == "") reason = "offline.checkpoint.candidate_load"; return(false); } ulong source_fingerprint = 0, candidate_fingerprint = 0; CNeuronConvOCL *hidden = dynamic_cast(candidate.Layer(1)); CNeuronConvOCL *head = dynamic_cast(candidate.Layer(2)); if(!actor.ParameterFingerprint(source_fingerprint) || !candidate.ParameterFingerprint(candidate_fingerprint) || source_fingerprint != candidate_fingerprint || CheckPointer(hidden) == POINTER_INVALID || CheckPointer(head) == POINTER_INVALID) { delete candidate; m_failed = true; reason = "offline.checkpoint.candidate_identity"; return(false); } const uint hidden_step = hidden.NativeOptimizerStep(), head_step = head.NativeOptimizerStep(); delete candidate; string verify_actor_sha = "", verify_ledger_sha = ""; if(!DPCCCheckpointFileSHA(actor_path, common, actor_limit, verify_actor_sha, reason) || verify_actor_sha != actor_sha || !DPCCCheckpointFileSHA(ledger_path, common, text_limit, verify_ledger_sha, reason) || verify_ledger_sha != ledger_sha) { m_failed = true; reason = "offline.checkpoint.artifact_changed"; return(false); } const bool forecast_bound = (StringLen(m_forecast_receipt_sha) > 0 && StringLen(m_forecast_raw_sha) > 0); string manifest = (forecast_bound ? "DPCC_OFFLINE_CHECKPOINT_V2\n" : "DPCC_OFFLINE_CHECKPOINT_V1\n"); DPCCOfflineCheckpointField(manifest, actor_path); DPCCOfflineCheckpointField(manifest, actor_sha); DPCCOfflineCheckpointField(manifest, ledger_path); DPCCOfflineCheckpointField(manifest, ledger_sha); DPCCOfflineCheckpointField(manifest, (string)m_next); DPCCOfflineCheckpointField(manifest, m_profile.seed_sha256); DPCCOfflineCheckpointField(manifest, m_profile.profile_sha256); DPCCOfflineCheckpointField(manifest, m_profile.fit_plan_sha256); DPCCOfflineCheckpointField(manifest, m_profile.model_sha256); DPCCOfflineCheckpointField(manifest, m_profile.training_sha256); DPCCOfflineCheckpointField(manifest, m_profile.bank_sha256); DPCCOfflineCheckpointField(manifest, m_profile.time_index_sha256); DPCCOfflineCheckpointField(manifest, m_profile.decoder_sha256); DPCCOfflineCheckpointField(manifest, StringFormat("%016I64X", source_fingerprint)); DPCCOfflineCheckpointField(manifest, (string)hidden_step); DPCCOfflineCheckpointField(manifest, (string)head_step); if(forecast_bound) { DPCCOfflineCheckpointField(manifest, m_forecast_receipt_sha); DPCCOfflineCheckpointField(manifest, m_forecast_raw_sha); DPCCOfflineCheckpointField(manifest, m_forecast_raw_path); } DPCCOfflineCheckpointField(manifest, (forecast_bound ? "END_DPCC_OFFLINE_CHECKPOINT_V2" : "END_DPCC_OFFLINE_CHECKPOINT_V1")); string expected_manifest_sha = ""; uchar manifest_bytes[]; if(StringToCharArray(manifest, manifest_bytes, 0, WHOLE_ARRAY, CP_UTF8) <= 1 || (ulong)(ArraySize(manifest_bytes) - 1) > text_limit || !DPCCCheckpointTextSHA(manifest, expected_manifest_sha, reason) || !DPCCCheckpointWriteText(manifest_path, common, manifest, reason)) { m_failed = true; return(false); } string readback = "", actual_manifest_sha = ""; if(!ReadCheckpointText(manifest_path, common, readback, reason) || readback != manifest || !DPCCCheckpointFileSHA(manifest_path, common, text_limit, actual_manifest_sha, reason) || actual_manifest_sha != expected_manifest_sha) { m_failed = true; reason = "offline.checkpoint.manifest_readback"; return(false); } manifest_sha = actual_manifest_sha; m_actor = GetPointer(actor); m_durable = true; reason = ""; return(true); } //--- Restore into a fresh CNet and publish neither its pointer nor the ledger until all joins pass. bool RestoreCheckpoint(const string manifest_path, const string trusted_manifest_sha, const bool common, const SDPCCOfflinePolicyProfile &profile, const SDPCCOfflineFitKey &keys[], const SDPCCOfflinePolicyAdmission &admission, const SDPCCConfig &config, CDPCCMarketClock &clock, COpenCLMy *context, CNet *&actor_out, string &reason, const ulong maximum_bytes = 536870912, const string forecast_receipt_sha = "", const string forecast_raw_sha = "", const string forecast_raw_path = "") { reason = ""; const ulong actor_limit = (maximum_bytes < 536870912 ? maximum_bytes : (ulong)536870912); const ulong text_limit = (maximum_bytes < 1048576 ? maximum_bytes : (ulong)1048576); if(m_initialized || m_failed || m_inflight || actor_out != NULL || maximum_bytes == 0 || !DPCCArchitectureSHA256Encoding(trusted_manifest_sha) || CheckPointer(context) == POINTER_INVALID || ((StringLen(forecast_receipt_sha) == 0) != (StringLen(forecast_raw_sha) == 0)) || (StringLen(forecast_receipt_sha) > 0 && (!DPCCArchitectureSHA256Encoding(forecast_receipt_sha) || !DPCCArchitectureSHA256Encoding(forecast_raw_sha) || StringLen(forecast_raw_path) == 0)) || (StringLen(forecast_receipt_sha) == 0 && StringLen(forecast_raw_path) > 0)) { reason = "offline.restore.arguments"; return(false); } string canonical = "", actual_manifest_sha = "", manifest = ""; if(!DPCCCheckpointCanonicalPath(manifest_path, canonical, reason) || canonical != manifest_path || !DPCCCheckpointFileSHA(manifest_path, common, text_limit, actual_manifest_sha, reason) || actual_manifest_sha != trusted_manifest_sha || !ReadCheckpointText(manifest_path, common, manifest, reason)) { reason = "offline.restore.trusted_manifest"; return(false); } string fields[]; int manifest_field_count = StringSplit(manifest, '\n', fields); if(manifest_field_count > 0 && fields[manifest_field_count - 1] == "") manifest_field_count = ArrayResize(fields, manifest_field_count - 1); const bool forecast_bound = (StringLen(forecast_receipt_sha) > 0); const int expected_manifest_fields = (forecast_bound ? 21 : 18); if(manifest_field_count != expected_manifest_fields || fields[0] != (forecast_bound ? "DPCC_OFFLINE_CHECKPOINT_V2" : "DPCC_OFFLINE_CHECKPOINT_V1") || fields[expected_manifest_fields - 1] != (forecast_bound ? "END_DPCC_OFFLINE_CHECKPOINT_V2" : "END_DPCC_OFFLINE_CHECKPOINT_V1") || (forecast_bound && (fields[17] != forecast_receipt_sha || fields[18] != forecast_raw_sha || fields[19] != forecast_raw_path))) { reason = "offline.restore.manifest_schema"; return(false); } string seed_sha = "", profile_sha = "", plan_sha = ""; if(!DPCCOfflinePolicyDigests(profile, keys, seed_sha, profile_sha, plan_sha, reason) || !admission.admitted || admission.rows != (ulong)ArraySize(keys) || seed_sha != admission.seed_sha256 || seed_sha != profile.seed_sha256 || profile_sha != admission.profile_sha256 || profile_sha != profile.profile_sha256 || plan_sha != admission.fit_plan_sha256 || plan_sha != profile.fit_plan_sha256 || fields[6] != seed_sha || fields[7] != profile_sha || fields[8] != plan_sha || fields[9] != profile.model_sha256 || fields[10] != profile.training_sha256 || fields[11] != profile.bank_sha256 || fields[12] != profile.time_index_sha256 || fields[13] != profile.decoder_sha256) { reason = "offline.restore.binding"; return(false); } ulong next = 0; if(!DPCCCheckpointReadUnsigned(fields[5], (ulong)ArraySize(keys), next)) { reason = "offline.restore.cursor"; return(false); } string canonical_actor = "", canonical_ledger = ""; if(!DPCCCheckpointCanonicalPath(fields[1], canonical_actor, reason) || canonical_actor != fields[1] || !DPCCCheckpointCanonicalPath(fields[3], canonical_ledger, reason) || canonical_ledger != fields[3] || !DPCCArchitectureSHA256Encoding(fields[2]) || !DPCCArchitectureSHA256Encoding(fields[4 + 0]) || !DPCCArchitectureSHA256Encoding(fields[13])) { reason = "offline.restore.artifact_fields"; return(false); } string rebuilt = (forecast_bound ? "DPCC_OFFLINE_CHECKPOINT_V2\n" : "DPCC_OFFLINE_CHECKPOINT_V1\n"); for(int i = 1; i <= 4; i++) DPCCOfflineCheckpointField(rebuilt, fields[i]); DPCCOfflineCheckpointField(rebuilt, (string)next); for(int i = 6; i <= 13; i++) DPCCOfflineCheckpointField(rebuilt, fields[i]); ulong expected_fingerprint = 0; if(!DPCCOfflineCheckpointReadHex64(fields[14], expected_fingerprint)) { reason = "offline.restore.fingerprint_encoding"; return(false); } ulong hidden_step = 0, head_step = 0; if(!DPCCCheckpointReadUnsigned(fields[15], UINT_MAX, hidden_step) || !DPCCCheckpointReadUnsigned(fields[16], UINT_MAX, head_step) || (ulong)profile.initial_step + next != hidden_step || hidden_step != head_step) { reason = "offline.restore.optimizer_cursor"; return(false); } DPCCOfflineCheckpointField(rebuilt, fields[14]); DPCCOfflineCheckpointField(rebuilt, (string)hidden_step); DPCCOfflineCheckpointField(rebuilt, (string)head_step); if(forecast_bound) { DPCCOfflineCheckpointField(rebuilt, fields[17]); DPCCOfflineCheckpointField(rebuilt, fields[18]); DPCCOfflineCheckpointField(rebuilt, fields[19]); } DPCCOfflineCheckpointField(rebuilt, (forecast_bound ? "END_DPCC_OFFLINE_CHECKPOINT_V2" : "END_DPCC_OFFLINE_CHECKPOINT_V1")); if(rebuilt != manifest) { reason = "offline.restore.manifest_noncanonical"; return(false); } string actual_actor_sha = "", actual_ledger_sha = "", ledger = ""; if(!DPCCCheckpointFileSHA(fields[1], common, actor_limit, actual_actor_sha, reason) || actual_actor_sha != fields[2] || !DPCCCheckpointFileSHA(fields[3], common, text_limit, actual_ledger_sha, reason) || actual_ledger_sha != fields[4] || !ReadCheckpointText(fields[3], common, ledger, reason)) { reason = "offline.restore.artifact_hash"; return(false); } string ledger_fields[]; int ledger_field_count = StringSplit(ledger, '\n', ledger_fields); if(ledger_field_count > 0 && ledger_fields[ledger_field_count - 1] == "") ledger_field_count = ArrayResize(ledger_fields, ledger_field_count - 1); const int expected_ledger_fields = (forecast_bound ? 105 : 102); if(ledger_field_count != expected_ledger_fields || ledger_fields[0] != (forecast_bound ? "DPCC_OFFLINE_LEDGER_V2" : "DPCC_OFFLINE_LEDGER_V1") || ledger_fields[expected_ledger_fields - 1] != (forecast_bound ? "END_DPCC_OFFLINE_LEDGER_V2" : "END_DPCC_OFFLINE_LEDGER_V1") || (forecast_bound && (ledger_fields[101] != forecast_receipt_sha || ledger_fields[102] != forecast_raw_sha || ledger_fields[103] != forecast_raw_path))) { reason = "offline.restore.ledger_schema"; return(false); } for(int i = 0; i < 8; i++) if(ledger_fields[i + 1] != fields[i + 6]) { reason = "offline.restore.ledger_binding"; return(false); } ulong ledger_next = 0, key_count = 0; if(!DPCCCheckpointReadUnsigned(ledger_fields[9], (ulong)ArraySize(keys), ledger_next) || ledger_next != next || !DPCCCheckpointReadUnsigned(ledger_fields[10], (ulong)ArraySize(keys), key_count) || key_count != (ulong)ArraySize(keys)) { reason = "offline.restore.ledger_cursor"; return(false); } float state[]; ArrayResize(state, 33); for(int i = 0; i < 33; i++) if(!DPCCOfflineCheckpointReadFloat(ledger_fields[32 + i], state[i])) { reason = "offline.restore.state_encoding"; return(false); } const string expected_complete = (string)(next == (ulong)ArraySize(keys)); const string expected_terminal = (string)(state[24] != 0); const string expected_floor = (string)DPCCTrainingFloorReached(state[0], state[1]); if(ledger_fields[29] != expected_complete || ledger_fields[30] != expected_terminal || ledger_fields[31] != expected_floor) { reason = "offline.restore.terminal_floor_join"; return(false); } for(int i = 0; i < 24; i++) { float risk = 0.0f; if(!DPCCOfflineCheckpointReadFloat(ledger_fields[65 + i], risk)) { reason = "offline.restore.risk_encoding"; return(false); } UDPCCObservedFloatBits saved, expected; saved.value = risk; expected.value = profile.seed.risk_limits[i]; if(saved.bits != expected.bits) { reason = "offline.restore.risk_binding"; return(false); } } SDPCCSimulatedPositionMetadata position; ZeroMemory(position); position.initialized = (ledger_fields[89] == "1"); if((ledger_fields[89] != "0" && ledger_fields[89] != "1")) { reason = "offline.restore.position_flag"; return(false); } long position_values[]; ArrayResize(position_values, 10); for(int i = 0; i < 10; i++) { position_values[i] = StringToInteger(ledger_fields[90 + i]); if((string)position_values[i] != ledger_fields[90 + i]) { reason = "offline.restore.position_integer"; return(false); } } position.snapshot_bar = position_values[0]; position.snapshot_server = position_values[1]; position.time_domain = "UTC_SECONDS_V1"; position.snapshot_time = position_values[2]; position.bar_server_open = position_values[3]; position.bar_server_close = position_values[4]; position.bar_open_time = position_values[5]; position.bar_close_time = position_values[6]; position.first_open_bar = position_values[7]; position.first_open_server = position_values[8]; position.first_open_time = position_values[9]; if(!DPCCOfflineCheckpointReadFloat(ledger_fields[100], position.signed_lots)) { reason = "offline.restore.position_lots"; return(false); } if(!ValidateCheckpointPosition(profile, next, keys, state, position, clock, reason)) return(false); CDPCCOfflineRowOwner staged; if(!staged.Initialize(profile, keys, admission, config, reason, forecast_receipt_sha, forecast_raw_sha, forecast_raw_path)) return(false); staged.m_next = next; if(ArrayCopy(staged.m_state, state) != 33) { reason = "offline.restore.staged_state_copy"; return(false); } staged.m_position = position; staged.m_floor_reached = DPCCTrainingFloorReached(state[0], state[1]); string expected_ledger = ""; if(!staged.BuildCheckpointLedger(expected_ledger, reason) || expected_ledger != ledger) { reason = "offline.restore.cursor_position_join"; return(false); } if(!DPCCCheckpointFileSHA(manifest_path, common, text_limit, actual_manifest_sha, reason) || actual_manifest_sha != trusted_manifest_sha || !DPCCCheckpointFileSHA(fields[1], common, actor_limit, actual_actor_sha, reason) || actual_actor_sha != fields[2] || !DPCCCheckpointFileSHA(fields[3], common, text_limit, actual_ledger_sha, reason) || actual_ledger_sha != fields[4]) { reason = "offline.restore.rehash_before_publish"; return(false); } CNet *candidate = new CNet; if(CheckPointer(candidate) == POINTER_INVALID || !DPCCCreatePolicyNet(candidate, config.shape, false, context, reason)) { if(CheckPointer(candidate) != POINTER_INVALID) delete candidate; if(reason == "") reason = "offline.restore.candidate_create"; return(false); } float candidate_error = 0, candidate_undefined = 0, candidate_forecast = 0; datetime candidate_time = 0; if(!candidate.Load(fields[1], candidate_error, candidate_undefined, candidate_forecast, candidate_time, common) || !staged.ValidateCheckpointActor(candidate, next, reason)) { delete candidate; if(reason == "") reason = "offline.restore.actor_load_validate"; return(false); } ulong fingerprint = 0; if(!candidate.ParameterFingerprint(fingerprint) || fingerprint != expected_fingerprint || !DPCCCheckpointFileSHA(fields[1], common, actor_limit, actual_actor_sha, reason) || actual_actor_sha != fields[2] || !DPCCCheckpointFileSHA(fields[3], common, text_limit, actual_ledger_sha, reason) || actual_ledger_sha != fields[4] || !DPCCCheckpointFileSHA(manifest_path, common, text_limit, actual_manifest_sha, reason) || actual_manifest_sha != trusted_manifest_sha) { delete candidate; reason = "offline.restore.actor_identity_rehash"; return(false); } if(!Initialize(profile, keys, admission, config, reason, forecast_receipt_sha, forecast_raw_sha, forecast_raw_path) || ArrayCopy(m_state, state) != 33) { delete candidate; m_failed = true; reason = "offline.restore.owner_publish"; return(false); } m_next = next; m_position = position; m_floor_reached = staged.m_floor_reached; m_restored_actor_path = fields[1]; m_restored_actor_sha = actual_actor_sha; m_restored_actor_fingerprint = fingerprint; m_actor = candidate; m_clock = GetPointer(clock); m_durable = true; actor_out = candidate; reason = ""; return(true); } //--- The legacy selected decoder SHA is retained only as stage/admission identity. //--- normalized_scenarios is supplied by the separately pinned T15 Forecast/Decoder. bool Step(CDPCCMarketClock &clock, const SDPCCOfflineFitWitness &row, const int observation_record, const string observation_window, const string stage_decoder_sha, const long observation_time, const float &normalized_scenarios[], const float &row_context[], const float &target_native_ohlc[], CNet &actor, CNet &factual, CNet &scoring, SDPCCOfflineRowResult &out, string &reason) { DPCCClearOfflineRowResult(out); reason = ""; if(!m_initialized || m_failed || m_inflight || !m_durable || m_next >= (ulong)ArraySize(m_keys)) { if(m_initialized && !m_durable) m_failed = true; return(Fail("offline.row.owner_boundary", reason)); } if(m_clock != NULL && m_clock != GetPointer(clock)) return(Fail("offline.row.clock_owner_identity", reason)); const SDPCCOfflineFitKey key = m_keys[(int)m_next]; const uint horizon = m_profile.horizon; if(!row.admitted || row.split != "fit" || row.horizon != (long)horizon || row.record_sha256 != key.record_sha256 || row.window_id != key.window_id || row.episode_id != key.episode_id || row.decision_bar != key.decision_bar || row.decision_time != key.decision_time || row.decision_identity != key.decision_identity || row.context_identity != key.context_identity || row.owner_identity != key.owner_identity || observation_record != key.record_index || observation_window != key.window_id || stage_decoder_sha != m_profile.decoder_sha256 || observation_time != key.decision_time || ArraySize(row.bar_indices) != (int)horizon || ArraySize(row.bar_open_times) != (int)horizon || ArraySize(row.bar_close_times) != (int)horizon || ArraySize(normalized_scenarios) != (int)(m_config.shape.K * horizon * 4)) return(Fail("offline.row.record_decoder_binding", reason)); for(int i = 0; i < ArraySize(normalized_scenarios); i++) if(!MathIsValidNumber((double)normalized_scenarios[i])) return(Fail("offline.row.decoder_nonfinite", reason)); float causal[]; if(!DPCCPackDecoderInput(m_config.shape, row_context, causal, reason)) return(false); long opens[], closes[], ordinals[]; if(ArrayResize(opens, (int)horizon) != (int)horizon || ArrayResize(closes, (int)horizon) != (int)horizon || ArrayResize(ordinals, (int)horizon) != (int)horizon) return(Fail("offline.row.clock_capacity", reason)); SDPCCClockBar first; for(uint h = 0; h < horizon; h++) { SDPCCClockBar bar; long utc = 0, end = 0; if(!clock.IndexedBar(key.decision_bar + h, bar, utc, end, reason) || bar.ordinal != key.decision_bar + (long)h || row.bar_indices[h] != bar.ordinal || row.bar_open_times[h] != utc || row.bar_close_times[h] != end || (h == 0 && (utc != key.decision_time || (m_next == 0 && (bar.server_open != m_profile.seed.snapshot_server || utc != m_profile.seed.snapshot_utc))))) return(Fail("offline.row.clock_binding", reason)); if(h == 0) first = bar; opens[h] = bar.server_open; closes[h] = bar.server_close; ordinals[h] = bar.ordinal; } COpenCLMy *device = actor.GetOpenCL(); CNeuronScenarioRolloutOCL *native = dynamic_cast(factual.Layer(1)); CNeuronScenarioRolloutOCL *score_native = dynamic_cast(scoring.Layer(1)); SDPCCShape physical = m_config.shape; physical.K = 1; physical.M = 1; physical.Depth = 0; SDPCCConfig native_config = m_config; native_config.shape = physical; if(CheckPointer(device) == POINTER_INVALID) return(Fail("offline.row.native_actor_context", reason)); if(factual.GetOpenCL() != device || scoring.GetOpenCL() != device) return(Fail("offline.row.native_context_equality", reason)); if(GetPointer(actor) == GetPointer(factual) || GetPointer(actor) == GetPointer(scoring) || GetPointer(factual) == GetPointer(scoring)) return(Fail("offline.row.native_net_identity", reason)); if(CheckPointer(native) == POINTER_INVALID || CheckPointer(score_native) == POINTER_INVALID) return(Fail("offline.row.native_layer_identity", reason)); if(native.ContractSignature() != DPCCFixedSignature(native_config, defNeuronScenarioRolloutOCL)) return(Fail("offline.row.native_context_signature", reason)); if(score_native.ContractSignature() != native.ContractSignature()) return(Fail("offline.row.native_scoring_signature", reason)); if(m_actor != NULL && m_actor != GetPointer(actor)) return(Fail("offline.row.actor_owner_identity", reason)); if(m_factual != NULL && m_factual != GetPointer(factual)) return(Fail("offline.row.factual_owner_identity", reason)); if(m_scoring != NULL && m_scoring != GetPointer(scoring)) return(Fail("offline.row.scoring_owner_identity", reason)); SDPCCLayout source_layout, unused; uint tokens = 0; if(!DPCCCompositeLayouts(m_config.shape, source_layout, unused, tokens)) return(Fail("offline.row.layout", reason)); float context[]; if(ArrayCopy(context, row_context) != ArraySize(row_context)) return(Fail("offline.row.context_capacity", reason)); for(int i = 0; i < 24; i++) { context[(int)source_layout.offsets[4] + i] = m_state[i]; if(context[(int)source_layout.offsets[6] + i] != m_profile.seed.risk_limits[i]) return(Fail("offline.row.risk_binding", reason)); } float step_values[]; ENUM_SYMBOL_CALC_MODE calc = SYMBOL_CALC_MODE_FOREX; if(!DPCCBuildHistoricalBarStepValues(m_config.shape, context, target_native_ohlc, key.decision_bar, opens, closes, ordinals, row.bar_open_times[0], row.bar_close_times[0], step_values, calc, reason)) return(false); //--- From this point, a failure may leave a pending native token or changed Actor. m_inflight = true; m_actor = GetPointer(actor); m_factual = GetPointer(factual); m_scoring = GetPointer(scoring); CBufferFloat prior, scenarios; if(!DPCCUploadHistoricalBuffer(scenarios, normalized_scenarios, device)) return(Fail("offline.row.native_buffers", reason)); const uint episode_step = (uint)m_next; const uint episode_horizon = (uint)(m_profile.last_decision_bar - m_profile.first_decision_bar + horizon); SDPCCHistoricalPreparedBar prepared; if(!DPCCPrepareHistoricalBar(*native, step_values, key.decision_bar, key.decision_identity, first.server_open, first.server_close, row.bar_open_times[0], row.bar_close_times[0], calc, m_state, m_position, episode_step, episode_horizon, m_next == 0, prior, prepared, reason)) return(Fail(reason, reason)); if(m_next == 0) for(int i = 0; i < 33; i++) { UDPCCObservedFloatBits actual, expected; actual.value = m_state[i]; expected.value = prepared.state[i]; if(actual.bits != expected.bits) return(Fail("offline.row.native_seed_mismatch.state_" + (string)i, reason)); } CBufferFloat *continuation = (m_next == 0 ? NULL : GetPointer(prior)); SDPCCOfflineRowResult candidate; DPCCClearOfflineRowResult(candidate); if(ArrayCopy(candidate.account13, prepared.account13) != 13) return(Fail("offline.row.account_capacity", reason)); if(!DPCCUpdateHistoricalOracleActor(actor, scoring, m_config.shape, GetPointer(scenarios), candidate.account13, context, target_native_ohlc, first.server_open, key.decision_bar, opens, closes, ordinals, m_profile.coordinate_deltas, m_config, 0.0f, continuation, episode_step, episode_horizon, candidate.pre_update_plan, candidate.executed_action, candidate.teacher_plan, candidate.valid_horizons, candidate.teacher_score, candidate.teacher_source, reason)) return(Fail(reason, reason)); SDPCCHistoricalCommittedBar committed; if(!DPCCCommitHistoricalBar(*native, prepared, candidate.executed_action, committed, reason)) return(Fail(reason, reason)); if(ArrayCopy(candidate.state, committed.state) != 33) return(Fail("offline.row.state_capacity", reason)); candidate.position = committed.position; candidate.reward = committed.reward; candidate.decision_bar = key.decision_bar; candidate.next_row = m_next + 1; candidate.committed = true; candidate.floor_reached = committed.floor_reached; if(ArrayCopy(m_state, candidate.state) != 33) return(Fail("offline.row.boundary_capacity", reason)); m_position = candidate.position; m_next++; m_floor_reached = candidate.floor_reached; m_clock = GetPointer(clock); m_inflight = false; m_durable = false; out = candidate; return(true); } }; //+------------------------------------------------------------------+ //| Online owner profile binds caller provenance and fixed cadence | //+------------------------------------------------------------------+ //| PhysicalAccountDay.mqh | //+------------------------------------------------------------------+ #ifndef DPCC_PHYSICAL_ACCOUNT_DAY_MQH #define DPCC_PHYSICAL_ACCOUNT_DAY_MQH //+------------------------------------------------------------------+ //| PhysicalAccount.mqh | //+------------------------------------------------------------------+ #ifndef DPCC_PHYSICAL_ACCOUNT_MQH #define DPCC_PHYSICAL_ACCOUNT_MQH //+------------------------------------------------------------------+ //| Physical account observation and caller-owned ledger | //+------------------------------------------------------------------+ struct SDPCCPhysicalPolicy { long login; string server; string currency; string symbol; long magic; string deployment_id; uint max_positions; uint max_history_deals; uint max_quote_age_seconds; double volume_atol; double money_atol; double money_rtol; }; struct SDPCCPhysicalLedger { long login; string server; string currency; string symbol; long magic; string deployment_id; string checkpoint_id; datetime previous_mark; double previous_balance; double previous_equity; datetime snapshot_time; double snapshot_equity; datetime day_open; double day_start_equity; double running_peak_equity; bool peak_initialized; bool terminal; bool day_reset_observed; long deal_cursor_msc; ulong deal_cursor_ticket; double realized_booked; double paid_costs_booked; bool booked_costs_confirmed; string booked_costs_evidence; }; struct SDPCCPhysicalConfirmedPosition { ulong ticket; ulong identifier; long updated_msc; double volume; double open_price; double sl; double tp; double swap; }; struct SDPCCPhysicalConfirmation { string checkpoint_id; datetime asof; bool accrued_cost_known; double accrued_unbooked_cost; string accrued_cost_evidence; bool margin_allocation_known; double owned_margin; double other_margin; string margin_evidence; SDPCCPhysicalConfirmedPosition positions[]; }; struct SDPCCPhysicalPosition { ulong ticket; ulong identifier; long magic; long type; long opened_msc; long updated_msc; double volume; double open_price; double sl; double tp; double profit; double swap; bool history_owned; }; struct SDPCCPhysicalDeal { ulong ticket; ulong position_id; long magic; long type; long entry; long time_msc; string symbol; double volume; double profit; double commission; double fee; double swap; bool history_owned; }; struct SDPCCPhysicalObservation { long login; string server; string currency; datetime asof; long margin_mode; uint all_positions; uint all_orders; double balance; double equity; double margin; double free_margin; double credit; double account_profit; MqlTick tick; SDPCCPhysicalPosition positions[]; SDPCCPhysicalDeal deals[]; }; struct SDPCCPhysicalAccount { bool complete; bool native_observed; bool clock_bound; long asof_utc; long previous_mark_utc; long quote_utc_msc; long day_open_utc; datetime asof; datetime previous_mark; long quote_msc; uint all_positions; uint all_orders; SDPCCPhysicalPosition positions[]; long login; string server; string currency; string symbol; string deployment_id; string checkpoint_id; string ownership_evidence; bool original_symbol_scope_verified; double account[24]; bool present[24]; float features[13]; double buy_lots; double sell_lots; double buy_profit; double sell_profit; double position_discount; double previous_balance; double previous_equity; SDPCCPhysicalLedger next_ledger; }; //+------------------------------------------------------------------+ //| Physical helpers | //+------------------------------------------------------------------+ bool DPCCPhysicalRefuse(const string message, string &reason) { reason = "BLOCKED.physical." + message; return(false); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalDayOpen | //+------------------------------------------------------------------+ datetime DPCCPhysicalDayOpen(const datetime value) { MqlDateTime parts; if(!TimeToStruct(value, parts)) return(0); parts.hour = 0; parts.min = 0; parts.sec = 0; return(StructToTime(parts)); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalPolicyCheck | //+------------------------------------------------------------------+ bool DPCCPhysicalPolicyCheck(const SDPCCPhysicalPolicy &policy, string &reason) { if(policy.login <= 0 || policy.server == "" || policy.currency == "" || policy.symbol == "" || policy.magic <= 0 || policy.deployment_id == "" || policy.max_positions == 0 || policy.max_history_deals == 0) return(DPCCPhysicalRefuse("policy.identity", reason)); if(!MathIsValidNumber(policy.volume_atol) || policy.volume_atol < 0 || !MathIsValidNumber(policy.money_atol) || policy.money_atol < 0 || !MathIsValidNumber(policy.money_rtol) || policy.money_rtol < 0) return(DPCCPhysicalRefuse("policy.tolerances", reason)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalAfterCursor | //+------------------------------------------------------------------+ bool DPCCPhysicalAfterCursor(const SDPCCPhysicalDeal &deal, const SDPCCPhysicalLedger &ledger) { return(deal.time_msc > ledger.deal_cursor_msc || (deal.time_msc == ledger.deal_cursor_msc && deal.ticket > ledger.deal_cursor_ticket)); } //+------------------------------------------------------------------+ //| Read one deal without resetting the selected history list | //+------------------------------------------------------------------+ bool DPCCPhysicalReadDeal(const ulong ticket, SDPCCPhysicalDeal &deal, string &reason) { ZeroMemory(deal); deal.ticket = ticket; long position_id = 0; if(ticket == 0 || !HistoryDealGetInteger(ticket, DEAL_POSITION_ID, position_id) || !HistoryDealGetInteger(ticket, DEAL_MAGIC, deal.magic) || !HistoryDealGetInteger(ticket, DEAL_TYPE, deal.type) || !HistoryDealGetInteger(ticket, DEAL_ENTRY, deal.entry) || !HistoryDealGetInteger(ticket, DEAL_TIME_MSC, deal.time_msc)) return(DPCCPhysicalRefuse("deal.integer_observation", reason)); deal.position_id = (ulong)position_id; if(!HistoryDealGetString(ticket, DEAL_SYMBOL, deal.symbol)) return(DPCCPhysicalRefuse("deal.symbol_observation", reason)); if(!HistoryDealGetDouble(ticket, DEAL_VOLUME, deal.volume) || !HistoryDealGetDouble(ticket, DEAL_PROFIT, deal.profit) || !HistoryDealGetDouble(ticket, DEAL_COMMISSION, deal.commission) || !HistoryDealGetDouble(ticket, DEAL_FEE, deal.fee) || !HistoryDealGetDouble(ticket, DEAL_SWAP, deal.swap)) return(DPCCPhysicalRefuse("deal.money_observation", reason)); const double values[] = {deal.volume, deal.profit, deal.commission, deal.fee, deal.swap}; for(int i = 0; i < ArraySize(values); i++) if(!MathIsValidNumber(values[i])) return(DPCCPhysicalRefuse("deal.nonfinite", reason)); return(true); } //+------------------------------------------------------------------+ //| Prove the whole position identifier belongs to one owner | //+------------------------------------------------------------------+ bool DPCCPhysicalProvePosition(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalPosition &position, const SDPCCPhysicalDeal &history[], const datetime asof, string &reason, CDPCCMarketClock *clock = NULL, const long captured_asof_utc = -1) { long checked_asof_utc = 0; if(clock) { if(CheckPointer(clock) == POINTER_INVALID) return(DPCCPhysicalRefuse("ownership.invalid_clock_owner", reason)); if(!clock.Convert((long)asof, checked_asof_utc, reason)) return(false); if(captured_asof_utc < 0 || checked_asof_utc != captured_asof_utc) return(DPCCPhysicalRefuse("ownership.UTC_capture_asof_binding", reason)); } if(ArraySize(history) == 0 || (uint)ArraySize(history) > policy.max_history_deals) return(DPCCPhysicalRefuse("ownership.history_count", reason)); double signed_volume = 0; bool first_trade = true; long last_msc = 0; long last_utc_msc = 0; ulong last_ticket = 0; for(int i = 0; i < ArraySize(history); i++) { const SDPCCPhysicalDeal deal = history[i]; if(deal.type != DEAL_TYPE_BUY && deal.type != DEAL_TYPE_SELL) return(DPCCPhysicalRefuse("ownership.unsupported_position_deal", reason)); if(deal.position_id != position.identifier || deal.symbol != policy.symbol || deal.magic != policy.magic || deal.time_msc <= 0 || deal.time_msc / 1000 > (long)asof || deal.volume <= 0 || !MathIsValidNumber(deal.volume) || deal.time_msc < last_msc || (deal.time_msc == last_msc && deal.ticket <= last_ticket)) return(DPCCPhysicalRefuse("ownership.mixed_or_incomplete_history", reason)); if(first_trade && deal.entry != DEAL_ENTRY_IN) return(DPCCPhysicalRefuse("ownership.missing_opening_deal", reason)); if(deal.entry != DEAL_ENTRY_IN && deal.entry != DEAL_ENTRY_OUT && deal.entry != DEAL_ENTRY_INOUT && deal.entry != DEAL_ENTRY_OUT_BY) return(DPCCPhysicalRefuse("ownership.unsupported_entry", reason)); const double change = deal.type == DEAL_TYPE_BUY ? deal.volume : -deal.volume; if((deal.entry == DEAL_ENTRY_IN && signed_volume * change < 0) || ((deal.entry == DEAL_ENTRY_OUT || deal.entry == DEAL_ENTRY_OUT_BY) && (signed_volume * change >= 0 || deal.volume > MathAbs(signed_volume) + policy.volume_atol)) || (deal.entry == DEAL_ENTRY_INOUT && (signed_volume * change >= 0 || deal.volume <= MathAbs(signed_volume)))) return(DPCCPhysicalRefuse("ownership.entry_volume_semantics", reason)); signed_volume += change; first_trade = false; if(clock) { long deal_utc = 0; if(!clock.ConvertMilliseconds(deal.time_msc, deal_utc, reason)) return(false); if(deal_utc < last_utc_msc || deal_utc / 1000 > checked_asof_utc) return(DPCCPhysicalRefuse("ownership.UTC_deal_order_or_causality", reason)); last_utc_msc = deal_utc; } last_msc = deal.time_msc; last_ticket = deal.ticket; } const double expected = position.type == POSITION_TYPE_BUY ? position.volume : -position.volume; if(!MathIsValidNumber(signed_volume) || MathAbs(signed_volume - expected) > policy.volume_atol) return(DPCCPhysicalRefuse("ownership.history_volume_mismatch", reason)); return(true); } //+------------------------------------------------------------------+ //| Read a selected position and its complete identifier history | //+------------------------------------------------------------------+ bool DPCCPhysicalReadPosition(const SDPCCPhysicalPolicy &policy, const ulong ticket, const datetime asof, SDPCCPhysicalPosition &position, string &reason, CDPCCMarketClock *clock = NULL, const long captured_asof_utc = -1) { ZeroMemory(position); position.ticket = ticket; if(!PositionSelectByTicket(ticket)) return(DPCCPhysicalRefuse("position.select", reason)); long identifier = 0; if(!PositionGetInteger(POSITION_IDENTIFIER, identifier) || !PositionGetInteger(POSITION_MAGIC, position.magic) || !PositionGetInteger(POSITION_TYPE, position.type) || !PositionGetInteger(POSITION_TIME_MSC, position.opened_msc) || !PositionGetInteger(POSITION_TIME_UPDATE_MSC, position.updated_msc)) return(DPCCPhysicalRefuse("position.integer_observation", reason)); position.identifier = (ulong)identifier; if(!PositionGetDouble(POSITION_VOLUME, position.volume) || !PositionGetDouble(POSITION_PRICE_OPEN, position.open_price) || !PositionGetDouble(POSITION_SL, position.sl) || !PositionGetDouble(POSITION_TP, position.tp) || !PositionGetDouble(POSITION_PROFIT, position.profit) || !PositionGetDouble(POSITION_SWAP, position.swap)) return(DPCCPhysicalRefuse("position.money_observation", reason)); if(!HistorySelectByPosition(position.identifier)) return(DPCCPhysicalRefuse("ownership.history_select", reason)); const int count = HistoryDealsTotal(); if(count <= 0 || (uint)count > policy.max_history_deals) return(DPCCPhysicalRefuse("ownership.history_budget", reason)); SDPCCPhysicalDeal history[]; if(ArrayResize(history, count) != count) return(DPCCPhysicalRefuse("ownership.history_capacity", reason)); for(int i = 0; i < count; i++) if(!DPCCPhysicalReadDeal(HistoryDealGetTicket(i), history[i], reason)) return(false); position.history_owned = DPCCPhysicalProvePosition(policy, position, history, asof, reason, clock, captured_asof_utc); return(position.history_owned); } //+------------------------------------------------------------------+ //| Read terminal values and reject observable concurrent changes | //+------------------------------------------------------------------+ bool DPCCPhysicalObserve(const SDPCCPhysicalPolicy &policy, const datetime history_from, SDPCCPhysicalObservation &observation, string &reason, CDPCCMarketClock *clock = NULL) { ArrayFree(observation.positions); ArrayFree(observation.deals); ZeroMemory(observation); reason = ""; if(clock && CheckPointer(clock) == POINTER_INVALID) return(DPCCPhysicalRefuse("invalid_clock_owner", reason)); if(!DPCCPhysicalPolicyCheck(policy, reason)) return(false); observation.asof = TimeCurrent(); long captured_asof_utc = 0; if(clock && !clock.Convert((long)observation.asof, captured_asof_utc, reason)) return(false); const datetime server_now = TimeTradeServer(); long server_age = (long)(server_now - observation.asof); if(clock && !clock.Elapsed((long)observation.asof, (long)server_now, server_age, reason)) return(false); if(!TerminalInfoInteger(TERMINAL_CONNECTED) || server_now < observation.asof || (ulong)server_age > policy.max_quote_age_seconds) return(DPCCPhysicalRefuse("observation.disconnected_or_stale_server_time", reason)); observation.login = AccountInfoInteger(ACCOUNT_LOGIN); observation.server = AccountInfoString(ACCOUNT_SERVER); observation.currency = AccountInfoString(ACCOUNT_CURRENCY); observation.margin_mode = AccountInfoInteger(ACCOUNT_MARGIN_MODE); observation.balance = AccountInfoDouble(ACCOUNT_BALANCE); observation.equity = AccountInfoDouble(ACCOUNT_EQUITY); observation.margin = AccountInfoDouble(ACCOUNT_MARGIN); observation.free_margin = AccountInfoDouble(ACCOUNT_MARGIN_FREE); observation.credit = AccountInfoDouble(ACCOUNT_CREDIT); observation.account_profit = AccountInfoDouble(ACCOUNT_PROFIT); if(observation.login != policy.login || observation.server != policy.server || observation.currency != policy.currency || history_from < 0 || history_from > observation.asof) return(DPCCPhysicalRefuse("observation.account_identity_or_time", reason)); if(!SymbolInfoTick(policy.symbol, observation.tick)) return(DPCCPhysicalRefuse("observation.quote", reason)); long quote_age = (long)(server_now - observation.tick.time); if(clock && !clock.Elapsed((long)observation.tick.time, (long)server_now, quote_age, reason)) return(false); if(observation.tick.time <= 0 || observation.tick.time > server_now || (ulong)quote_age > policy.max_quote_age_seconds) return(DPCCPhysicalRefuse("observation.quote_age_against_server_clock", reason)); const int total = PositionsTotal(); if(total < 0 || (uint)total > policy.max_positions) return(DPCCPhysicalRefuse("observation.position_budget", reason)); observation.all_positions = (uint)total; observation.all_orders = (uint)OrdersTotal(); for(int i = 0; i < total; i++) { const ulong ticket = PositionGetTicket(i); string symbol = ""; if(ticket == 0 || !PositionGetString(POSITION_SYMBOL, symbol)) return(DPCCPhysicalRefuse("observation.position_symbol", reason)); if(symbol != policy.symbol) continue; const int index = ArraySize(observation.positions); if(ArrayResize(observation.positions, index + 1) != index + 1) return(DPCCPhysicalRefuse("observation.position_capacity", reason)); if(!DPCCPhysicalReadPosition(policy, ticket, observation.asof, observation.positions[index], reason, clock, captured_asof_utc)) return(false); } //--- History starts at the persisted cursor's server second; the exact cursor filters overlap. if(!HistorySelect(history_from, observation.asof)) return(DPCCPhysicalRefuse("observation.ledger_history_select", reason)); const int deals = HistoryDealsTotal(); if(deals < 0 || (uint)deals > policy.max_history_deals) return(DPCCPhysicalRefuse("observation.ledger_history_budget", reason)); if(ArrayResize(observation.deals, deals) != deals) return(DPCCPhysicalRefuse("observation.ledger_history_capacity", reason)); for(int i = 0; i < deals; i++) if(!DPCCPhysicalReadDeal(HistoryDealGetTicket(i), observation.deals[i], reason)) return(false); //--- Closed positions also need whole-identifier ownership, not only the closing deal's magic. for(int i = 0; i < deals; i++) { if((observation.deals[i].type != DEAL_TYPE_BUY && observation.deals[i].type != DEAL_TYPE_SELL) || observation.deals[i].symbol != policy.symbol || observation.deals[i].magic != policy.magic) continue; const ulong identifier = observation.deals[i].position_id; bool already_proven = false; for(int j = 0; j < i; j++) if(observation.deals[j].position_id == identifier && observation.deals[j].history_owned) already_proven = true; if(already_proven) { observation.deals[i].history_owned = true; continue; } SDPCCPhysicalPosition target; ZeroMemory(target); target.identifier = identifier; target.type = POSITION_TYPE_BUY; for(int j = 0; j < ArraySize(observation.positions); j++) if(observation.positions[j].identifier == identifier) target = observation.positions[j]; if(!HistorySelectByPosition(identifier)) return(DPCCPhysicalRefuse("observation.closed_identifier_history_select", reason)); const int count = HistoryDealsTotal(); if(count <= 0 || (uint)count > policy.max_history_deals) return(DPCCPhysicalRefuse("observation.closed_identifier_history_budget", reason)); SDPCCPhysicalDeal history[]; if(ArrayResize(history, count) != count) return(DPCCPhysicalRefuse("observation.closed_identifier_history_capacity", reason)); for(int j = 0; j < count; j++) if(!DPCCPhysicalReadDeal(HistoryDealGetTicket(j), history[j], reason)) return(false); if(!DPCCPhysicalProvePosition(policy, target, history, observation.asof, reason, clock, captured_asof_utc)) return(false); observation.deals[i].history_owned = true; } //--- Account and each owned position must remain identical through the read boundary. if(TimeCurrent() != observation.asof || PositionsTotal() != total || (uint)OrdersTotal() != observation.all_orders || AccountInfoInteger(ACCOUNT_LOGIN) != observation.login || AccountInfoString(ACCOUNT_SERVER) != observation.server || AccountInfoString(ACCOUNT_CURRENCY) != observation.currency || AccountInfoDouble(ACCOUNT_BALANCE) != observation.balance || AccountInfoDouble(ACCOUNT_EQUITY) != observation.equity || AccountInfoDouble(ACCOUNT_MARGIN) != observation.margin || AccountInfoDouble(ACCOUNT_MARGIN_FREE) != observation.free_margin || AccountInfoDouble(ACCOUNT_CREDIT) != observation.credit || AccountInfoDouble(ACCOUNT_PROFIT) != observation.account_profit) return(DPCCPhysicalRefuse("observation.concurrent_account_change", reason)); for(int i = 0; i < ArraySize(observation.positions); i++) { const SDPCCPhysicalPosition position = observation.positions[i]; if(!PositionSelectByTicket(position.ticket) || (ulong)PositionGetInteger(POSITION_IDENTIFIER) != position.identifier || PositionGetInteger(POSITION_MAGIC) != position.magic || PositionGetInteger(POSITION_TYPE) != position.type || PositionGetInteger(POSITION_TIME_MSC) != position.opened_msc || PositionGetInteger(POSITION_TIME_UPDATE_MSC) != position.updated_msc || PositionGetDouble(POSITION_VOLUME) != position.volume || PositionGetDouble(POSITION_PRICE_OPEN) != position.open_price || PositionGetDouble(POSITION_SL) != position.sl || PositionGetDouble(POSITION_TP) != position.tp || PositionGetDouble(POSITION_PROFIT) != position.profit || PositionGetDouble(POSITION_SWAP) != position.swap) return(DPCCPhysicalRefuse("observation.concurrent_position_change", reason)); } MqlTick final_tick; if(!SymbolInfoTick(policy.symbol, final_tick) || final_tick.time_msc != observation.tick.time_msc || final_tick.bid != observation.tick.bid || final_tick.ask != observation.tick.ask) return(DPCCPhysicalRefuse("observation.concurrent_quote_change", reason)); return(true); } //+------------------------------------------------------------------+ //| Bootstrap an observed flat account with no activity this day | //+------------------------------------------------------------------+ bool DPCCPhysicalInitializeFlat(const SDPCCPhysicalPolicy &policy, const string checkpoint_id, const string no_unbooked_cost_evidence, SDPCCPhysicalLedger &ledger, string &reason) { ZeroMemory(ledger); if(checkpoint_id == "" || no_unbooked_cost_evidence == "") return(DPCCPhysicalRefuse("initialize.explicit_no_unbooked_cost_evidence", reason)); const datetime day_open = DPCCPhysicalDayOpen(TimeCurrent()); SDPCCPhysicalObservation observation; if(!DPCCPhysicalObserve(policy, day_open, observation, reason)) return(false); if(day_open <= 0 || DPCCPhysicalDayOpen(observation.asof) != day_open || observation.all_positions != 0 || observation.all_orders != 0 || observation.margin != 0 || observation.credit != 0 || observation.account_profit != 0 || observation.balance != observation.equity || !MathIsValidNumber(observation.balance) || observation.balance <= 0 || ArraySize(observation.deals) != 0) return(DPCCPhysicalRefuse("initialize.requires_flat_unchanged_day", reason)); ledger.login = policy.login; ledger.server = policy.server; ledger.currency = policy.currency; ledger.symbol = policy.symbol; ledger.magic = policy.magic; ledger.deployment_id = policy.deployment_id; ledger.checkpoint_id = checkpoint_id; ledger.previous_mark = observation.asof; ledger.previous_balance = observation.balance; ledger.previous_equity = observation.equity; ledger.snapshot_time = observation.asof; ledger.snapshot_equity = observation.equity; ledger.day_open = day_open; ledger.day_start_equity = observation.equity; ledger.running_peak_equity = observation.equity; ledger.peak_initialized = true; ledger.terminal = false; ledger.day_reset_observed = false; ledger.deal_cursor_msc = (long)observation.asof * 1000; ledger.deal_cursor_ticket = 0; ledger.realized_booked = 0; ledger.paid_costs_booked = 0; ledger.booked_costs_confirmed = true; ledger.booked_costs_evidence = no_unbooked_cost_evidence; return(true); } //+------------------------------------------------------------------+ //| Derive CTX-02 Account24 and original symbol-scoped ACCOUNT13 | //+------------------------------------------------------------------+ bool DPCCPhysicalBuild(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, const SDPCCPhysicalObservation &source, SDPCCPhysicalAccount &output, string &reason, CDPCCMarketClock *clock = NULL) { ArrayFree(output.positions); ZeroMemory(output); SDPCCPhysicalAccount candidate; ZeroMemory(candidate); reason = ""; if(clock && CheckPointer(clock) == POINTER_INVALID) return(DPCCPhysicalRefuse("invalid_clock_owner", reason)); if(!DPCCPhysicalPolicyCheck(policy, reason)) return(false); if(ledger.login != policy.login || ledger.server != policy.server || ledger.currency != policy.currency || ledger.symbol != policy.symbol || ledger.magic != policy.magic || ledger.deployment_id != policy.deployment_id || ledger.checkpoint_id == "" || source.login != policy.login || source.server != policy.server || source.currency != policy.currency || confirmation.checkpoint_id != ledger.checkpoint_id || confirmation.asof != source.asof) return(DPCCPhysicalRefuse("build.identity", reason)); const double observed[] = {source.balance, source.equity, source.margin, source.free_margin, ledger.previous_balance, ledger.previous_equity, ledger.snapshot_equity, ledger.day_start_equity, ledger.running_peak_equity, ledger.realized_booked, ledger.paid_costs_booked }; for(int i = 0; i < ArraySize(observed); i++) if(!MathIsValidNumber(observed[i])) return(DPCCPhysicalRefuse("build.finite", reason)); if(source.asof <= 0 || ledger.previous_mark <= 0 || ledger.previous_mark > source.asof || ledger.snapshot_time <= 0 || ledger.snapshot_time > ledger.previous_mark || ledger.deal_cursor_msc < (long)ledger.snapshot_time * 1000 || ledger.deal_cursor_msc / 1000 > (long)source.asof || source.balance <= 0 || source.equity <= 0 || source.margin < 0 || ledger.previous_balance <= 0 || ledger.previous_equity <= 0 || ledger.snapshot_equity <= 0 || ledger.day_start_equity <= 0 || !ledger.peak_initialized || ledger.running_peak_equity < ledger.previous_equity || ledger.day_open != DPCCPhysicalDayOpen(source.asof) || (ledger.day_open > ledger.previous_mark && !ledger.day_reset_observed) || !ledger.booked_costs_confirmed || ledger.booked_costs_evidence == "" || ledger.paid_costs_booked < 0) return(DPCCPhysicalRefuse("build.persistent_snapshot_day_peak_or_cost_state", reason)); if(!confirmation.accrued_cost_known || confirmation.accrued_cost_evidence == "" || !MathIsValidNumber(confirmation.accrued_unbooked_cost) || confirmation.accrued_unbooked_cost < 0) return(DPCCPhysicalRefuse("build.unknown_accrued_unbooked_cost", reason)); long quote_age = (long)(source.asof - source.tick.time); long mark_age = (long)(source.asof - ledger.previous_mark); if(clock) { long snapshot_utc = 0, cursor_utc = 0; if(!clock.Convert((long)source.asof, candidate.asof_utc, reason) || !clock.Convert((long)ledger.previous_mark, candidate.previous_mark_utc, reason) || !clock.Convert((long)ledger.snapshot_time, snapshot_utc, reason) || !clock.ConvertMilliseconds(ledger.deal_cursor_msc, cursor_utc, reason) || !clock.Convert((long)ledger.day_open, candidate.day_open_utc, reason) || !clock.ConvertMilliseconds(source.tick.time_msc, candidate.quote_utc_msc, reason) || !clock.Elapsed((long)source.tick.time, (long)source.asof, quote_age, reason) || !clock.Elapsed((long)ledger.previous_mark, (long)source.asof, mark_age, reason)) return(false); if(snapshot_utc > candidate.previous_mark_utc || candidate.day_open_utc > candidate.asof_utc || cursor_utc / 1000 > candidate.asof_utc) return(DPCCPhysicalRefuse("build.UTC_snapshot_or_day_causality", reason)); candidate.clock_bound = true; } if(source.tick.time <= 0 || source.tick.time > source.asof || (ulong)quote_age > policy.max_quote_age_seconds || source.tick.time_msc / 1000 != (long)source.tick.time || !MathIsValidNumber(source.tick.bid) || !MathIsValidNumber(source.tick.ask) || source.tick.bid <= 0 || source.tick.ask < source.tick.bid) return(DPCCPhysicalRefuse("build.quote_age_or_geometry", reason)); if(source.margin_mode != ACCOUNT_MARGIN_MODE_RETAIL_NETTING && source.margin_mode != ACCOUNT_MARGIN_MODE_RETAIL_HEDGING && source.margin_mode != ACCOUNT_MARGIN_MODE_EXCHANGE) return(DPCCPhysicalRefuse("build.account_margin_mode", reason)); if(source.all_positions > policy.max_positions || (uint)ArraySize(source.positions) > source.all_positions || (uint)ArraySize(source.deals) > policy.max_history_deals) return(DPCCPhysicalRefuse("build.observation_budget", reason)); //--- A single signed state admits only identically protected positions in one direction. double weighted_price = 0; double total_volume = 0; long direction = -1; for(int i = 0; i < ArraySize(source.positions); i++) { const SDPCCPhysicalPosition position = source.positions[i]; const double values[] = {position.volume, position.open_price, position.sl, position.tp, position.profit, position.swap }; for(int j = 0; j < ArraySize(values); j++) if(!MathIsValidNumber(values[j])) return(DPCCPhysicalRefuse("build.position_finite", reason)); if(!position.history_owned || position.magic != policy.magic || position.identifier == 0 || position.ticket == 0 || position.volume <= 0 || position.open_price <= 0 || position.sl < 0 || position.tp < 0 || position.opened_msc <= 0 || position.opened_msc / 1000 > (long)source.asof || position.updated_msc / 1000 > (long)source.asof || (position.type != POSITION_TYPE_BUY && position.type != POSITION_TYPE_SELL)) return(DPCCPhysicalRefuse("build.position_source_or_ownership", reason)); if(i > 0 && (position.type != direction || position.sl != source.positions[0].sl || position.tp != source.positions[0].tp)) return(DPCCPhysicalRefuse("build.hedging_cannot_collapse_direction_or_protection", reason)); if(i > 0 && source.margin_mode != ACCOUNT_MARGIN_MODE_RETAIL_HEDGING) return(DPCCPhysicalRefuse("build.netting_multiple_positions", reason)); for(int j = 0; j < i; j++) if(position.ticket == source.positions[j].ticket || position.identifier == source.positions[j].identifier) return(DPCCPhysicalRefuse("build.duplicate_position", reason)); direction = position.type; weighted_price += position.open_price * position.volume; total_volume += position.volume; if(position.type == POSITION_TYPE_BUY) { candidate.buy_lots += position.volume; candidate.buy_profit += position.profit; } else { candidate.sell_lots += position.volume; candidate.sell_profit += position.profit; } long position_age = (long)source.asof - position.opened_msc / 1000; if(clock) { long opened_utc = 0, updated_utc = 0; if(!clock.ConvertMilliseconds(position.opened_msc, opened_utc, reason) || !clock.ConvertMilliseconds(position.updated_msc, updated_utc, reason) || !clock.Elapsed(position.opened_msc / 1000, (long)source.asof, position_age, reason)) return(false); if(updated_utc < opened_utc || updated_utc / 1000 > candidate.asof_utc) return(DPCCPhysicalRefuse("build.UTC_position_causality", reason)); } candidate.position_discount += position_age / 36000.0 * MathAbs(position.profit); } double owned_margin = 0; double other_margin = source.margin; if(total_volume > 0) { if(ArraySize(confirmation.positions) != ArraySize(source.positions)) return(DPCCPhysicalRefuse("build.confirmation_position_count", reason)); for(int i = 0; i < ArraySize(source.positions); i++) { const SDPCCPhysicalConfirmedPosition bound = confirmation.positions[i]; const SDPCCPhysicalPosition actual = source.positions[i]; if(bound.ticket != actual.ticket || bound.identifier != actual.identifier || bound.updated_msc != actual.updated_msc || bound.volume != actual.volume || bound.open_price != actual.open_price || bound.sl != actual.sl || bound.tp != actual.tp || bound.swap != actual.swap) return(DPCCPhysicalRefuse("build.stale_confirmation_position_identity", reason)); } if(!confirmation.margin_allocation_known || confirmation.margin_evidence == "" || !MathIsValidNumber(confirmation.owned_margin) || !MathIsValidNumber(confirmation.other_margin) || confirmation.owned_margin < 0 || confirmation.other_margin < 0) return(DPCCPhysicalRefuse("build.unknown_open_position_margin_allocation", reason)); owned_margin = confirmation.owned_margin; other_margin = confirmation.other_margin; if(MathAbs(owned_margin + other_margin - source.margin) > policy.money_atol + policy.money_rtol * MathAbs(source.margin)) return(DPCCPhysicalRefuse("build.margin_total_binding", reason)); } if(total_volume == 0 && confirmation.accrued_unbooked_cost != 0) return(DPCCPhysicalRefuse("build.flat_unbooked_cost_not_representable", reason)); if(ledger.terminal && total_volume > 0) return(DPCCPhysicalRefuse("build.terminal_with_open_position", reason)); const double free_expected = source.equity - owned_margin - other_margin; if(MathAbs(source.free_margin - free_expected) > policy.money_atol + policy.money_rtol * MathMax(MathAbs(source.free_margin), MathAbs(free_expected))) return(DPCCPhysicalRefuse("build.free_margin_reconciliation", reason)); //--- Booked money uses an exact (millisecond,ticket) cursor and refuses foreign cash activity. SDPCCPhysicalLedger next = ledger; long last_msc = 0; long last_utc_msc = 0; ulong last_ticket = 0; double balance_delta = 0; for(int i = 0; i < ArraySize(source.deals); i++) { const SDPCCPhysicalDeal deal = source.deals[i]; const double values[] = {deal.volume, deal.profit, deal.commission, deal.fee, deal.swap}; for(int j = 0; j < ArraySize(values); j++) if(!MathIsValidNumber(values[j])) return(DPCCPhysicalRefuse("build.deal_finite", reason)); if(deal.ticket == 0 || deal.time_msc <= 0 || deal.time_msc / 1000 > (long)source.asof || deal.time_msc < last_msc || (deal.time_msc == last_msc && deal.ticket <= last_ticket)) return(DPCCPhysicalRefuse("build.deal_order_or_time", reason)); if(clock) { long deal_utc = 0; if(!clock.ConvertMilliseconds(deal.time_msc, deal_utc, reason)) return(false); if(deal_utc < last_utc_msc || deal_utc / 1000 > candidate.asof_utc) return(DPCCPhysicalRefuse("build.UTC_deal_order_or_causality", reason)); last_utc_msc = deal_utc; } last_msc = deal.time_msc; last_ticket = deal.ticket; if(!DPCCPhysicalAfterCursor(deal, ledger)) continue; if((deal.type != DEAL_TYPE_BUY && deal.type != DEAL_TYPE_SELL) || deal.symbol != policy.symbol || deal.magic != policy.magic || !deal.history_owned || deal.position_id == 0 || deal.volume <= 0 || deal.commission > 0 || deal.fee > 0 || (deal.entry != DEAL_ENTRY_IN && deal.entry != DEAL_ENTRY_OUT && deal.entry != DEAL_ENTRY_INOUT && deal.entry != DEAL_ENTRY_OUT_BY)) return(DPCCPhysicalRefuse("build.unsupported_or_foreign_cash_activity", reason)); const double gross = deal.profit + MathMax(deal.swap, 0); const double paid = -deal.commission - deal.fee + MathMax(-deal.swap, 0); next.realized_booked += gross; next.paid_costs_booked += paid; balance_delta += gross - paid; next.deal_cursor_msc = deal.time_msc; next.deal_cursor_ticket = deal.ticket; } if(MathAbs((source.balance - ledger.previous_balance) - balance_delta) > policy.money_atol + policy.money_rtol * MathAbs(source.balance)) return(DPCCPhysicalRefuse("build.booked_history_balance_reconciliation", reason)); candidate.account[0] = source.balance; candidate.account[1] = source.equity; candidate.account[2] = candidate.buy_lots - candidate.sell_lots; candidate.account[3] = total_volume > 0 ? weighted_price / total_volume : 0; candidate.account[4] = total_volume > 0 ? source.positions[0].sl : 0; candidate.account[5] = total_volume > 0 ? source.positions[0].tp : 0; candidate.account[6] = confirmation.accrued_unbooked_cost; candidate.account[7] = ledger.day_start_equity; candidate.account[8] = MathMax(ledger.running_peak_equity, source.equity); candidate.account[9] = ledger.snapshot_equity; candidate.account[10] = owned_margin; candidate.account[11] = other_margin; candidate.account[12] = source.free_margin; candidate.account[13] = next.realized_booked; candidate.account[14] = next.paid_costs_booked; candidate.account[15] = source.tick.bid; candidate.account[16] = source.tick.ask; candidate.account[17] = candidate.account[4] > 0 ? 1 : 0; candidate.account[18] = candidate.account[5] > 0 ? 1 : 0; candidate.account[19] = ledger.terminal ? 1 : 0; candidate.account[20] = ledger.peak_initialized ? 1 : 0; candidate.account[21] = (double)mark_age; candidate.account[22] = ledger.day_reset_observed ? 1 : 0; candidate.account[23] = 0; float account13[]; string account13_reason = ""; if(!DPCCBuildAccount13Features(source.balance, ledger.previous_balance, source.equity, ledger.previous_equity, candidate.buy_lots, candidate.sell_lots, candidate.buy_profit, candidate.sell_profit, candidate.position_discount, (long)source.asof, account13, account13_reason)) { ZeroMemory(candidate); return(DPCCPhysicalRefuse("build.account13_float_range", reason)); } if(ArraySize(account13) != 13) { ZeroMemory(candidate); return(DPCCPhysicalRefuse("build.account13_float_range", reason)); } for(int i = 0; i < 13; i++) candidate.features[i] = account13[i]; for(int i = 0; i < 24; i++) { if(!MathIsValidNumber(candidate.account[i]) || !MathIsValidNumber((double)(float)candidate.account[i])) { ZeroMemory(candidate); return(DPCCPhysicalRefuse("build.account_float_range", reason)); } candidate.present[i] = true; } for(int i = 0; i < 13; i++) if(!MathIsValidNumber((double)candidate.features[i])) { ZeroMemory(candidate); return(DPCCPhysicalRefuse("build.account13_float_range", reason)); } next.previous_mark = source.asof; next.previous_balance = source.balance; next.previous_equity = source.equity; next.running_peak_equity = candidate.account[8]; next.day_reset_observed = false; candidate.next_ledger = next; candidate.asof = source.asof; candidate.previous_mark = ledger.previous_mark; candidate.quote_msc = source.tick.time_msc; candidate.all_positions = source.all_positions; candidate.all_orders = source.all_orders; if(ArrayCopy(candidate.positions, source.positions) != ArraySize(source.positions)) { ArrayFree(candidate.positions); ZeroMemory(candidate); return(DPCCPhysicalRefuse("build.output_position_copy", reason)); } candidate.login = policy.login; candidate.server = policy.server; candidate.currency = policy.currency; candidate.symbol = policy.symbol; candidate.deployment_id = policy.deployment_id; candidate.checkpoint_id = ledger.checkpoint_id; candidate.previous_balance = ledger.previous_balance; candidate.previous_equity = ledger.previous_equity; candidate.original_symbol_scope_verified = true; candidate.ownership_evidence = "NATIVE_POSITION_ID_HISTORY_V1/" + policy.deployment_id; candidate.complete = true; output = candidate; return(true); } //+------------------------------------------------------------------+ //| Capture actual terminal data without committing caller state | //+------------------------------------------------------------------+ bool DPCCPhysicalCapture(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, SDPCCPhysicalAccount &output, string &reason, CDPCCMarketClock *clock = NULL) { ArrayFree(output.positions); ZeroMemory(output); SDPCCPhysicalObservation source; if(!DPCCPhysicalObserve(policy, (datetime)(ledger.deal_cursor_msc / 1000), source, reason, clock)) return(false); if(!DPCCPhysicalBuild(policy, ledger, confirmation, source, output, reason, clock)) return(false); if(clock) { SDPCCMarketClockCapture current; if(!clock.ReadCapture(current, reason) || current.broker_second != (long)source.asof || current.broker_tick_msc != source.tick.time_msc || current.utc_second != output.asof_utc) { ArrayFree(output.positions); ZeroMemory(output); return(DPCCPhysicalRefuse("capture.clock_boundary_changed", reason)); } } output.native_observed = true; return(true); } #endif //--- Shared causal H1 chart context for Action6 Test and StudyOnline. #ifndef DPCC_ACTION6_CHART_CONTEXT_OWNER_MQH #define DPCC_ACTION6_CHART_CONTEXT_OWNER_MQH struct SDPCCAction6ChartSession { int day; long from_second, to_second; }; struct SDPCCAction6ChartLedger { bool seeded, peak_initialized; long previous_decision, day_open, deal_cursor_msc; ulong deal_cursor_ticket; int direction; long first_open_msc; double previous_balance, previous_equity, day_start_equity, running_peak_equity; double realized_booked, paid_costs_booked; }; //+------------------------------------------------------------------+ //| Declaration CDPCCAction6ChartContextOwner | //+------------------------------------------------------------------+ class CDPCCAction6ChartContextOwner { private: bool m_ready, m_failed, m_sessions_ready; string m_symbol, m_currency, m_profile_sha; SDPCCConfig m_config; SDPCCExecutionProfileSnapshot m_execution; SDPCCHistoricalFinancialProfile m_profile; SDPCCAction6ChartLedger m_ledger; SDPCCAction6ChartSession m_quote_sessions[], m_trade_sessions[]; COpenCLMy *m_context; CBufferFloat m_snapshot_work, m_context_work, m_flags_work, m_status_work; bool Refuse(string &reason, const string detail) { reason = "BLOCKED:action6.chart_context." + detail; return(false); } bool ReadSessionSeries(const string symbol, const bool quote, SDPCCAction6ChartSession &out[], string &reason) { for(int day = 0; day < 7; day++) { for(uint index = 0; index < 64; index++) { datetime from = 0, to = 0; ResetLastError(); const bool ok = (quote ? SymbolInfoSessionQuote(symbol, (ENUM_DAY_OF_WEEK)day, index, from, to) : SymbolInfoSessionTrade(symbol, (ENUM_DAY_OF_WEEK)day, index, from, to)); if(!ok) break; MqlDateTime first, last; if(!TimeToStruct(from, first) || !TimeToStruct(to, last)) return(Refuse(reason, "session_time_decode")); const long start = (long)first.hour * 3600 + (long)first.min * 60 + first.sec; long finish = (long)last.hour * 3600 + (long)last.min * 60 + last.sec; const long from_day = (long)from / 86400; const long to_day = (long)to / 86400; const long day_delta = to_day - from_day; if(start < 0 || start >= 86400 || day_delta < 0 || day_delta > 1) return(Refuse(reason, "session_day_offset")); if(day_delta > 0) finish += day_delta * 86400; else if(finish < start) finish += 86400; if(finish <= start || finish - start > 86400) return(Refuse(reason, "session_time_ambiguous")); const int count = ArraySize(out); if(ArrayResize(out, count + 1) != count + 1) return(Refuse(reason, "session_capacity")); out[count].day = day; out[count].from_second = start; out[count].to_second = finish; } } return(true); } bool SessionOverlap(const long open, const long close, string &reason) { MqlDateTime civil; if(open <= 0 || close <= open || !TimeToStruct((datetime)open, civil)) return(Refuse(reason, "session_candidate_time")); const long day_open = open - (long)civil.hour * 3600 - (long)civil.min * 60 - civil.sec; for(int day_offset = -1; day_offset <= 0; day_offset++) { const long base = day_open + (long)day_offset * 86400; if(base <= 0) continue; MqlDateTime day_parts; if(!TimeToStruct((datetime)base, day_parts)) return(Refuse(reason, "session_weekday")); for(int q = 0; q < ArraySize(m_quote_sessions); q++) { if(m_quote_sessions[q].day != day_parts.day_of_week) continue; const long q_from = base + m_quote_sessions[q].from_second; const long q_to = base + m_quote_sessions[q].to_second; for(int t = 0; t < ArraySize(m_trade_sessions); t++) { if(m_trade_sessions[t].day != day_parts.day_of_week) continue; const long t_from = base + m_trade_sessions[t].from_second; const long t_to = base + m_trade_sessions[t].to_second; const long overlap_from = MathMax(MathMax(q_from, t_from), open); const long overlap_to = MathMin(MathMin(q_to, t_to), close); if(overlap_from < overlap_to) return(true); } } } return(false); } bool PrepareSessions(string &reason) { ArrayFree(m_quote_sessions); ArrayFree(m_trade_sessions); if(!ReadSessionSeries(m_symbol, true, m_quote_sessions, reason)) return(false); if(!ReadSessionSeries(m_symbol, false, m_trade_sessions, reason)) return(false); const int quote_count = ArraySize(m_quote_sessions); const int trade_count = ArraySize(m_trade_sessions); if(quote_count == 0 && trade_count > 0) { if(ArrayResize(m_quote_sessions, trade_count) != trade_count) return(Refuse(reason, "quote_session_fallback_capacity")); for(int i = 0; i < trade_count; i++) m_quote_sessions[i] = m_trade_sessions[i]; } if(ArraySize(m_quote_sessions) == 0 || ArraySize(m_trade_sessions) == 0) { reason = StringFormat( "BLOCKED:action6.chart_context.terminal_session_schedule_unavailable:quote=%d:trade=%d", ArraySize(m_quote_sessions), ArraySize(m_trade_sessions)); return(false); } m_sessions_ready = true; return(true); } bool PrepareProfile(const SDPCCRuntimeConfig &runtime, const float &limits24[], CBufferFloat *market_ohlc, string &reason) { if(MQLInfoInteger(MQL_TESTER) == 0) return(Refuse(reason, "tester_flat_start_required")); if(_Period != PERIOD_H1 || runtime.financial.shape.H == 0 || ArraySize(limits24) != 24 || !DPCCValidateFixedConfig(runtime.financial.shape, runtime.financial)) return(Refuse(reason, "profile_or_H1_chart")); if(CheckPointer(market_ohlc) == POINTER_INVALID) return(Refuse(reason, "market_buffer_missing")); m_symbol = _Symbol; m_config = runtime.financial; SDPCCLayout layout; if(!DPCCBuildRolloutLayout(m_config.shape, layout) || layout.total > (ulong)INT_MAX || layout.offsets[7] > layout.total) return(Refuse(reason, "native_layout")); m_context = market_ohlc.GetOpenCL(); if(CheckPointer(m_context) == POINTER_INVALID || market_ohlc.Total() != (int)layout.lengths[0] || !NNQValid(m_context, market_ohlc, layout.lengths[0])) return(Refuse(reason, "market_buffer_context_or_shape")); string payload = "", digest = ""; if(!DPCCCaptureExecutionProfile(m_symbol, PERIOD_H1, "", 0, m_execution, payload, digest, reason, true)) return(false); if(!DPCCPrepareHistoricalFinancialProfile(m_config, limits24, m_execution, payload, true, false, m_profile, reason, true, true)) return(false); if(!PrepareSessions(reason)) return(false); if(!m_snapshot_work.BufferInit((uint)(layout.offsets[7] - layout.lengths[0]), 0.0f) || !m_context_work.BufferInit((uint)layout.total, 0.0f) || !m_flags_work.BufferInit((uint)layout.total, 0.0f) || !m_status_work.BufferInit(1, 0.0f) || !m_snapshot_work.BufferCreate(m_context) || !m_context_work.BufferCreate(m_context) || !m_flags_work.BufferCreate(m_context) || !m_status_work.BufferCreate(m_context)) return(Refuse(reason, "workspace_create")); m_currency = m_execution.account_currency; m_profile_sha = m_profile.sha256; m_ready = true; return(true); } bool BuildSchedule(const long decision, const int horizon, const int events, long &opens[], long &closes[], float &counts[], float &calendar[], string &reason) { if(horizon <= 0 || events <= 0 || ArrayResize(opens, horizon) != horizon || ArrayResize(closes, horizon) != horizon || ArrayResize(counts, horizon) != horizon || horizon > INT_MAX / (events * 6) || ArrayResize(calendar, horizon * events * 6) != horizon * events * 6) return(Refuse(reason, "schedule_capacity")); ArrayInitialize(counts, 0.0f); ArrayInitialize(calendar, 0.0f); long candidate = decision; int searched = 0; for(int h = 0; h < horizon; h++) { bool found = false; while(searched <= DPCC_HISTORY_CALENDAR_MAX_SPAN_SECONDS / 3600) { if(candidate > LONG_MAX - 3600) return(Refuse(reason, "schedule_time_overflow")); if(SessionOverlap(candidate, candidate + 3600, reason)) {found = true; break;} if(StringFind(reason, "BLOCKED:") == 0) return(false); candidate += 3600; searched++; } if(!found) return(Refuse(reason, "H1_session_horizon_unavailable")); opens[h] = candidate; closes[h] = candidate + 3600; if(h == 0 && opens[h] != decision) return(Refuse(reason, "decision_session_binding")); const long previous_close = (h == 0 ? decision : closes[h - 1]); long midnight = 0; if(!DPCCServerCalendarNextMidnight(previous_close, midnight, reason)) return(false); while(midnight <= closes[h]) { MqlDateTime previous_day; if(!TimeToStruct((datetime)(midnight - 1), previous_day)) return(Refuse(reason, "calendar_weekday")); const double factor = m_profile.swap_factors[previous_day.day_of_week]; if(factor < 0 || !MathIsValidNumber(factor)) return(Refuse(reason, "swap_factor_invalid")); const bool swap = (factor > 0 && (m_profile.swap_cash_buy != 0 || m_profile.swap_cash_sell != 0)); if(swap && m_profile.swap_profit_currency) return(Refuse(reason, "swap_profit_currency_conversion_not_in_fixed_ABI")); if(swap && !DPCCAppendHistoricalCalendarEvent(h, events, 2, midnight - 15, previous_close, opens[h], closes[h], m_profile.swap_cash_buy, m_profile.swap_cash_sell, factor, counts, calendar, reason)) return(false); if(!DPCCAppendHistoricalCalendarEvent(h, events, 1, midnight, previous_close, opens[h], closes[h], 0, 0, 0, counts, calendar, reason)) return(false); if(midnight == closes[h]) break; long next = 0; if(!DPCCServerCalendarNextMidnight(midnight, next, reason)) return(false); midnight = next; } candidate += 3600; searched++; } return(true); } bool CapturePositions(const long asof, double &buy_lots, double &sell_lots, double &buy_profit, double &sell_profit, double &weighted_price, double &position_swap, double &discount, long &first_open_msc, double &sl, double &tp, string &reason) { buy_lots = 0; sell_lots = 0; buy_profit = 0; sell_profit = 0; weighted_price = 0; position_swap = 0; discount = 0; first_open_msc = 0; sl = 0; tp = 0; if(OrdersTotal() != 0) return(Refuse(reason, "pending_orders_not_representable")); const int total = PositionsTotal(); int direction = 0; for(int i = 0; i < total; i++) { const string symbol = PositionGetSymbol(i); if(symbol == "") return(Refuse(reason, "position_selection")); if(symbol != m_symbol) return(Refuse(reason, "foreign_symbol_exposure")); long type = 0, opened = 0, identifier = 0; double volume = 0, price = 0, current_sl = 0, current_tp = 0, profit = 0, swap = 0; if(!PositionGetInteger(POSITION_TYPE, type) || !PositionGetInteger(POSITION_TIME_MSC, opened) || !PositionGetInteger(POSITION_IDENTIFIER, identifier) || !PositionGetDouble(POSITION_VOLUME, volume) || !PositionGetDouble(POSITION_PRICE_OPEN, price) || !PositionGetDouble(POSITION_SL, current_sl) || !PositionGetDouble(POSITION_TP, current_tp) || !PositionGetDouble(POSITION_PROFIT, profit) || !PositionGetDouble(POSITION_SWAP, swap) || identifier <= 0 || opened <= 0 || opened / 1000 > asof || volume <= 0 || price <= 0 || !MathIsValidNumber(volume) || !MathIsValidNumber(price) || !MathIsValidNumber(profit) || !MathIsValidNumber(swap) || !MathIsValidNumber(current_sl) || !MathIsValidNumber(current_tp)) return(Refuse(reason, "position_snapshot_invalid")); if(type != POSITION_TYPE_BUY && type != POSITION_TYPE_SELL) return(Refuse(reason, "position_type")); const int side = (type == POSITION_TYPE_BUY ? 1 : -1); if(direction != 0 && side != direction) return(Refuse(reason, "mixed_direction_exposure")); if(direction == 0) { direction = side; sl = current_sl; tp = current_tp; } else if(sl != current_sl || tp != current_tp) return(Refuse(reason, "mixed_protection_not_in_account24")); if(first_open_msc == 0 || opened < first_open_msc) first_open_msc = opened; weighted_price += price * volume; position_swap += swap; discount += (double)(asof - opened / 1000) / 36000.0 * MathAbs(profit); if(side > 0) { buy_lots += volume; buy_profit += profit; } else { sell_lots += volume; sell_profit += profit; } } return(true); } bool AdvanceDeals(const long asof_msc, const double current_balance, SDPCCAction6ChartLedger &next, string &reason) { if(!next.seeded) return(true); const datetime from = (datetime)(next.deal_cursor_msc / 1000); const datetime asof = (datetime)(asof_msc / 1000); if(asof_msc <= 0 || from <= 0 || from > asof || !HistorySelect(from, asof)) return(Refuse(reason, "incremental_history_select")); double balance_delta = 0; long last_msc = 0; ulong last_ticket = 0; const int total = HistoryDealsTotal(); for(int i = 0; i < total; i++) { const ulong ticket = HistoryDealGetTicket(i); SDPCCPhysicalDeal deal; if(ticket == 0 || !DPCCPhysicalReadDeal(ticket, deal, reason)) return(false); if(deal.time_msc > asof_msc || deal.time_msc < last_msc || (deal.time_msc == last_msc && deal.ticket <= last_ticket)) return(Refuse(reason, "incremental_history_order")); last_msc = deal.time_msc; last_ticket = deal.ticket; if(deal.time_msc < m_ledger.deal_cursor_msc || (deal.time_msc == m_ledger.deal_cursor_msc && deal.ticket <= m_ledger.deal_cursor_ticket)) continue; if(deal.symbol != m_symbol || (deal.type != DEAL_TYPE_BUY && deal.type != DEAL_TYPE_SELL) || deal.volume <= 0 || (deal.entry != DEAL_ENTRY_IN && deal.entry != DEAL_ENTRY_OUT && deal.entry != DEAL_ENTRY_INOUT && deal.entry != DEAL_ENTRY_OUT_BY)) return(Refuse(reason, "unmodeled_or_foreign_deal")); if(deal.commission != 0 || deal.fee != 0) return(Refuse(reason, "commission_deal_not_mapped_to_native_execution_slots")); const double gross = deal.profit + MathMax(deal.swap, 0); const double paid = MathMax(-deal.swap, 0); if(!MathIsValidNumber(gross) || !MathIsValidNumber(paid)) return(Refuse(reason, "deal_financial_nonfinite")); next.realized_booked += gross; next.paid_costs_booked += paid; balance_delta += gross - paid; next.deal_cursor_msc = deal.time_msc; next.deal_cursor_ticket = deal.ticket; } const double unit = MathPow(10.0, -(double)m_execution.account_currency_digits); const double tolerance = unit * 0.51 + 1.0e-9 * MathMax(MathAbs(current_balance), 1.0); if(MathAbs((current_balance - m_ledger.previous_balance) - balance_delta) > tolerance) return(Refuse(reason, "incremental_history_balance_reconciliation")); //--- Advance an exact millisecond watermark so an idle strategy never rescans //--- its full post-start history. Same-millisecond later tickets remain visible. const long watermark = asof_msc - 1; if(watermark > next.deal_cursor_msc) { next.deal_cursor_msc = watermark; next.deal_cursor_ticket = 0; } return(true); } public: CDPCCAction6ChartContextOwner(void) { m_ready = false; m_failed = false; m_sessions_ready = false; m_context = NULL; m_symbol = ""; m_currency = ""; m_profile_sha = ""; ZeroMemory(m_config); ZeroMemory(m_execution); ZeroMemory(m_profile); ZeroMemory(m_ledger); } void Reset(void) { if(m_snapshot_work.GetIndex() >= 0) m_snapshot_work.BufferFree(); if(m_context_work.GetIndex() >= 0) m_context_work.BufferFree(); if(m_flags_work.GetIndex() >= 0) m_flags_work.BufferFree(); if(m_status_work.GetIndex() >= 0) m_status_work.BufferFree(); ArrayFree(m_quote_sessions); ArrayFree(m_trade_sessions); m_ready = false; m_failed = false; m_sessions_ready = false; m_context = NULL; m_symbol = ""; m_currency = ""; m_profile_sha = ""; ZeroMemory(m_config); ZeroMemory(m_execution); ZeroMemory(m_profile); ZeroMemory(m_ledger); } bool Build(const SDPCCRuntimeConfig &runtime, const float &limits24[], CBufferFloat *market_ohlc, const long decision_server_time, CBufferFloat *&context_out, float &account13[], string &reason) { context_out = NULL; ArrayFree(account13); reason = ""; if(m_failed) return(Refuse(reason, "owner_poisoned")); if(MQLInfoInteger(MQL_TESTER) == 0 || _Period != PERIOD_H1 || _Symbol == "" || decision_server_time <= 0 || ArraySize(limits24) != 24 || runtime.financial.shape.H == 0) return(Refuse(reason, "tester_H1_input_contract")); if(!m_ready && !PrepareProfile(runtime, limits24, market_ohlc, reason)) return(false); if(_Symbol != m_symbol || CheckPointer(market_ohlc) == POINTER_INVALID || market_ohlc.GetOpenCL() != m_context || runtime.financial.price_basis != m_config.price_basis || runtime.financial.quote_adapter != m_config.quote_adapter || runtime.financial.shape.K != m_config.shape.K || runtime.financial.shape.H != m_config.shape.H || runtime.financial.shape.E != m_config.shape.E || market_ohlc.Total() <= 0) return(Refuse(reason, "chart_or_model_profile_changed")); const long chart_open = (long)iTime(m_symbol, PERIOD_H1, 0); MqlTick tick; long chart_mode = 0; double tick_size = 0; if(chart_open != decision_server_time || !SymbolInfoTick(m_symbol, tick) || tick.time_msc <= 0 || tick.time_msc / 1000 != tick.time || tick.time < decision_server_time || tick.time >= decision_server_time + 3600 || !SymbolInfoInteger(m_symbol, SYMBOL_CHART_MODE, chart_mode) || !SymbolInfoDouble(m_symbol, SYMBOL_TRADE_TICK_SIZE, tick_size) || tick_size <= 0 || !MathIsValidNumber(tick.bid) || !MathIsValidNumber(tick.ask) || tick.bid <= 0 || tick.ask < tick.bid) return(Refuse(reason, "decision_tick_or_chart_binding")); const double price = (m_config.price_basis == 0 ? tick.bid : tick.last); if((chart_mode != SYMBOL_CHART_MODE_BID && chart_mode != SYMBOL_CHART_MODE_LAST) || (m_config.price_basis == 0 && chart_mode != SYMBOL_CHART_MODE_BID) || (m_config.price_basis == 1 && chart_mode != SYMBOL_CHART_MODE_LAST) || !MathIsValidNumber(price) || price <= 0 || m_config.quote_adapter != m_config.price_basis || limits24[13] != 0.0f) return(Refuse(reason, "chart_price_or_zero_slippage_contract")); double buy_lots = 0, sell_lots = 0, buy_profit = 0, sell_profit = 0, weighted_price = 0; double position_swap = 0, discount = 0, sl = 0, tp = 0; long first_open_msc = 0; if(!CapturePositions(tick.time, buy_lots, sell_lots, buy_profit, sell_profit, weighted_price, position_swap, discount, first_open_msc, sl, tp, reason)) return(false); const double balance = AccountInfoDouble(ACCOUNT_BALANCE), equity = AccountInfoDouble(ACCOUNT_EQUITY); const double margin = AccountInfoDouble(ACCOUNT_MARGIN), free_margin = AccountInfoDouble(ACCOUNT_MARGIN_FREE); const double account_values[] = {balance, equity, margin, free_margin, position_swap, discount, weighted_price}; for(int i = 0; i < ArraySize(account_values); i++) if(!MathIsValidNumber(account_values[i])) return(Refuse(reason, "account_snapshot_nonfinite")); if(balance <= 0) return(Refuse(reason, StringFormat("account_balance_nonpositive value=%.8f", balance))); if(equity <= 0) return(Refuse(reason, StringFormat("account_equity_nonpositive value=%.8f", equity))); if(margin < 0) return(Refuse(reason, StringFormat("account_margin_negative value=%.8f", margin))); if((buy_lots + sell_lots) == 0 && margin > 0) return(Refuse(reason, "margin_without_modeled_position")); SDPCCAction6ChartLedger next = m_ledger; if(!next.seeded) { if(PositionsTotal() != 0 || OrdersTotal() != 0) return(Refuse(reason, "initial_tester_state_not_flat")); next.seeded = true; next.previous_decision = decision_server_time; next.day_open = (long)DPCCPhysicalDayOpen((datetime)tick.time); next.previous_balance = balance; next.previous_equity = equity; next.day_start_equity = equity; next.running_peak_equity = equity; next.peak_initialized = true; next.deal_cursor_msc = tick.time_msc - 1; next.deal_cursor_ticket = 0; } else { if(decision_server_time <= next.previous_decision || next.day_open <= 0) return(Refuse(reason, "decision_order_or_position_ledger")); if(!AdvanceDeals(tick.time_msc, balance, next, reason)) return(false); const long current_day = (long)DPCCPhysicalDayOpen((datetime)tick.time); if(current_day > next.day_open) { next.day_open = current_day; next.day_start_equity = equity; } next.running_peak_equity = MathMax(next.running_peak_equity, equity); } const int direction = (buy_lots > 0 ? 1 : (sell_lots > 0 ? -1 : 0)); if(direction != 0 && first_open_msc <= 0) return(Refuse(reason, "position_first_open_missing")); next.direction = direction; next.first_open_msc = first_open_msc; const long position_age = (direction == 0 ? 0 : tick.time - first_open_msc / 1000); float account[24]; ArrayInitialize(account, 0.0f); account[0] = (float)balance; account[1] = (float)equity; account[2] = (float)(buy_lots - sell_lots); account[3] = (float)((buy_lots + sell_lots) > 0 ? weighted_price / (buy_lots + sell_lots) : 0); account[4] = (float)sl; account[5] = (float)tp; account[6] = (float)MathMax(-position_swap, 0); account[7] = (float)next.day_start_equity; account[8] = (float)next.running_peak_equity; account[9] = (float)next.previous_equity; account[10] = (float)margin; account[11] = 0; account[12] = (float)free_margin; account[13] = (float)next.realized_booked; account[14] = (float)next.paid_costs_booked; account[15] = (float)tick.bid; account[16] = (float)tick.ask; account[17] = (sl > 0 ? 1.0f : 0.0f); account[18] = (tp > 0 ? 1.0f : 0.0f); account[19] = ((balance < 50 || equity < 50) ? 1.0f : 0.0f); account[20] = (next.peak_initialized ? 1.0f : 0.0f); account[21] = (float)position_age; account[22] = (next.day_open != (long)DPCCPhysicalDayOpen((datetime)next.previous_decision) ? 1.0f : 0.0f); for(int i = 0; i < 24; i++) if(!MathIsValidNumber((double)account[i])) return(Refuse(reason, "account24_float_range")); float features[]; if(!DPCCBuildAccount13Features(balance, next.previous_balance, equity, next.previous_equity, buy_lots, sell_lots, buy_profit, sell_profit, discount, decision_server_time, features, reason) || ArraySize(features) != 13) return(false); for(int i = 0; i < 13; i++) if(!MathIsValidNumber((double)features[i])) return(Refuse(reason, "account13_nonfinite")); long opens[], closes[]; float counts[], calendar[]; if(!BuildSchedule(decision_server_time, (int)m_config.shape.H, (int)m_config.shape.E, opens, closes, counts, calendar, reason)) return(false); float instrument[48]; for(int i = 0; i < 48; i++) instrument[i] = m_profile.instrument[i]; instrument[14] = (float)tick.ask; instrument[33] = (m_config.price_basis == 1 ? (float)tick.last : 0.0f); const float quote_age = (float)MathMax(0, tick.time - (long)tick.time_msc / 1000); instrument[36] = quote_age; instrument[37] = (m_config.price_basis == 1 ? quote_age : 0.0f); float execution[24]; ArrayInitialize(execution, 0.0f); const float spread = (float)(tick.ask - tick.bid); for(int i = 0; i < 4; i++) execution[i] = spread; execution[13] = 1.0f; execution[14] = 1.0f; execution[15] = 1.0f; execution[16] = (float)instrument[23]; execution[17] = execution[16]; execution[20] = (m_config.price_basis == 1 ? (float)(tick.bid - instrument[33]) : 0.0f); execution[22] = 0.0f; CBufferFloat *result = NULL; if(!DPCCBuildDecisionFinancialContext(m_config, market_ohlc, account, instrument, m_profile.limits, execution, decision_server_time, decision_server_time, opens, closes, counts, calendar, m_snapshot_work, m_context_work, m_flags_work, m_status_work, result, reason)) return(false); if(CheckPointer(result) == POINTER_INVALID || result != GetPointer(m_context_work)) return(Refuse(reason, "context_publish")); next.previous_decision = decision_server_time; next.previous_balance = balance; next.previous_equity = equity; m_ledger = next; context_out = result; if(ArrayCopy(account13, features) != 13) { context_out = NULL; ArrayFree(account13); return(Refuse(reason, "account13_publish")); } return(true); } }; #endif //+------------------------------------------------------------------+ //| An actual server-midnight observation bound to prior state | //+------------------------------------------------------------------+ struct SDPCCPhysicalDayBoundary { SDPCCPhysicalLedger prior; string capture_id; string clock_basis; datetime observed_at; datetime day_open; double day_start_equity; bool native_observed; }; //+------------------------------------------------------------------+ //| Match the complete retained account checkpoint for day import | //+------------------------------------------------------------------+ bool DPCCPhysicalDaySamePrior(const SDPCCPhysicalLedger &left, const SDPCCPhysicalLedger &right) { return(left.login == right.login && left.server == right.server && left.currency == right.currency && left.symbol == right.symbol && left.magic == right.magic && left.deployment_id == right.deployment_id && left.checkpoint_id == right.checkpoint_id && left.previous_mark == right.previous_mark && left.previous_balance == right.previous_balance && left.previous_equity == right.previous_equity && left.snapshot_time == right.snapshot_time && left.snapshot_equity == right.snapshot_equity && left.day_open == right.day_open && left.day_start_equity == right.day_start_equity && left.running_peak_equity == right.running_peak_equity && left.peak_initialized == right.peak_initialized && left.terminal == right.terminal && left.day_reset_observed == right.day_reset_observed && left.deal_cursor_msc == right.deal_cursor_msc && left.deal_cursor_ticket == right.deal_cursor_ticket && left.realized_booked == right.realized_booked && left.paid_costs_booked == right.paid_costs_booked && left.booked_costs_confirmed == right.booked_costs_confirmed && left.booked_costs_evidence == right.booked_costs_evidence); } //+------------------------------------------------------------------+ //| Validate the retained state without estimating a new day equity | //+------------------------------------------------------------------+ bool DPCCPhysicalDayPrior(const SDPCCPhysicalPolicy &owner, const SDPCCPhysicalLedger &prior, string &reason) { if(!DPCCPhysicalPolicyCheck(owner, reason)) return(false); if(prior.login != owner.login || prior.server != owner.server || prior.currency != owner.currency || prior.symbol != owner.symbol || prior.magic != owner.magic || prior.deployment_id != owner.deployment_id || prior.checkpoint_id == "") return(DPCCPhysicalRefuse("day.prior_identity", reason)); const double values[] = {prior.previous_balance, prior.previous_equity, prior.snapshot_equity, prior.day_start_equity, prior.running_peak_equity, prior.realized_booked, prior.paid_costs_booked }; for(int i = 0; i < ArraySize(values); i++) if(!MathIsValidNumber(values[i])) return(DPCCPhysicalRefuse("day.prior_finite", reason)); if(prior.previous_mark <= 0 || prior.snapshot_time <= 0 || prior.snapshot_time > prior.previous_mark || prior.previous_balance <= 0 || prior.previous_equity <= 0 || prior.snapshot_equity <= 0 || prior.day_start_equity <= 0 || prior.day_open != DPCCPhysicalDayOpen(prior.previous_mark) || !prior.peak_initialized || prior.running_peak_equity < prior.previous_equity || !prior.booked_costs_confirmed || prior.booked_costs_evidence == "" || prior.paid_costs_booked < 0 || prior.deal_cursor_msc < (long)prior.snapshot_time * 1000 || prior.deal_cursor_msc / 1000 > (long)prior.previous_mark) return(DPCCPhysicalRefuse("day.prior_state", reason)); return(true); } //+------------------------------------------------------------------+ //| Capture actual equity exactly at the native server day boundary | //+------------------------------------------------------------------+ bool DPCCPhysicalCaptureDayBoundary(const SDPCCPhysicalPolicy &owner, const SDPCCPhysicalLedger &prior, const string capture_id, SDPCCPhysicalDayBoundary &boundary, string &reason) { const SDPCCPhysicalLedger retained = prior; ZeroMemory(boundary); reason = ""; if(!DPCCPhysicalDayPrior(owner, retained, reason)) return(false); const datetime now = TimeCurrent(); const datetime day_open = DPCCPhysicalDayOpen(now); if(capture_id == "" || now <= 0 || now != day_open || retained.previous_mark >= day_open || retained.day_open >= day_open) return(DPCCPhysicalRefuse("day.requires_actual_midnight_capture", reason)); SDPCCPhysicalObservation observed; if(!DPCCPhysicalObserve(owner, (datetime)(retained.deal_cursor_msc / 1000), observed, reason)) return(false); if(observed.asof != day_open || TimeCurrent() != day_open || !MathIsValidNumber(observed.equity) || observed.equity <= 0) return(DPCCPhysicalRefuse("day.capture_crossed_boundary_or_invalid_equity", reason)); boundary.prior = retained; boundary.capture_id = capture_id; boundary.clock_basis = "SERVER_TIMECURRENT_V1"; boundary.observed_at = observed.asof; boundary.day_open = day_open; boundary.day_start_equity = observed.equity; boundary.native_observed = true; return(true); } //+------------------------------------------------------------------+ //| Import captured day equity while preserving prior financial | //| state | //+------------------------------------------------------------------+ bool DPCCPhysicalImportDayBoundary(const SDPCCPhysicalPolicy &owner, const SDPCCPhysicalLedger &prior, const SDPCCPhysicalDayBoundary &boundary, const datetime account_asof, SDPCCPhysicalLedger &imported, string &reason) { const SDPCCPhysicalLedger retained = prior; const SDPCCPhysicalDayBoundary captured = boundary; ZeroMemory(imported); reason = ""; if(!DPCCPhysicalDayPrior(owner, retained, reason)) return(false); if(!captured.native_observed || captured.capture_id == "" || captured.clock_basis != "SERVER_TIMECURRENT_V1") return(DPCCPhysicalRefuse("day.missing_actual_captured_source", reason)); if(!DPCCPhysicalDaySamePrior(retained, captured.prior)) return(DPCCPhysicalRefuse("day.stale_captured_prior", reason)); if(captured.observed_at <= 0 || captured.observed_at != captured.day_open || captured.day_open != DPCCPhysicalDayOpen(captured.observed_at) || captured.day_open <= retained.previous_mark || captured.day_open <= retained.day_open || account_asof < captured.observed_at || DPCCPhysicalDayOpen(account_asof) != captured.day_open) return(DPCCPhysicalRefuse("day.source_time_or_import_day", reason)); if(!MathIsValidNumber(captured.day_start_equity) || captured.day_start_equity <= 0) return(DPCCPhysicalRefuse("day.captured_equity", reason)); //--- Retain financial history and include only the new actual boundary observation in the peak. imported = retained; imported.day_open = captured.day_open; imported.day_start_equity = captured.day_start_equity; imported.running_peak_equity = MathMax(retained.running_peak_equity, captured.day_start_equity); imported.day_reset_observed = true; return(true); } #endif //+------------------------------------------------------------------+ //| Physical prepared-open receipt and exact caller-owned ledger | //+------------------------------------------------------------------+ struct SDPCCPhysicalOpenMark { SDPCCPhysicalPolicy policy; SDPCCPhysicalLedger ledger; SDPCCMarketClockCapture clock; SDPCCPhysicalDayBoundary day_boundary; double scale; string input_ledger_sha256, snapshot_sha256, confirmation_sha256; float account13[13]; }; union UDPCCPhysicalDoubleBits { double value; ulong bits; }; //+------------------------------------------------------------------+ //| Function DPCCPhysicalDoubleWire | //+------------------------------------------------------------------+ string DPCCPhysicalDoubleWire(const double value) { UDPCCPhysicalDoubleBits bits; bits.value = value; return(StringFormat("%016I64X", bits.bits)); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalReadDouble | //+------------------------------------------------------------------+ bool DPCCPhysicalReadDouble(const string wire, double &value) { UDPCCPhysicalDoubleBits bits; if(!DPCCOfflineCheckpointReadHex64(wire, bits.bits)) return(false); value = bits.value; return(MathIsValidNumber(value)); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalWireText | //+------------------------------------------------------------------+ bool DPCCPhysicalWireText(const string value) { return(StringLen(value) <= 512 && StringFind(value, "|") < 0 && StringFind(value, "\n") < 0 && StringFind(value, "\r") < 0 && StringFind(value, "\t") < 0); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalLedgerSHA | //+------------------------------------------------------------------+ bool DPCCPhysicalLedgerSHA(const SDPCCPhysicalLedger &ledger, string &sha, string &reason) { string wire = "DPCC_PHYSICAL_LEDGER_V1|"; wire += (string)(ulong)ledger.login + "|"; wire += ledger.server + "|"; wire += ledger.currency + "|"; wire += ledger.symbol + "|"; wire += (string)(ulong)ledger.magic + "|"; wire += ledger.deployment_id + "|"; wire += ledger.checkpoint_id + "|"; wire += (string)(ulong)ledger.previous_mark + "|"; wire += DPCCPhysicalDoubleWire(ledger.previous_balance) + "|"; wire += DPCCPhysicalDoubleWire(ledger.previous_equity) + "|"; wire += (string)(ulong)ledger.snapshot_time + "|"; wire += DPCCPhysicalDoubleWire(ledger.snapshot_equity) + "|"; wire += (string)(ulong)ledger.day_open + "|"; wire += DPCCPhysicalDoubleWire(ledger.day_start_equity) + "|"; wire += DPCCPhysicalDoubleWire(ledger.running_peak_equity) + "|"; wire += (string)(ulong)ledger.peak_initialized + "|"; wire += (string)(ulong)ledger.terminal + "|"; wire += (string)(ulong)ledger.day_reset_observed + "|"; wire += (string)(ulong)ledger.deal_cursor_msc + "|"; wire += (string)(ulong)ledger.deal_cursor_ticket + "|"; wire += DPCCPhysicalDoubleWire(ledger.realized_booked) + "|"; wire += DPCCPhysicalDoubleWire(ledger.paid_costs_booked) + "|"; wire += (string)(ulong)ledger.booked_costs_confirmed + "|"; wire += ledger.booked_costs_evidence + "|"; return(DPCCCheckpointTextSHA(wire, sha, reason)); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalPolicySHA | //+------------------------------------------------------------------+ bool DPCCPhysicalPolicySHA(const SDPCCPhysicalPolicy &policy, string &sha, string &reason) { string wire = "DPCC_PHYSICAL_POLICY_V1|"; wire += (string)(ulong)policy.login + "|"; wire += policy.server + "|"; wire += policy.currency + "|"; wire += policy.symbol + "|"; wire += (string)(ulong)policy.magic + "|"; wire += policy.deployment_id + "|"; wire += (string)(ulong)policy.max_positions + "|"; wire += (string)(ulong)policy.max_history_deals + "|"; wire += (string)(ulong)policy.max_quote_age_seconds + "|"; wire += DPCCPhysicalDoubleWire(policy.volume_atol) + "|"; wire += DPCCPhysicalDoubleWire(policy.money_atol) + "|"; wire += DPCCPhysicalDoubleWire(policy.money_rtol) + "|"; return(DPCCCheckpointTextSHA(wire, sha, reason)); } //+------------------------------------------------------------------+ //| Check a retained open receipt without accessing an account | //+------------------------------------------------------------------+ bool DPCCPhysicalOpenMarkCheck(const SDPCCPhysicalOpenMark &mark, string &reason) { if(!DPCCPhysicalPolicyCheck(mark.policy, reason)) return(false); const SDPCCPhysicalLedger ledger = mark.ledger; const SDPCCMarketClockCapture clock = mark.clock; if(StringLen(mark.policy.server) == 0 || StringLen(mark.policy.symbol) == 0 || StringLen(mark.policy.deployment_id) == 0 || StringLen(mark.policy.currency) == 0 || !MathIsValidNumber( mark.scale) || mark.scale <= 0 || ledger.login != mark.policy.login || ledger.server != mark.policy.server || ledger.currency != mark.policy.currency || ledger.symbol != mark.policy.symbol || ledger.magic != mark.policy.magic || ledger.deployment_id != mark.policy.deployment_id || StringLen(ledger.checkpoint_id) == 0 || !ledger.booked_costs_confirmed || StringLen(ledger.booked_costs_evidence) == 0 || !ledger.peak_initialized || ledger.day_reset_observed || ledger.previous_balance <= 0 || ledger.previous_equity <= 0 || ledger.snapshot_equity <= 0 || ledger.day_start_equity <= 0 || ledger.running_peak_equity < ledger.previous_equity || ledger.paid_costs_booked < 0 || ledger.terminal != (ledger.previous_balance < 50 || ledger.previous_equity < 50) || ledger.previous_mark != clock.broker_second || ledger.snapshot_time <= 0 || ledger.snapshot_time > ledger.previous_mark || ledger.snapshot_time > LONG_MAX / 1000 || ledger.day_open != DPCCPhysicalDayOpen(ledger.previous_mark) || ledger.deal_cursor_msc < (long)ledger.snapshot_time * 1000 || ledger.deal_cursor_msc / 1000 > (long)ledger.previous_mark || clock.timeframe != PERIOD_H1 || clock.symbol != mark.policy.symbol || clock.broker_second <= 0 || clock.broker_second != clock.broker_bar_open || clock.broker_bar_close <= clock.broker_bar_open || clock.decision_bar_index < 0 || clock.broker_tick_msc / 1000 != clock.broker_second || clock.utc_tick_msc / 1000 != clock.utc_second || clock.broker_tick_msc % 1000 != clock.utc_tick_msc % 1000 || clock.utc_second <= 0 || StringLen(clock.map_path) == 0 || StringLen(clock.map_id) == 0 || !DPCCResolverLowerSha256(clock.map_sha256) || !DPCCResolverLowerSha256(mark.input_ledger_sha256) || !DPCCResolverLowerSha256(mark.snapshot_sha256) || !DPCCResolverLowerSha256(mark.confirmation_sha256)) return(DPCCPhysicalRefuse("open_mark.binding", reason)); if(StringLen(mark.day_boundary.capture_id) > 0) { SDPCCPhysicalLedger imported; string imported_sha = ""; if(!DPCCPhysicalImportDayBoundary(mark.policy, mark.day_boundary.prior, mark.day_boundary, ledger.previous_mark, imported, reason)) return(false); if(!DPCCPhysicalLedgerSHA(imported, imported_sha, reason)) return(false); if(imported_sha != mark.input_ledger_sha256 || mark.day_boundary.observed_at != ledger.previous_mark || mark.day_boundary.day_start_equity != ledger.previous_equity || imported.day_open != ledger.day_open || imported.day_start_equity != ledger.day_start_equity) return(DPCCPhysicalRefuse("open_mark.day_import", reason)); } else { SDPCCPhysicalDayBoundary empty; ZeroMemory(empty); string empty_sha = "", actual_sha = ""; if(!DPCCPhysicalLedgerSHA(empty.prior, empty_sha, reason) || !DPCCPhysicalLedgerSHA(mark.day_boundary.prior, actual_sha, reason)) return(false); if(actual_sha != empty_sha || StringLen(mark.day_boundary.clock_basis) > 0 || mark.day_boundary.observed_at != 0 || mark.day_boundary.day_open != 0 || mark.day_boundary.day_start_equity != 0 || mark.day_boundary.native_observed) return(DPCCPhysicalRefuse("open_mark.empty_day_proof", reason)); } const double money[] = {ledger.previous_balance, ledger.previous_equity, ledger.snapshot_equity, ledger.day_start_equity, ledger.running_peak_equity, ledger.realized_booked, ledger.paid_costs_booked }; for(int i = 0; i < ArraySize(money); i++) if(!MathIsValidNumber(money[i])) return(DPCCPhysicalRefuse("open_mark.money", reason)); for(int i = 0; i < 13; i++) if(!MathIsValidNumber(mark.account13[i])) return(DPCCPhysicalRefuse("open_mark.account13", reason)); return(true); } //+------------------------------------------------------------------+ //| Encode every ledger, clock and feature field without rounding | //+------------------------------------------------------------------+ bool DPCCPhysicalOpenMarkWire(const SDPCCPhysicalOpenMark &mark, string &wire, string &reason) { wire = ""; if(!DPCCPhysicalOpenMarkCheck(mark, reason)) return(false); const string texts[] = {mark.policy.server, mark.policy.currency, mark.policy.symbol, mark.policy.deployment_id, mark.ledger.server, mark.ledger.currency, mark.ledger.symbol, mark.ledger.deployment_id, mark.ledger.checkpoint_id, mark.ledger.booked_costs_evidence, mark.clock.symbol, mark.clock.map_path, mark.clock.map_sha256, mark.clock.map_id, mark.day_boundary.prior.server, mark.day_boundary.prior.currency, mark.day_boundary.prior.symbol, mark.day_boundary.prior.deployment_id, mark.day_boundary.prior.checkpoint_id, mark.day_boundary.prior.booked_costs_evidence, mark.day_boundary.capture_id, mark.day_boundary.clock_basis, mark.input_ledger_sha256, mark.snapshot_sha256, mark.confirmation_sha256}; for(int i = 0; i < ArraySize(texts); i++) if(!DPCCPhysicalWireText(texts[i])) return(DPCCPhysicalRefuse("open_mark.text", reason)); wire = "DPCC_PHYSICAL_OPEN_V1|"; wire += (string)(ulong)mark.policy.login + "|"; wire += mark.policy.server + "|"; wire += mark.policy.currency + "|"; wire += mark.policy.symbol + "|"; wire += (string)(ulong)mark.policy.magic + "|"; wire += mark.policy.deployment_id + "|"; wire += (string)(ulong)mark.policy.max_positions + "|"; wire += (string)(ulong)mark.policy.max_history_deals + "|"; wire += (string)(ulong)mark.policy.max_quote_age_seconds + "|"; wire += DPCCPhysicalDoubleWire(mark.policy.volume_atol) + "|"; wire += DPCCPhysicalDoubleWire(mark.policy.money_atol) + "|"; wire += DPCCPhysicalDoubleWire(mark.policy.money_rtol) + "|"; wire += (string)(ulong)mark.ledger.login + "|"; wire += mark.ledger.server + "|"; wire += mark.ledger.currency + "|"; wire += mark.ledger.symbol + "|"; wire += (string)(ulong)mark.ledger.magic + "|"; wire += mark.ledger.deployment_id + "|"; wire += mark.ledger.checkpoint_id + "|"; wire += (string)(ulong)mark.ledger.previous_mark + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.previous_balance) + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.previous_equity) + "|"; wire += (string)(ulong)mark.ledger.snapshot_time + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.snapshot_equity) + "|"; wire += (string)(ulong)mark.ledger.day_open + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.day_start_equity) + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.running_peak_equity) + "|"; wire += (string)(ulong)mark.ledger.peak_initialized + "|"; wire += (string)(ulong)mark.ledger.terminal + "|"; wire += (string)(ulong)mark.ledger.day_reset_observed + "|"; wire += (string)(ulong)mark.ledger.deal_cursor_msc + "|"; wire += (string)(ulong)mark.ledger.deal_cursor_ticket + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.realized_booked) + "|"; wire += DPCCPhysicalDoubleWire(mark.ledger.paid_costs_booked) + "|"; wire += (string)(ulong)mark.ledger.booked_costs_confirmed + "|"; wire += mark.ledger.booked_costs_evidence + "|"; wire += (string)(ulong)mark.clock.native_observed + "|"; wire += mark.clock.symbol + "|"; wire += mark.clock.map_path + "|"; wire += mark.clock.map_sha256 + "|"; wire += mark.clock.map_id + "|"; wire += (string)(ulong)mark.clock.timeframe + "|"; wire += (string)(ulong)mark.clock.broker_second + "|"; wire += (string)(ulong)mark.clock.broker_tick_msc + "|"; wire += (string)(ulong)mark.clock.utc_second + "|"; wire += (string)(ulong)mark.clock.utc_tick_msc + "|"; wire += (string)(ulong)mark.clock.decision_bar_index + "|"; wire += (string)(ulong)mark.clock.broker_bar_open + "|"; wire += (string)(ulong)mark.clock.broker_bar_close + "|"; wire += (string)(ulong)mark.day_boundary.prior.login + "|"; wire += mark.day_boundary.prior.server + "|"; wire += mark.day_boundary.prior.currency + "|"; wire += mark.day_boundary.prior.symbol + "|"; wire += (string)(ulong)mark.day_boundary.prior.magic + "|"; wire += mark.day_boundary.prior.deployment_id + "|"; wire += mark.day_boundary.prior.checkpoint_id + "|"; wire += (string)(ulong)mark.day_boundary.prior.previous_mark + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.previous_balance) + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.previous_equity) + "|"; wire += (string)(ulong)mark.day_boundary.prior.snapshot_time + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.snapshot_equity) + "|"; wire += (string)(ulong)mark.day_boundary.prior.day_open + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.day_start_equity) + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.running_peak_equity) + "|"; wire += (string)(ulong)mark.day_boundary.prior.peak_initialized + "|"; wire += (string)(ulong)mark.day_boundary.prior.terminal + "|"; wire += (string)(ulong)mark.day_boundary.prior.day_reset_observed + "|"; wire += (string)(ulong)mark.day_boundary.prior.deal_cursor_msc + "|"; wire += (string)(ulong)mark.day_boundary.prior.deal_cursor_ticket + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.realized_booked) + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.prior.paid_costs_booked) + "|"; wire += (string)(ulong)mark.day_boundary.prior.booked_costs_confirmed + "|"; wire += mark.day_boundary.prior.booked_costs_evidence + "|"; wire += mark.day_boundary.capture_id + "|"; wire += mark.day_boundary.clock_basis + "|"; wire += (string)(ulong)mark.day_boundary.observed_at + "|"; wire += (string)(ulong)mark.day_boundary.day_open + "|"; wire += DPCCPhysicalDoubleWire(mark.day_boundary.day_start_equity) + "|"; wire += (string)(ulong)mark.day_boundary.native_observed + "|"; wire += DPCCPhysicalDoubleWire(mark.scale) + "|"; wire += mark.input_ledger_sha256 + "|"; wire += mark.snapshot_sha256 + "|"; wire += mark.confirmation_sha256 + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[0]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[1]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[2]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[3]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[4]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[5]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[6]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[7]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[8]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[9]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[10]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[11]) + "|"; wire += DPCCOfflineCheckpointFloat(mark.account13[12]) + "|"; wire += "END_DPCC_PHYSICAL_OPEN_V1"; return(true); } //+------------------------------------------------------------------+ //| Function DPCCPhysicalReadOpenMark | //+------------------------------------------------------------------+ bool DPCCPhysicalReadOpenMark(const string wire, SDPCCPhysicalOpenMark &mark, string &reason) { ZeroMemory(mark); string fields[]; if(StringLen(wire) > 32768 || StringSplit(wire, '|', fields) != 98 || fields[0] != "DPCC_PHYSICAL_OPEN_V1" || fields[98 - 1] != "END_DPCC_PHYSICAL_OPEN_V1") return(DPCCPhysicalRefuse("open_mark.wire", reason)); SDPCCPhysicalOpenMark candidate; ZeroMemory(candidate); int cursor = 1; ulong number = 0; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.policy.login = (long)number; candidate.policy.server = fields[cursor++]; candidate.policy.currency = fields[cursor++]; candidate.policy.symbol = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.policy.magic = (long)number; candidate.policy.deployment_id = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)UINT_MAX, number)) return(false); candidate.policy.max_positions = (uint)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)UINT_MAX, number)) return(false); candidate.policy.max_history_deals = (uint)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)UINT_MAX, number)) return(false); candidate.policy.max_quote_age_seconds = (uint)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.policy.volume_atol)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.policy.money_atol)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.policy.money_rtol)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.login = (long)number; candidate.ledger.server = fields[cursor++]; candidate.ledger.currency = fields[cursor++]; candidate.ledger.symbol = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.magic = (long)number; candidate.ledger.deployment_id = fields[cursor++]; candidate.ledger.checkpoint_id = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.previous_mark = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.previous_balance)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.previous_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.snapshot_time = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.snapshot_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.day_open = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.day_start_equity)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.running_peak_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.ledger.peak_initialized = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.ledger.terminal = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.ledger.day_reset_observed = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.deal_cursor_msc = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.ledger.deal_cursor_ticket = (ulong)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.realized_booked)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.ledger.paid_costs_booked)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.ledger.booked_costs_confirmed = (bool)number; candidate.ledger.booked_costs_evidence = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.clock.native_observed = (bool)number; candidate.clock.symbol = fields[cursor++]; candidate.clock.map_path = fields[cursor++]; candidate.clock.map_sha256 = fields[cursor++]; candidate.clock.map_id = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.timeframe = (ENUM_TIMEFRAMES)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.broker_second = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.broker_tick_msc = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.utc_second = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.utc_tick_msc = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.decision_bar_index = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.broker_bar_open = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.clock.broker_bar_close = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.login = (long)number; candidate.day_boundary.prior.server = fields[cursor++]; candidate.day_boundary.prior.currency = fields[cursor++]; candidate.day_boundary.prior.symbol = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.magic = (long)number; candidate.day_boundary.prior.deployment_id = fields[cursor++]; candidate.day_boundary.prior.checkpoint_id = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.previous_mark = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.previous_balance)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.previous_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.snapshot_time = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.snapshot_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.day_open = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.day_start_equity)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.running_peak_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.day_boundary.prior.peak_initialized = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.day_boundary.prior.terminal = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.day_boundary.prior.day_reset_observed = (bool)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.deal_cursor_msc = (long)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.prior.deal_cursor_ticket = (ulong)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.realized_booked)) return(false); if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.prior.paid_costs_booked)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.day_boundary.prior.booked_costs_confirmed = (bool)number; candidate.day_boundary.prior.booked_costs_evidence = fields[cursor++]; candidate.day_boundary.capture_id = fields[cursor++]; candidate.day_boundary.clock_basis = fields[cursor++]; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.observed_at = (datetime)number; if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)LONG_MAX, number)) return(false); candidate.day_boundary.day_open = (datetime)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.day_boundary.day_start_equity)) return(false); if(!DPCCCheckpointReadUnsigned(fields[cursor++], (ulong)1, number)) return(false); candidate.day_boundary.native_observed = (bool)number; if(!DPCCPhysicalReadDouble(fields[cursor++], candidate.scale)) return(false); candidate.input_ledger_sha256 = fields[cursor++]; candidate.snapshot_sha256 = fields[cursor++]; candidate.confirmation_sha256 = fields[cursor++]; if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[0])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[1])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[2])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[3])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[4])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[5])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[6])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[7])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[8])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[9])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[10])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[11])) return(false); if(!DPCCOfflineCheckpointReadFloat(fields[cursor++], candidate.account13[12])) return(false); string rebuilt = ""; if(!DPCCPhysicalOpenMarkWire(candidate, rebuilt, reason) || rebuilt != wire) return(DPCCPhysicalRefuse("open_mark.noncanonical", reason)); mark = candidate; return(true); } //+------------------------------------------------------------------+ //| Seal only the existing reconciled physical result at an H1 open | //+------------------------------------------------------------------+ bool DPCCPhysicalSealOpen(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, const SDPCCPhysicalAccount &account, const SDPCCMarketClockCapture &capture, const double scale, const SDPCCPhysicalDayBoundary &day_boundary, SDPCCPhysicalOpenMark &mark, string &reason) { ZeroMemory(mark); if(StringLen(confirmation.accrued_cost_evidence) == 0 || (ArraySize(account.positions) > 0 && StringLen(confirmation.margin_evidence) == 0)) return(DPCCPhysicalRefuse("open_mark.missing_cost_or_margin_evidence", reason)); if(!account.complete || !account.clock_bound || !account.original_symbol_scope_verified || account.all_orders != 0 || account.all_positions != (uint)ArraySize(account.positions) || account.asof != capture.broker_second || account.asof_utc != capture.utc_second || account.quote_msc != capture.broker_tick_msc || account.quote_utc_msc != capture.utc_tick_msc || account.native_observed != capture.native_observed || ledger.terminal) return(DPCCPhysicalRefuse("open_mark.incomplete_or_foreign_exposure", reason)); SDPCCPhysicalOpenMark candidate; ZeroMemory(candidate); candidate.policy = policy; candidate.ledger = account.next_ledger; candidate.ledger.terminal = account.account[0] < 50 || account.account[1] < 50; candidate.clock = capture; candidate.day_boundary = day_boundary; candidate.scale = scale; ArrayCopy(candidate.account13, account.features); if(!DPCCPhysicalLedgerSHA(ledger, candidate.input_ledger_sha256, reason)) return(false); string snapshot = account.ownership_evidence; for(int i = 0; i < 24; i++) snapshot += "|" + DPCCPhysicalDoubleWire(account.account[i]); for(int i = 0; i < ArraySize(account.positions); i++) { const SDPCCPhysicalPosition p = account.positions[i]; snapshot += "|" + (string)p.ticket + "|" + (string)p.identifier + "|" + (string)p.magic + "|" + (string)p.type + "|" + (string)p.opened_msc + "|" + (string)p.updated_msc; const double values[] = {p.volume, p.open_price, p.sl, p.tp, p.profit, p.swap}; for(int j = 0; j < ArraySize(values); j++) snapshot += "|" + DPCCPhysicalDoubleWire(values[j]); } if(!DPCCCheckpointTextSHA(snapshot, candidate.snapshot_sha256, reason)) return(false); string evidence = confirmation.accrued_cost_evidence + "|" + confirmation.margin_evidence + "|" + DPCCPhysicalDoubleWire(confirmation.accrued_unbooked_cost) + "|" + DPCCPhysicalDoubleWire(confirmation.owned_margin) + "|" + DPCCPhysicalDoubleWire( confirmation.other_margin); if(!DPCCCheckpointTextSHA(evidence, candidate.confirmation_sha256, reason)) return(false); string wire = ""; if(!DPCCPhysicalOpenMarkWire(candidate, wire, reason)) return(false); mark = candidate; return(true); } //+------------------------------------------------------------------+ //| Pure synthetic receipt uses the real physical and clock builders | //+------------------------------------------------------------------+ bool DPCCPhysicalBuildOpen(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, const SDPCCPhysicalObservation &source, CDPCCMarketClock &clock, const long ordinal, const double scale, SDPCCPhysicalOpenMark &mark, string &reason) { ZeroMemory(mark); SDPCCMarketClockCapture capture; if(!clock.Derive((long)source.asof, source.tick.time_msc, ordinal, (long)source.asof, capture, reason)) return(false); capture.symbol = policy.symbol; capture.timeframe = PERIOD_H1; SDPCCPhysicalAccount account; if(!DPCCPhysicalBuild(policy, ledger, confirmation, source, account, reason, GetPointer(clock))) return(false); SDPCCPhysicalDayBoundary empty; ZeroMemory(empty); return(DPCCPhysicalSealOpen(policy, ledger, confirmation, account, capture, scale, empty, mark, reason)); } //+------------------------------------------------------------------+ //| Native receipt requires the existing exact current capture | //+------------------------------------------------------------------+ bool DPCCPhysicalCaptureOpen(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, CDPCCMarketClock &clock, const double scale, SDPCCPhysicalOpenMark &mark, string &reason) { ZeroMemory(mark); SDPCCPhysicalAccount account; if(!DPCCPhysicalCapture(policy, ledger, confirmation, account, reason, GetPointer(clock))) return(false); SDPCCMarketClockCapture capture; if(!clock.ReadCapture(capture, reason)) return(false); SDPCCPhysicalDayBoundary empty; ZeroMemory(empty); return(DPCCPhysicalSealOpen(policy, ledger, confirmation, account, capture, scale, empty, mark, reason)); } //+------------------------------------------------------------------+ //| Day transition reuses the observed-boundary import contract | //+------------------------------------------------------------------+ bool DPCCPhysicalBuildOpenAfterDay(const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &prior, const SDPCCPhysicalDayBoundary &boundary, const SDPCCPhysicalConfirmation &confirmation, const SDPCCPhysicalObservation &source, CDPCCMarketClock &clock, const long ordinal, const double scale, SDPCCPhysicalOpenMark &mark, string &reason) { ZeroMemory(mark); SDPCCPhysicalLedger imported; if(!DPCCPhysicalImportDayBoundary(policy, prior, boundary, source.asof, imported, reason)) return(false); SDPCCMarketClockCapture capture; if(!clock.Derive((long)source.asof, source.tick.time_msc, ordinal, (long)source.asof, capture, reason)) return(false); capture.symbol = policy.symbol; capture.timeframe = PERIOD_H1; SDPCCPhysicalAccount account; if(!DPCCPhysicalBuild(policy, imported, confirmation, source, account, reason, GetPointer(clock))) return(false); return(DPCCPhysicalSealOpen(policy, imported, confirmation, account, capture, scale, boundary, mark, reason)); } //+------------------------------------------------------------------+ //| Actual equity delta between consecutive confirmed opens | //+------------------------------------------------------------------+ bool DPCCPhysicalActualReward(const SDPCCPhysicalOpenMark &previous, const SDPCCPhysicalOpenMark &next, float &reward, float &bootstrap_mask, string &reason) { reward = 0; bootstrap_mask = 0; string old_policy = "", new_policy = "", ledger_sha = ""; if(!DPCCPhysicalOpenMarkCheck(previous, reason) || !DPCCPhysicalOpenMarkCheck(next, reason) || !DPCCPhysicalPolicySHA(previous.policy, old_policy, reason) || !DPCCPhysicalPolicySHA(next.policy, new_policy, reason) || !DPCCPhysicalLedgerSHA(previous.ledger, ledger_sha, reason)) return(false); if(StringLen(next.day_boundary.capture_id) > 0) { SDPCCPhysicalLedger imported; if(!DPCCPhysicalDaySamePrior(previous.ledger, next.day_boundary.prior) || !DPCCPhysicalImportDayBoundary(previous.policy, previous.ledger, next.day_boundary, next.ledger.previous_mark, imported, reason) || !DPCCPhysicalLedgerSHA(imported, ledger_sha, reason)) return(DPCCPhysicalRefuse("actual_reward.day_chain", reason)); } if(previous.ledger.terminal || old_policy != new_policy || previous.scale != next.scale || previous.ledger.checkpoint_id != next.ledger.checkpoint_id || ledger_sha != next.input_ledger_sha256 || previous.clock.native_observed != next.clock.native_observed || previous.clock.map_path != next.clock.map_path || previous.clock.map_sha256 != next.clock.map_sha256 || previous.clock.map_id != next.clock.map_id || previous.clock.decision_bar_index == LONG_MAX || next.clock.decision_bar_index != previous.clock.decision_bar_index + 1 || next.clock.broker_bar_open < previous.clock.broker_bar_close || next.clock.utc_second <= previous.clock.utc_second || next.ledger.deal_cursor_msc < previous.ledger.deal_cursor_msc || (next.ledger.deal_cursor_msc == previous.ledger.deal_cursor_msc && next.ledger.deal_cursor_ticket < previous.ledger.deal_cursor_ticket)) return(DPCCPhysicalRefuse("actual_reward.chain", reason)); const double delta = (next.ledger.previous_equity - previous.ledger.previous_equity) / previous.scale; if(!MathIsValidNumber(delta) || !MathIsValidNumber((double)(float)delta)) return(DPCCPhysicalRefuse("actual_reward.float_range", reason)); reward = (float)delta; bootstrap_mask = next.ledger.terminal ? 0.0f : 1.0f; return(true); } struct SDPCCOnlineOwnerProfile { SDPCCShape shape; string offline_actor_sha256, decoder_sha256, caller_sha256, sampler_sha256; uint actor_every, actor_target_every, critic_target_every, selector_seed; float gamma, probability_weight, value_weight, actor_upstream, risk_parameter, mean_weight, target_tau; int risk_type, actor_critic_selector; bool physical_native_required; }; //+------------------------------------------------------------------+ //| Online boundary stores supplied observations without acquisition | //+------------------------------------------------------------------+ struct SDPCCOnlineOwnerState { //--- 0 initial, 1 recorded, 2 resolved, 3 applied. uint phase; ulong updates, actor_updates, actor_target_updates, critic_target_updates, sampler_cursor; ulong decision_identity, next_decision_identity; float scenarios[], next_scenarios[]; float account13[13], next_account13[13], behavior[6]; float reward, bootstrap_mask, sample_u1, sample_u2; string physical_mark_wire, physical_next_wire; string execution_transcript_wire; }; //+------------------------------------------------------------------+ //| Reject separators and controls in transcript text fields | //+------------------------------------------------------------------+ bool DPCCExecutionTranscriptText(const string value) { if(StringLen(value) > 512) return(false); for(int i = 0; i < StringLen(value); i++) { const ushort code = StringGetCharacter(value, i); if(code == '|' || code < 32 || (code >= 127 && code <= 159)) return(false); } return(true); } //+------------------------------------------------------------------+ //| Reconstruct the immutable intent and compare its reduced summary | //+------------------------------------------------------------------+ bool DPCCExecutionTranscriptReplay(const SDPCCExecutionIntent &stored, const SDPCCExecutionReceipt &receipts[], SDPCCExecutionIntent &replayed, string &reason) { reason = ""; const int count = ArraySize(receipts); if(count > DPCC_EXECUTION_RECEIPT_CAPACITY || stored.receipt_count != (uint)count || !DPCCExecutionTranscriptText(stored.deployment_id) || !DPCCExecutionTranscriptText(stored.symbol)) {reason = "execution_transcript.capacity_or_text"; return(false);} SDPCCExecutionIntent candidate; if(!DPCCPrepareExecutionIntent(stored.intent_id, stored.decision_identity, stored.deployment_id, stored.magic, stored.symbol, stored.policy_sha256, stored.checkpoint_sha256, stored.account_snapshot_sha256, stored.current_signed_lots, stored.requested_signed_delta_lots, stored.target_signed_lots, stored.actions, candidate, reason) || candidate.intent_sha256 != stored.intent_sha256) {reason = "execution_transcript.immutable_intent"; return(false);} for(int i = 0; i < count; i++) { if(!DPCCExecutionTranscriptText(receipts[i].deployment_id) || !DPCCExecutionTranscriptText(receipts[i].symbol) || !DPCCApplyExecutionReceipt(candidate, receipts[i], reason)) return(false); //--- Reducer duplicates are idempotent; a durable transcript stores each receipt once. if(candidate.receipt_count != (uint)i + 1) {reason = "execution_transcript.duplicate_receipt"; return(false);} } if(candidate.order_ticket != stored.order_ticket || DPCCPhysicalDoubleWire(candidate.filled_signed_lots) != DPCCPhysicalDoubleWire(stored.filled_signed_lots) || candidate.order_state != stored.order_state || candidate.last_order_state_time_msc != stored.last_order_state_time_msc) {reason = "execution_transcript.summary_mismatch"; return(false);} for(int i = 0; i < count; i++) if(candidate.receipt_keys[i] != stored.receipt_keys[i] || candidate.receipt_digests[i] != stored.receipt_digests[i]) {reason = "execution_transcript.receipt_index_mismatch"; return(false);} if(stored.terminal_binding_wire != "") { MqlTradeRequest request; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint id = 0; if(!DPCCReadTerminalRequest(stored, request, login, server, path, quote, dispatch, id, reason)) return(false); candidate.terminal_binding_wire = stored.terminal_binding_wire; candidate.terminal_result_wire = stored.terminal_result_wire; if(stored.terminal_result_wire != "") { MqlTradeResult result; if(!DPCCReadTerminalResult(stored, result, reason)) return(false); } } else if(stored.terminal_result_wire != "") {reason = "execution_transcript.result_without_request"; return(false);} replayed = candidate; return(true); } //+------------------------------------------------------------------+ //| Serialize complete ordered receipts with exact double bits | //+------------------------------------------------------------------+ bool DPCCExecutionTranscriptWire(const SDPCCExecutionIntent &intent, const SDPCCExecutionReceipt &receipts[], const string owner_sha, const string mark_sha, string &wire, string &reason) { wire = ""; SDPCCExecutionIntent replayed; if(!DPCCResolverLowerSha256(owner_sha) || !DPCCResolverLowerSha256(mark_sha) || !DPCCExecutionTranscriptReplay(intent, receipts, replayed, reason)) return(false); const bool terminal = intent.terminal_binding_wire != ""; const string schema = (terminal ? "DPCC_EXECUTION_TRANSCRIPT_TERMINAL_V2" : "DPCC_EXECUTION_TRANSCRIPT_V1"); string candidate = schema + "|" + owner_sha + "|" + mark_sha + "|"; candidate += (string)intent.intent_id + "|" + (string)intent.decision_identity + "|" + intent.deployment_id + "|"; candidate += (string)intent.magic + "|" + intent.symbol + "|" + intent.policy_sha256 + "|" + intent.checkpoint_sha256 + "|"; candidate += intent.account_snapshot_sha256 + "|" + DPCCPhysicalDoubleWire(intent.current_signed_lots) + "|"; candidate += DPCCPhysicalDoubleWire(intent.requested_signed_delta_lots) + "|"; candidate += DPCCPhysicalDoubleWire(intent.target_signed_lots) + "|"; for(int i = 0; i < 6; i++) candidate += DPCCPhysicalDoubleWire(intent.actions[i]) + "|"; candidate += intent.intent_sha256 + "|" + (string)intent.order_ticket + "|"; candidate += DPCCPhysicalDoubleWire(intent.filled_signed_lots) + "|" + (string)intent.order_state + "|"; candidate += (string)intent.last_order_state_time_msc + "|" + (string)intent.receipt_count + "|"; if(terminal) candidate += intent.terminal_binding_wire + "|" + intent.terminal_result_wire + "|"; for(int i = 0; i < ArraySize(receipts); i++) { const SDPCCExecutionReceipt r = receipts[i]; candidate += (string)r.intent_id + "|" + (string)r.order_ticket + "|" + (string)r.deal_id + "|"; candidate += (string)r.magic + "|" + (string)r.time_msc + "|" + r.deployment_id + "|" + r.symbol + "|"; candidate += r.intent_sha256 + "|" + (string)r.kind + "|" + (string)r.status + "|"; candidate += DPCCPhysicalDoubleWire(r.signed_fill_lots) + "|" + DPCCPhysicalDoubleWire(r.fill_price) + "|"; candidate += r.canonical_digest + "|"; } candidate += "END_" + schema; if(StringLen(candidate) > 524288) {reason = "execution_transcript.wire_size"; return(false);} wire = candidate; return(true); } //+------------------------------------------------------------------+ //| Parse, prepare and replay before publishing recovered execution | //+------------------------------------------------------------------+ bool DPCCExecutionTranscriptRead(const string wire, const string owner_sha, const string mark_sha, SDPCCExecutionIntent &intent, SDPCCExecutionReceipt &receipts[], string &reason) { reason = ""; string fields[]; if(StringLen(wire) > 524288 || StringFind(wire, "\n") >= 0 || StringFind(wire, "\r") >= 0) {reason = "execution_transcript.wire_size_or_lines"; return(false);} const int fields_count = StringSplit(wire, '|', fields); const bool terminal = fields_count > 0 && fields[0] == "DPCC_EXECUTION_TRANSCRIPT_TERMINAL_V2"; const string schema = (terminal ? "DPCC_EXECUTION_TRANSCRIPT_TERMINAL_V2" : "DPCC_EXECUTION_TRANSCRIPT_V1"); ulong count = 0, id = 0, decision = 0, ticket = 0, status = 0, time = 0; if(fields_count < 27 || fields[0] != schema || fields[1] != owner_sha || fields[2] != mark_sha || fields[fields_count - 1] != "END_" + schema || !DPCCCheckpointReadUnsigned(fields[25], DPCC_EXECUTION_RECEIPT_CAPACITY, count) || fields_count != 27 + (terminal ? 2 : 0) + 13 * (int)count || !DPCCCheckpointReadUnsigned(fields[3], ULONG_MAX, id) || !DPCCCheckpointReadUnsigned(fields[4], (ulong)LONG_MAX, decision) || !DPCCCheckpointReadUnsigned(fields[21], ULONG_MAX, ticket) || !DPCCCheckpointReadUnsigned(fields[23], DPCC_ORDER_CANCELED, status) || !DPCCCheckpointReadUnsigned(fields[24], (ulong)LONG_MAX, time)) {reason = "execution_transcript.wire_binding_or_integer"; return(false);} SDPCCExecutionIntent candidate; double current = 0, delta = 0, target = 0, filled = 0, actions[6]; if(!DPCCPhysicalReadDouble(fields[11], current) || !DPCCPhysicalReadDouble(fields[12], delta) || !DPCCPhysicalReadDouble(fields[13], target) || !DPCCPhysicalReadDouble(fields[22], filled)) {reason = "execution_transcript.double"; return(false);} for(int i = 0; i < 6; i++) if(!DPCCPhysicalReadDouble(fields[14 + i], actions[i])) return(false); const long magic = StringToInteger(fields[6]); if((string)magic != fields[6] || !DPCCPrepareExecutionIntent(id, decision, fields[5], magic, fields[7], fields[8], fields[9], fields[10], current, delta, target, actions, candidate, reason) || candidate.intent_sha256 != fields[20]) {reason = "execution_transcript.immutable_intent"; return(false);} if(terminal) { candidate.terminal_binding_wire = fields[26]; candidate.terminal_result_wire = fields[27]; MqlTradeRequest request; long login = 0, quote = 0, dispatch = 0; string server = "", path = ""; uint request_id = 0; if(!DPCCReadTerminalRequest(candidate, request, login, server, path, quote, dispatch, request_id, reason)) return(false); if(candidate.terminal_result_wire != "") { MqlTradeResult result; if(!DPCCReadTerminalResult(candidate, result, reason)) return(false); } } SDPCCExecutionReceipt decoded[]; if(ArrayResize(decoded, (int)count) != (int)count) return(false); for(int i = 0; i < (int)count; i++) { const int offset = 26 + (terminal ? 2 : 0) + 13 * i; SDPCCExecutionReceipt r; ZeroMemory(r); ulong kind = 0, receipt_status = 0, receipt_time = 0; if(!DPCCCheckpointReadUnsigned(fields[offset], ULONG_MAX, r.intent_id) || !DPCCCheckpointReadUnsigned(fields[offset + 1], ULONG_MAX, r.order_ticket) || !DPCCCheckpointReadUnsigned(fields[offset + 2], ULONG_MAX, r.deal_id) || !DPCCCheckpointReadUnsigned(fields[offset + 4], (ulong)LONG_MAX, receipt_time) || !DPCCCheckpointReadUnsigned(fields[offset + 8], DPCC_EXECUTION_DEAL_FILL, kind) || !DPCCCheckpointReadUnsigned(fields[offset + 9], DPCC_ORDER_CANCELED, receipt_status) || !DPCCPhysicalReadDouble(fields[offset + 10], r.signed_fill_lots) || !DPCCPhysicalReadDouble(fields[offset + 11], r.fill_price)) {reason = "execution_transcript.receipt_encoding"; return(false);} r.magic = StringToInteger(fields[offset + 3]); if((string)r.magic != fields[offset + 3]) return(false); r.time_msc = (long)receipt_time; r.deployment_id = fields[offset + 5]; r.symbol = fields[offset + 6]; r.intent_sha256 = fields[offset + 7]; r.kind = (ENUM_DPCC_EXECUTION_RECEIPT_KIND)kind; r.status = (ENUM_DPCC_EXECUTION_ORDER_STATE)receipt_status; r.canonical_digest = fields[offset + 12]; if(!DPCCApplyExecutionReceipt(candidate, r, reason)) return(false); if(candidate.receipt_count != (uint)i + 1) {reason = "execution_transcript.duplicate_receipt"; return(false);} decoded[i] = r; } if(candidate.order_ticket != ticket || candidate.order_state != (ENUM_DPCC_EXECUTION_ORDER_STATE)status || DPCCPhysicalDoubleWire(candidate.filled_signed_lots) != DPCCPhysicalDoubleWire(filled) || candidate.last_order_state_time_msc != (long)time) {reason = "execution_transcript.summary_mismatch"; return(false);} string rebuilt = ""; if(!DPCCExecutionTranscriptWire(candidate, decoded, owner_sha, mark_sha, rebuilt, reason) || rebuilt != wire) {if(reason == "")reason = "execution_transcript.noncanonical"; return(false);} if(ArrayResize(receipts, (int)count) != (int)count) return(false); for(int i = 0; i < (int)count; i++) receipts[i] = decoded[i]; intent = candidate; return(true); } //+------------------------------------------------------------------+ //| Join recovered execution to the exact owner and physical receipt | //+------------------------------------------------------------------+ bool DPCCOnlineExecutionRead(const SDPCCOnlineOwnerState &state, const string owner_sha, SDPCCExecutionIntent &intent, SDPCCExecutionReceipt &receipts[], string &reason) { if(state.execution_transcript_wire == "" || state.physical_mark_wire == "" || state.phase == 0) {reason = "online.execution.no_recoverable_transcript"; return(false);} SDPCCPhysicalOpenMark mark; string mark_sha = "", policy_sha = ""; if(!DPCCPhysicalReadOpenMark(state.physical_mark_wire, mark, reason) || !DPCCCheckpointTextSHA(state.physical_mark_wire, mark_sha, reason) || !DPCCPhysicalPolicySHA(mark.policy, policy_sha, reason)) return(false); SDPCCExecutionIntent candidate; SDPCCExecutionReceipt decoded[]; if(!DPCCExecutionTranscriptRead(state.execution_transcript_wire, owner_sha, mark_sha, candidate, decoded, reason)) return(false); if(candidate.decision_identity != state.decision_identity || state.decision_identity != (ulong)mark.clock.decision_bar_index + 1 || candidate.policy_sha256 != policy_sha || candidate.deployment_id != mark.policy.deployment_id || candidate.magic != mark.policy.magic || candidate.symbol != mark.policy.symbol || candidate.account_snapshot_sha256 != mark.snapshot_sha256) {reason = "online.execution.owner_physical_binding"; return(false);} for(int i = 0; i < 6; i++) if(candidate.actions[i] != (double)state.behavior[i]) {reason = "online.execution.behavior_binding"; return(false);} for(int i = 0; i < 13; i++) if(state.account13[i] != mark.account13[i]) {reason = "online.execution.account_binding"; return(false);} if(ArrayResize(receipts, ArraySize(decoded)) != ArraySize(decoded)) return(false); for(int i = 0; i < ArraySize(decoded); i++) receipts[i] = decoded[i]; intent = candidate; return(true); } //+------------------------------------------------------------------+ //| Profile digest includes every shape and caller/cadence field | //+------------------------------------------------------------------+ bool DPCCOnlineOwnerProfileSHA(const SDPCCOnlineOwnerProfile &profile, string &sha, string &reason) { SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(profile.shape, layout, reason)) return(false); //--- A fixed text cap bounds checkpoint restore allocations. if(layout.scenario_count > 16384 || profile.actor_every == 0 || profile.actor_target_every == 0 || profile.critic_target_every == 0 || profile.actor_critic_selector < 0 || profile.actor_critic_selector > 2 || !DPCCResolverLowerSha256(profile.offline_actor_sha256) || !DPCCResolverLowerSha256(profile.decoder_sha256) || !DPCCResolverLowerSha256(profile.caller_sha256) || !DPCCResolverLowerSha256(profile.sampler_sha256)) { reason = "online.owner.profile_binding"; return(false); } const float values[] = {profile.gamma, profile.probability_weight, profile.value_weight, profile.actor_upstream, profile.risk_parameter, profile.mean_weight, profile.target_tau }; for(int i = 0; i < ArraySize(values); i++) if(!MathIsValidNumber(values[i])) { reason = "online.owner.profile_nonfinite"; return(false); } const bool risk_valid = profile.risk_type == defSRM_MEAN || ((profile.risk_type == defSRM_CVAR || profile.risk_type == defSRM_MEAN_CVAR) && profile.risk_parameter > 0.0f && profile.risk_parameter <= 1.0f) || (profile.risk_type == defSRM_EXPONENTIAL && profile.risk_parameter >= 0.0f) || (profile.risk_type == defSRM_DUAL_POWER && profile.risk_parameter >= 1.0f); if(profile.target_tau < 0.0f || profile.target_tau > 1.0f || profile.gamma < 0 || profile.gamma > 1 || profile.probability_weight < 0 || profile.value_weight < 0 || !risk_valid || (profile.risk_type == defSRM_MEAN_CVAR && (profile.mean_weight < 0 || profile.mean_weight > 1))) { reason = "online.owner.profile_learning"; return(false); } string wire = (profile.physical_native_required ? "DPCC_ONLINE_OWNER_PROFILE_NATIVE_V3\n" : "DPCC_ONLINE_OWNER_PROFILE_V3\n"); const uint dimensions[] = {profile.shape.K, profile.shape.V, profile.shape.H, profile.shape.D, profile.shape.M, profile.shape.J, profile.shape.E, profile.shape.PlanBatch, profile.shape.NActions, profile.shape.StateSize, profile.shape.Embedding, profile.shape.Heads, profile.shape.KeyDimension, profile.shape.Depth, profile.actor_every, profile.actor_target_every, profile.critic_target_every }; for(int i = 0; i < ArraySize(dimensions); i++) DPCCOfflineCheckpointField(wire, (string)dimensions[i]); DPCCOfflineCheckpointField(wire, profile.offline_actor_sha256); DPCCOfflineCheckpointField(wire, profile.decoder_sha256); DPCCOfflineCheckpointField(wire, profile.caller_sha256); DPCCOfflineCheckpointField(wire, profile.sampler_sha256); for(int i = 0; i < ArraySize(values); i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(values[i])); DPCCOfflineCheckpointField(wire, (string)profile.risk_type); DPCCOfflineCheckpointField(wire, (string)profile.actor_critic_selector); DPCCOfflineCheckpointField(wire, (string)profile.selector_seed); return(DPCCCheckpointTextSHA(wire, sha, reason)); } //+------------------------------------------------------------------+ //| Reads a bounded checkpoint as exact UTF-8 text | //+------------------------------------------------------------------+ bool DPCCOnlineOwnerReadText(const string path, const bool common, string &wire, string &reason) { wire = ""; const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "online.owner.text_open"; return(false); } const ulong size = FileSize(handle); uchar bytes[]; if(size == 0 || size > 1048576 || ArrayResize(bytes, (int)size) != (int)size) { FileClose(handle); reason = "online.owner.text_size"; return(false); } const bool complete = FileReadArray(handle, bytes, 0, (uint)size) == (uint)size && FileTell(handle) == size && FileSize(handle) == size; FileClose(handle); if(!complete) { reason = "online.owner.text_read"; return(false); } wire = CharArrayToString(bytes, 0, (int)size, CP_UTF8); uchar roundtrip[]; if(StringToCharArray(wire, roundtrip, 0, WHOLE_ARRAY, CP_UTF8) != (int)size + 1) { reason = "online.owner.text_encoding"; return(false); } for(int i = 0; i < (int)size; i++) if(roundtrip[i] != bytes[i]) { reason = "online.owner.text_encoding"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Owns transition boundaries; caller owns all six native networks | //+------------------------------------------------------------------+ //--- Caller grants exclusive mutation access to this sextuple until the owner is discarded. //--- Keep the supplied CNet/context alive; invoke no external TrainMode, WeightsUpdate or backward. //--- Direct ParameterFingerprint is an additional guard, not a complete ACSRM child fingerprint. //--- Profile SHA values attest caller provenance; this owner does not acquire or validate live receipts. class CDPCCOnlineOwner { private: SDPCCOnlineOwnerProfile m_profile; SDPCCOnlineOwnerState m_state; CNet *m_nets[6]; ulong m_fingerprints[6]; string m_profile_sha, m_last_manifest, m_last_sha; bool m_initialized, m_failed, m_durable, m_last_common; bool m_trusted_restore, m_replay_pending, m_replay_observation_bound, m_checkpointable; ulong m_replay_identity, m_resume_decision_identity; long m_resume_server_open; float m_replay_scenarios[], m_replay_account13[13], m_replay_behavior[6]; int m_checkpoint_lock; void ClearOwnerState(SDPCCOnlineOwnerState &state) { ZeroMemory(state); state.physical_mark_wire = ""; state.physical_next_wire = ""; state.execution_transcript_wire = ""; } //+------------------------------------------------------------------+ //| ClearReplayObservation releases the phase-zero runtime binding | //+------------------------------------------------------------------+ void ClearReplayObservation(void) { m_replay_observation_bound = false; ArrayFree(m_replay_scenarios); ArrayInitialize(m_replay_account13, 0.0f); ArrayInitialize(m_replay_behavior, 0.0f); } void CloseCheckpointLock(void) { if(m_checkpoint_lock != INVALID_HANDLE) FileClose(m_checkpoint_lock); m_checkpoint_lock = INVALID_HANDLE; } bool Fail(const string failure, string &reason) { CloseCheckpointLock(); m_failed = true; m_durable = false; m_checkpointable = false; reason = failure; return(false); } bool Boundary(string &reason, const bool allow_restore_replay = false) { if(m_replay_pending) { reason = "online.owner.replay_rebind_required"; return(false); } if(!m_initialized || m_failed) { reason = "online.owner.poisoned_or_unbound"; return(false); } if(m_trusted_restore && !allow_restore_replay) { reason = "online.owner.replay_begin_required"; return(false); } //--- Simulated observations are volatile; only completed model updates need publication. const bool volatile_simulated = !m_profile.physical_native_required && !m_checkpointable && m_state.physical_mark_wire == "" && m_state.physical_next_wire == "" && m_state.execution_transcript_wire == ""; if(!m_durable && !volatile_simulated) { reason = "online.owner.checkpoint_required"; return(false); } return(CheckNets(reason)); } bool ExecutionSettled(string &reason) { if(m_state.execution_transcript_wire == "") return(true); SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason)) return(false); if(intent.order_state == DPCC_ORDER_PENDING || intent.order_state == DPCC_ORDER_PARTIAL || intent.order_state == DPCC_ORDER_AMBIGUOUS) {reason = "online.execution.reconciliation_required"; return(false);} return(true); } bool CheckNets(string &reason) { for(int i = 0; i < 6; i++) if(CheckPointer(m_nets[i]) == POINTER_INVALID) return(Fail("online.owner.network_lifetime", reason)); if(!DPCCPolicyTuplePreflight(*m_nets[0], *m_nets[1], *m_nets[2], *m_nets[3], *m_nets[4], *m_nets[5], m_profile.shape, reason)) return(Fail(reason, reason)); for(int i = 0; i < 6; i++) { const bool training = i < 3; if(m_nets[i].WeightsUpdateEnabled() != training) { reason = "online.owner.network_permission"; return(false); } const int depth = (i == 0 || i == 3 ? 3 : 4); for(int l = 0; l < depth; l++) if(m_nets[i].Layer(l).TrainMode() != training) { reason = "online.owner.network_mode"; return(false); } } return(true); } bool Observation(const float &scenarios[], const float &account[], string &reason) { SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(m_profile.shape, layout, reason)) return(false); if(ArraySize(scenarios) != (int)layout.scenario_count || ArraySize(account) != 13) { reason = "online.owner.observation_shape"; return(false); } for(int i = 0; i < ArraySize(scenarios); i++) if(!MathIsValidNumber(scenarios[i])) { reason = "online.owner.scenario_nonfinite"; return(false); } for(int i = 0; i < 13; i++) if(!MathIsValidNumber(account[i])) { reason = "online.owner.account_nonfinite"; return(false); } return(true); } bool ValidateState(const SDPCCOnlineOwnerState &state, string &reason) { if(!Observation(state.scenarios, state.account13, reason) || !Observation(state.next_scenarios, state.next_account13, reason)) return(false); if(state.phase > 3 || state.updates >= UINT_MAX || state.actor_updates != state.updates / m_profile.actor_every || state.actor_target_updates != state.updates / m_profile.actor_target_every || state.critic_target_updates != state.updates / m_profile.critic_target_every || state.sampler_cursor != state.updates + (state.phase == 2 ? 1 : 0)) { reason = "online.owner.state_counters"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(state.behavior[i]) || state.behavior[i] < 0 || state.behavior[i] > 1) { reason = "online.owner.behavior_range"; return(false); } if(state.phase == 0) { if(state.decision_identity != 0 || state.next_decision_identity != 0 || state.reward != 0 || state.bootstrap_mask != 0 || state.sample_u1 != 0 || state.sample_u2 != 0) { reason = "online.owner.initial_state"; return(false); } } else if(state.decision_identity == 0 || (state.phase >= 2 && state.next_decision_identity <= state.decision_identity) || (state.phase == 3 && state.updates == 0)) { reason = "online.owner.decision_identity"; return(false); } if(StringLen(state.physical_mark_wire) > 0) { SDPCCPhysicalOpenMark mark, next; if(!DPCCPhysicalReadOpenMark(state.physical_mark_wire, mark, reason) || state.phase == 0 || (m_profile.physical_native_required && !mark.clock.native_observed) || state.decision_identity != (ulong)mark.clock.decision_bar_index + 1 || mark.ledger.terminal) return(DPCCPhysicalRefuse("owner.pending_mark", reason)); for(int i = 0; i < 13; i++) if(state.account13[i] != mark.account13[i]) return(DPCCPhysicalRefuse("owner.pending_account13", reason)); if(state.phase >= 2) { float actual_reward = 0, actual_mask = 0; if(!DPCCPhysicalReadOpenMark(state.physical_next_wire, next, reason) || !DPCCPhysicalActualReward(mark, next, actual_reward, actual_mask, reason) || state.next_decision_identity != (ulong)next.clock.decision_bar_index + 1 || state.reward != actual_reward || state.bootstrap_mask != actual_mask) return(DPCCPhysicalRefuse("owner.resolved_mark", reason)); for(int i = 0; i < 13; i++) if(state.next_account13[i] != next.account13[i]) return(DPCCPhysicalRefuse("owner.resolved_account13", reason)); } else if(StringLen(state.physical_next_wire) > 0) return(DPCCPhysicalRefuse("owner.unresolved_mark", reason)); } else if(StringLen(state.physical_next_wire) > 0 || (m_profile.physical_native_required && state.phase != 0)) return(DPCCPhysicalRefuse("owner.missing_mark", reason)); if(state.execution_transcript_wire != "") { SDPCCExecutionIntent execution; SDPCCExecutionReceipt receipts[]; if(!DPCCOnlineExecutionRead(state, m_profile_sha, execution, receipts, reason)) return(false); if(state.phase >= 2 && (execution.order_state == DPCC_ORDER_PENDING || execution.order_state == DPCC_ORDER_PARTIAL || execution.order_state == DPCC_ORDER_AMBIGUOUS)) {reason = "online.execution.reconciliation_required"; return(false);} } float y1 = 0, y2 = 0; if(!DPCCBuildOneBarTDTargets(state.reward, m_profile.gamma, state.bootstrap_mask, 0, 0, y1, y2, reason)) return(false); if(!MathIsValidNumber(state.sample_u1) || !MathIsValidNumber(state.sample_u2) || (state.phase >= 2 && state.bootstrap_mask == 1 && (state.sample_u1 <= 0 || state.sample_u1 >= 1 || state.sample_u2 <= 0 || state.sample_u2 >= 1))) { reason = "online.owner.state_sampler"; return(false); } if(state.phase <= 1) { if(state.next_decision_identity != 0 || state.reward != 0 || state.bootstrap_mask != 0 || state.sample_u1 != 0 || state.sample_u2 != 0) { reason = "online.owner.unresolved_state"; return(false); } for(int i = 0; i < ArraySize(state.next_scenarios); i++) if(state.next_scenarios[i] != 0) { reason = "online.owner.unresolved_next"; return(false); } for(int i = 0; i < 13; i++) if(state.next_account13[i] != 0) { reason = "online.owner.unresolved_next"; return(false); } } return(true); } bool StateWire(const SDPCCOnlineOwnerState &state, string &wire, string &reason) { if(!ValidateState(state, reason)) return(false); const bool physical = StringLen(state.physical_mark_wire) > 0; const bool execution = state.execution_transcript_wire != ""; const bool terminal = StringFind(state.execution_transcript_wire, "DPCC_EXECUTION_TRANSCRIPT_TERMINAL_V2|") == 0; wire = execution ? (terminal ? "DPCC_ONLINE_STATE_TERMINAL_V4\n" : "DPCC_ONLINE_STATE_EXECUTION_V3\n") : (physical ? "DPCC_ONLINE_STATE_PHYSICAL_V2\n" : "DPCC_ONLINE_STATE_V1\n"); DPCCOfflineCheckpointField(wire, m_profile_sha); const ulong counters[] = {state.phase, state.updates, state.actor_updates, state.actor_target_updates, state.critic_target_updates, state.sampler_cursor, state.decision_identity, state.next_decision_identity }; for(int i = 0; i < ArraySize(counters); i++) DPCCOfflineCheckpointField(wire, (string)counters[i]); const float scalars[] = {state.reward, state.bootstrap_mask, state.sample_u1, state.sample_u2}; for(int i = 0; i < 4; i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(scalars[i])); for(int i = 0; i < 13; i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(state.account13[i])); for(int i = 0; i < 13; i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(state.next_account13[i])); for(int i = 0; i < 6; i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(state.behavior[i])); for(int i = 0; i < ArraySize(state.scenarios); i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(state.scenarios[i])); for(int i = 0; i < ArraySize(state.next_scenarios); i++) DPCCOfflineCheckpointField(wire, DPCCOfflineCheckpointFloat(state.next_scenarios[i])); if(physical) { DPCCOfflineCheckpointField(wire, state.physical_mark_wire); DPCCOfflineCheckpointField(wire, state.physical_next_wire); } if(execution) DPCCOfflineCheckpointField(wire, state.execution_transcript_wire); DPCCOfflineCheckpointField(wire, execution ? (terminal ? "END_DPCC_ONLINE_STATE_TERMINAL_V4" : "END_DPCC_ONLINE_STATE_EXECUTION_V3") : (physical ? "END_DPCC_ONLINE_STATE_PHYSICAL_V2" : "END_DPCC_ONLINE_STATE_V1")); return(true); } bool ParseState(const string wire, SDPCCOnlineOwnerState &state, string &reason) { SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(m_profile.shape, layout, reason)) return(false); string fields[]; const int count = StringSplit(wire, '\n', fields); const bool terminal = count > 0 && fields[0] == "DPCC_ONLINE_STATE_TERMINAL_V4"; const bool execution = terminal || (count > 0 && fields[0] == "DPCC_ONLINE_STATE_EXECUTION_V3"); const bool physical = execution || (count > 0 && fields[0] == "DPCC_ONLINE_STATE_PHYSICAL_V2"); const string schema = execution ? (terminal ? "DPCC_ONLINE_STATE_TERMINAL_V4" : "DPCC_ONLINE_STATE_EXECUTION_V3") : (physical ? "DPCC_ONLINE_STATE_PHYSICAL_V2" : "DPCC_ONLINE_STATE_V1"); if(count != 48 + 2 * (int)layout.scenario_count + (physical ? 2 : 0) + (execution ? 1 : 0) || fields[0] != schema || fields[1] != m_profile_sha || fields[count - 2] != "END_" + schema || fields[count - 1] != "") { reason = "online.owner.state_wire"; return(false); } ClearOwnerState(state); ulong counters[8]; for(int i = 0; i < 8; i++) if(!DPCCCheckpointReadUnsigned(fields[2 + i], (ulong)LONG_MAX, counters[i])) { reason = "online.owner.state_integer"; return(false); } state.phase = (uint)counters[0]; if(counters[0] > 3) return(false); state.updates = counters[1]; state.actor_updates = counters[2]; state.actor_target_updates = counters[3]; state.critic_target_updates = counters[4]; state.sampler_cursor = counters[5]; state.decision_identity = counters[6]; state.next_decision_identity = counters[7]; float scalars[4]; for(int i = 0; i < 4; i++) if(!DPCCOfflineCheckpointReadFloat(fields[10 + i], scalars[i])) return(false); state.reward = scalars[0]; state.bootstrap_mask = scalars[1]; state.sample_u1 = scalars[2]; state.sample_u2 = scalars[3]; for(int i = 0; i < 13; i++) if(!DPCCOfflineCheckpointReadFloat(fields[14 + i], state.account13[i])) return(false); for(int i = 0; i < 13; i++) if(!DPCCOfflineCheckpointReadFloat(fields[27 + i], state.next_account13[i])) return(false); for(int i = 0; i < 6; i++) if(!DPCCOfflineCheckpointReadFloat(fields[40 + i], state.behavior[i])) return(false); if(ArrayResize(state.scenarios, (int)layout.scenario_count) != (int)layout.scenario_count || ArrayResize(state.next_scenarios, (int)layout.scenario_count) != (int)layout.scenario_count) return(false); for(int i = 0; i < (int)layout.scenario_count; i++) if(!DPCCOfflineCheckpointReadFloat(fields[46 + i], state.scenarios[i]) || !DPCCOfflineCheckpointReadFloat(fields[46 + (int)layout.scenario_count + i], state.next_scenarios[i])) return(false); if(physical) { state.physical_mark_wire = fields[count - 4 - (execution ? 1 : 0)]; state.physical_next_wire = fields[count - 3 - (execution ? 1 : 0)]; } //--- V1/V2 are readable observations and explicitly carry no recoverable execution. if(execution) { state.execution_transcript_wire = fields[count - 3]; if(state.execution_transcript_wire == "") {reason = "online.execution.empty_v3_transcript"; return(false);} } string rebuilt = ""; if(!StateWire(state, rebuilt, reason) || rebuilt != wire) { if(reason == "") reason = "online.owner.state_noncanonical"; return(false); } return(true); } bool LoadCandidate(const string path, const bool common, const int index, COpenCLMy *context, const ulong fingerprint, CNet *&candidate, string &reason) { candidate = new CNet; if(CheckPointer(candidate) == POINTER_INVALID) { reason = "online.owner.candidate_allocation"; return(false); } const bool critic = index != 0 && index != 3; if(!DPCCCreatePolicyNet(*candidate, m_profile.shape, critic, context, reason)) return(false); const int handle = FileOpen(path, FILE_BIN | FILE_READ | FILE_SHARE_READ | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "online.owner.candidate_open"; return(false); } const bool loaded = candidate.LoadDPCCPolicyTrainingStateOnly(handle) && FileTell(handle) == FileSize(handle); FileClose(handle); if(!loaded) { reason = "online.owner.candidate_parameters"; return(false); } //--- Runtime permissions are restored explicitly; targets are never re-copied. if(!candidate.TrainMode(index < 3) || !candidate.SetWeightsUpdate(index < 3) || !DPCCValidatePolicyNet(*candidate, m_profile.shape, critic, reason)) return(false); ulong actual = 0; if(!candidate.DPCCPolicyTrainingStateFingerprint(actual) || actual != fingerprint) { reason = "online.owner.candidate_fingerprint"; return(false); } return(true); } void DeleteCandidates(CNet *&candidates[]) { for(int i = 0; i < ArraySize(candidates); i++) { if(CheckPointer(candidates[i]) == POINTER_DYNAMIC) delete candidates[i]; candidates[i] = NULL; } } public: CDPCCOnlineOwner(void) { m_initialized = false; m_failed = false; m_durable = false; m_last_common = false; m_trusted_restore = false; m_replay_pending = false; m_replay_identity = 0; m_resume_decision_identity = 0; m_resume_server_open = 0; m_checkpointable = false; ClearReplayObservation(); m_checkpoint_lock = INVALID_HANDLE; for(int i = 0; i < 6; i++) { m_nets[i] = NULL; m_fingerprints[i] = 0; } ClearOwnerState(m_state); } ~CDPCCOnlineOwner(void) { CloseCheckpointLock(); } bool Poisoned(void) const { return(m_failed); } //--- Durable identifies an admitted model boundary; reconstructed finances remain volatile. bool Durable(void) const { return(m_initialized && !m_failed && m_durable && !m_trusted_restore && !m_replay_pending); } bool Snapshot(SDPCCOnlineOwnerState &state) { ClearOwnerState(state); if(!m_initialized || m_failed || m_replay_pending) return(false); state = m_state; return(true); } //+------------------------------------------------------------------+ //| BeginHistoricalReplay binds restart to restored model parameters | //+------------------------------------------------------------------+ bool BeginHistoricalReplay(const ulong start_decision_identity, ulong &replay_decision_identity, long &replay_server_open, string &reason) { replay_decision_identity = 0; replay_server_open = 0; reason = ""; //--- Only an authenticated simulated checkpoint may enter replay. if(!Boundary(reason, true)) return(false); if(!m_trusted_restore || m_profile.physical_native_required || m_state.physical_mark_wire != "" || m_state.physical_next_wire != "" || m_state.execution_transcript_wire != "") { reason = "online.owner.replay_restore_boundary"; return(false); } if(!ValidateState(m_state, reason)) return(false); const ulong identity = (m_state.phase == 0 ? (m_resume_decision_identity > 0 ? m_resume_decision_identity : start_decision_identity) : (m_state.phase == 3 ? m_state.next_decision_identity : m_state.decision_identity)); if(identity == 0 || identity > (ulong)LONG_MAX || m_state.decision_identity > (ulong)LONG_MAX || m_state.next_decision_identity > (ulong)LONG_MAX) { reason = "online.owner.replay_identity"; return(false); } if((m_resume_decision_identity == 0) != (m_resume_server_open == 0) || (m_resume_decision_identity > 0 && (identity != m_resume_decision_identity || m_resume_server_open <= 0))) { reason = "online.owner.replay_clock"; return(false); } if(m_state.phase == 3 && m_state.bootstrap_mask == 0) { reason = "online.owner.replay_terminal"; return(false); } //--- Replayed history is external to the owner and must not update its six networks. m_replay_identity = identity; m_replay_pending = true; m_trusted_restore = false; m_durable = false; m_checkpointable = false; replay_decision_identity = identity; replay_server_open = m_resume_server_open; return(true); } bool BeginHistoricalReplay(const ulong start_decision_identity, ulong &replay_decision_identity, string &reason) { long replay_server_open = 0; return(BeginHistoricalReplay(start_decision_identity, replay_decision_identity, replay_server_open, reason)); } //+------------------------------------------------------------------+ //| RebindHistoricalReplay publishes the reconstructed observation | //+------------------------------------------------------------------+ bool RebindHistoricalReplay(const float &scenarios[], const float &account[], const float &behavior[], const ulong decision_identity, const long server_open, string &reason) { reason = ""; if(!m_initialized || m_failed || !m_replay_pending || decision_identity != m_replay_identity || m_profile.physical_native_required || m_state.physical_mark_wire != "" || m_state.physical_next_wire != "" || m_state.execution_transcript_wire != "") { reason = "online.owner.replay_rebind_boundary"; return(false); } if((m_resume_decision_identity == 0 && server_open != 0) || (m_resume_decision_identity > 0 && (decision_identity != m_resume_decision_identity || server_open != m_resume_server_open))) { reason = "online.owner.replay_clock_mismatch"; return(false); } if(!CheckNets(reason) || !ValidateState(m_state, reason) || !Observation(scenarios, account, reason)) return(false); if(m_state.phase != 3) { if(ArraySize(behavior) != 6) { reason = "online.owner.replay_behavior_shape"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(behavior[i]) || behavior[i] < 0 || behavior[i] > 1) { reason = "online.owner.behavior_range"; return(false); } } SDPCCOnlineOwnerState staged = m_state; //--- Applied updates retain their canonical TD metadata; Record consumes the new next observation. if(staged.phase == 3) { if(ArrayCopy(staged.next_scenarios, scenarios) != ArraySize(scenarios)) { reason = "online.owner.replay_capacity"; return(false); } ArrayCopy(staged.next_account13, account); staged.next_decision_identity = decision_identity; } else if(staged.phase != 0) { if(ArrayCopy(staged.scenarios, scenarios) != ArraySize(scenarios)) { reason = "online.owner.replay_capacity"; return(false); } ArrayCopy(staged.account13, account); ArrayCopy(staged.behavior, behavior); ArrayInitialize(staged.next_scenarios, 0.0f); ArrayInitialize(staged.next_account13, 0.0f); staged.phase = 1; staged.decision_identity = decision_identity; staged.next_decision_identity = 0; staged.sampler_cursor = staged.updates; staged.reward = 0; staged.bootstrap_mask = 0; staged.sample_u1 = 0; staged.sample_u2 = 0; } if(!ValidateState(staged, reason)) return(false); //--- Phase zero remains canonical; its first observation is bound only in this runtime. if(staged.phase == 0) { if(ArrayCopy(m_replay_scenarios, scenarios) != ArraySize(scenarios)) { reason = "online.owner.replay_capacity"; return(false); } ArrayCopy(m_replay_account13, account); ArrayCopy(m_replay_behavior, behavior); m_replay_observation_bound = true; } //--- Admit the restored model after replay; reconstructed observations are never serialized. m_state = staged; m_replay_pending = false; m_durable = true; m_checkpointable = false; return(true); } bool RebindHistoricalReplay(const float &scenarios[], const float &account[], const float &behavior[], const ulong decision_identity, string &reason) { return(RebindHistoricalReplay(scenarios, account, behavior, decision_identity, 0, reason)); } //--- No sender is exposed. A recovered intent can never be prepared again for the same decision. bool ExecutionResubmissionBlocked(void) const { return(m_state.execution_transcript_wire != ""); } bool ReadExecutionIntent(SDPCCExecutionIntent &intent, SDPCCExecutionReceipt &receipts[], string &reason) { if(!m_initialized || m_failed || !m_durable) {reason = "online.execution.durable_owner_required"; return(false);} return(DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason)); } bool PrepareExecutionIntent(const ulong intent_id, const double current_signed_lots, const double requested_signed_delta_lots, const double target_signed_lots, string &reason) { if(!Boundary(reason)) return(false); if(m_state.phase != 1 || m_state.execution_transcript_wire != "" || m_state.physical_mark_wire == "" || !DPCCResolverLowerSha256(m_last_sha)) {reason = "online.execution.prepare_boundary"; return(false);} SDPCCPhysicalOpenMark mark; string policy_sha = "", mark_sha = "", wire = ""; if(!DPCCPhysicalReadOpenMark(m_state.physical_mark_wire, mark, reason) || !DPCCPhysicalPolicySHA(mark.policy, policy_sha, reason) || !DPCCCheckpointTextSHA(m_state.physical_mark_wire, mark_sha, reason)) return(false); double actions[6]; for(int i = 0; i < 6; i++) actions[i] = (double)m_state.behavior[i]; SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[]; //--- Provenance is the already durable predecessor, never the new enclosing checkpoint SHA. if(!DPCCPrepareExecutionIntent(intent_id, m_state.decision_identity, mark.policy.deployment_id, mark.policy.magic, mark.policy.symbol, policy_sha, m_last_sha, mark.snapshot_sha256, current_signed_lots, requested_signed_delta_lots, target_signed_lots, actions, intent, reason) || !DPCCExecutionTranscriptWire(intent, receipts, m_profile_sha, mark_sha, wire, reason)) return(false); SDPCCOnlineOwnerState staged = m_state; staged.execution_transcript_wire = wire; if(!ValidateState(staged, reason)) return(false); m_state = staged; m_durable = false; return(true); } bool ApplyExecutionReceipt(const SDPCCExecutionReceipt &receipt, string &reason) { if(!Boundary(reason)) return(false); if(m_state.phase != 1) {reason = "online.execution.receipt_boundary"; return(false);} SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason)) return(false); const uint before = intent.receipt_count; if(!DPCCApplyExecutionReceipt(intent, receipt, reason)) return(false); if(intent.receipt_count == before) {reason = "online.execution.duplicate_receipt"; return(false);} if(ArrayResize(receipts, (int)intent.receipt_count) != (int)intent.receipt_count) return(false); receipts[(int)before] = receipt; string mark_sha = "", wire = ""; if(!DPCCCheckpointTextSHA(m_state.physical_mark_wire, mark_sha, reason) || !DPCCExecutionTranscriptWire(intent, receipts, m_profile_sha, mark_sha, wire, reason)) return(false); SDPCCOnlineOwnerState staged = m_state; staged.execution_transcript_wire = wire; if(!ValidateState(staged, reason)) return(false); m_state = staged; m_durable = false; return(true); } //+------------------------------------------------------------------+ //| Persist a bound native request before any future dispatch | //+------------------------------------------------------------------+ bool BindTerminalRequest(const MqlTradeRequest &request, const long quote_time_msc, const long dispatch_time_msc, string &reason) { if(!Boundary(reason)) return(false); if(m_state.phase != 1) {reason = "online.terminal.prepare_phase"; return(false);} MqlTick quote; if(!SymbolInfoTick(request.symbol, quote) || quote.time_msc != quote_time_msc) {reason = "online.terminal.quote_identity"; return(false);} if(request.position != 0 && (!PositionSelectByTicket(request.position) || PositionGetString(POSITION_SYMBOL) != request.symbol || PositionGetInteger( POSITION_MAGIC) != (long)request.magic)) {reason = "online.terminal.position_ownership"; return(false);} SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason) || !DPCCBindTerminalRequest(intent, request, AccountInfoInteger(ACCOUNT_LOGIN), AccountInfoString(ACCOUNT_SERVER), TerminalInfoString(TERMINAL_DATA_PATH), quote_time_msc, dispatch_time_msc, reason)) return(false); string mark_sha = "", wire = ""; if(!DPCCCheckpointTextSHA(m_state.physical_mark_wire, mark_sha, reason) || !DPCCExecutionTranscriptWire(intent, receipts, m_profile_sha, mark_sha, wire, reason)) return(false); SDPCCOnlineOwnerState staged = m_state; staged.execution_transcript_wire = wire; if(!ValidateState(staged, reason)) return(false); m_state = staged; m_durable = false; return(true); } private: //+------------------------------------------------------------------+ //| Publish one atomic observed batch; each mutation needs a save | //+------------------------------------------------------------------+ bool PublishTerminalEvidence(SDPCCExecutionIntent &intent, const SDPCCExecutionReceipt &batch[], string &reason) { SDPCCExecutionIntent original; SDPCCExecutionReceipt receipts[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, original, receipts, reason)) return(false); for(int i = 0; i < ArraySize(batch); i++) { const uint before = intent.receipt_count; if(!DPCCApplyExecutionReceipt(intent, batch[i], reason)) return(false); if(intent.receipt_count == before) continue; const int n = ArraySize(receipts); if(ArrayResize(receipts, n + 1) != n + 1) return(false); receipts[n] = batch[i]; } if(intent.receipt_count == original.receipt_count && intent.terminal_binding_wire == original.terminal_binding_wire && intent.terminal_result_wire == original.terminal_result_wire) return(true); string mark_sha = "", wire = ""; if(!DPCCCheckpointTextSHA(m_state.physical_mark_wire, mark_sha, reason) || !DPCCExecutionTranscriptWire(intent, receipts, m_profile_sha, mark_sha, wire, reason)) return(false); SDPCCOnlineOwnerState staged = m_state; staged.execution_transcript_wire = wire; if(!ValidateState(staged, reason)) return(false); m_state = staged; m_durable = false; return(true); } public: bool ObserveTerminalResult(const MqlTradeRequest &request, const MqlTradeResult &result, const long observed_time_msc, string &reason) { if(!Boundary(reason)) return(false); SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[], batch[]; MqlTradeRequest stored; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason) || !DPCCExecutionTerminalLiveBinding(intent, stored, reason)) return(false); if(ArrayResize(batch, 1) != 1) return(false); if(!DPCCExecutionTerminalResult(intent, request, result, observed_time_msc, batch[0], reason)) return(false); return(PublishTerminalEvidence(intent, batch, reason)); } bool ObserveTerminalTransaction(const MqlTradeTransaction &transaction, const MqlTradeRequest &request, const MqlTradeResult &result, const long observed_time_msc, string &reason) { if(!Boundary(reason)) return(false); SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[], batch[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason) || !DPCCExecutionTerminalTransaction(intent, transaction, request, result, observed_time_msc, batch, reason)) return(false); return(PublishTerminalEvidence(intent, batch, reason)); } bool ReconcileTerminalHistory(string &reason) { if(!Boundary(reason)) return(false); SDPCCExecutionIntent intent; SDPCCExecutionReceipt receipts[], batch[]; if(!DPCCOnlineExecutionRead(m_state, m_profile_sha, intent, receipts, reason) || !DPCCExecutionTerminalReconcile(intent, batch, reason)) return(false); return(PublishTerminalEvidence(intent, batch, reason)); } bool Initialize(const SDPCCOnlineOwnerProfile &profile, CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, string &reason) { reason = ""; if(m_initialized || m_failed) { reason = "online.owner.initialize_requires_fresh_owner"; return(false); } string profile_sha = ""; if(!DPCCOnlineOwnerProfileSHA(profile, profile_sha, reason) || !DPCCPolicyTuplePreflight(actor, q1, q2, target_actor, target_q1, target_q2, profile.shape, reason)) return(false); CNet *nets[] = {GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; for(int i = 0; i < 6; i++) m_nets[i] = nets[i]; m_profile = profile; m_profile_sha = profile_sha; SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(profile.shape, layout, reason) || ArrayResize(m_state.scenarios, (int)layout.scenario_count) != (int)layout.scenario_count || ArrayResize(m_state.next_scenarios, (int)layout.scenario_count) != (int)layout.scenario_count) return(Fail("online.owner.initial_capacity", reason)); ArrayInitialize(m_state.scenarios, 0.0f); ArrayInitialize(m_state.next_scenarios, 0.0f); m_state.phase = 0; m_state.sampler_cursor = 0; m_resume_decision_identity = 0; m_resume_server_open = 0; if(!CheckNets(reason)) return(false); m_initialized = true; m_checkpointable = true; return(true); } //+------------------------------------------------------------------+ //| Bind the durable model clock to the completed native update | //+------------------------------------------------------------------+ bool SetResumeCursor(const ulong decision_identity, const long server_open, string &reason) { reason = ""; if(!m_initialized || m_failed || m_replay_pending || !m_checkpointable || m_state.phase != 3 || decision_identity == 0 || decision_identity > (ulong)LONG_MAX || server_open <= 0 || decision_identity != m_state.next_decision_identity || (m_resume_decision_identity > 0 && (decision_identity <= m_resume_decision_identity || server_open <= m_resume_server_open))) { reason = "online.owner.resume_cursor_boundary"; return(false); } m_resume_decision_identity = decision_identity; m_resume_server_open = server_open; return(true); } private: bool RecordBound(const float &scenarios[], const float &account[], const float &behavior[], const ulong decision_identity, string &reason, const string physical_wire = "") { reason = ""; if(!Boundary(reason)) return(false); if((m_state.phase != 0 && m_state.phase != 3) || (m_state.phase == 3 && m_state.bootstrap_mask == 0) || decision_identity == 0 || decision_identity > (ulong)LONG_MAX || (m_state.phase == 3 && decision_identity != m_state.next_decision_identity) || (m_state.phase == 0 && m_replay_identity != 0 && decision_identity != m_replay_identity) || ArraySize(behavior) != 6 || !Observation(scenarios, account, reason)) { if(reason == "") reason = "online.owner.record_boundary"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(behavior[i]) || behavior[i] < 0 || behavior[i] > 1) { reason = "online.owner.behavior_range"; return(false); } if(m_state.phase == 0 && m_replay_observation_bound) { if(physical_wire != "" || ArraySize(m_replay_scenarios) != ArraySize(scenarios)) { reason = "online.owner.replay_observation_binding"; return(false); } for(int i = 0; i < ArraySize(scenarios); i++) if(scenarios[i] != m_replay_scenarios[i]) { reason = "online.owner.replay_observation_binding"; return(false); } for(int i = 0; i < 13; i++) if(account[i] != m_replay_account13[i]) { reason = "online.owner.replay_account_binding"; return(false); } for(int i = 0; i < 6; i++) if(behavior[i] != m_replay_behavior[i]) { reason = "online.owner.replay_behavior_binding"; return(false); } } if(m_state.phase == 3) { for(int i = 0; i < ArraySize(scenarios); i++) if(scenarios[i] != m_state.next_scenarios[i]) { reason = "online.owner.observation_chain"; return(false); } for(int i = 0; i < 13; i++) if(account[i] != m_state.next_account13[i]) { reason = "online.owner.account_chain"; return(false); } } SDPCCPhysicalOpenMark physical_mark; if(physical_wire != "") { if(!DPCCPhysicalReadOpenMark(physical_wire, physical_mark, reason) || physical_mark.ledger.terminal || (m_profile.physical_native_required && !physical_mark.clock.native_observed) || decision_identity != (ulong)physical_mark.clock.decision_bar_index + 1 || (m_state.phase == 3 && physical_wire != m_state.physical_next_wire)) return(DPCCPhysicalRefuse("owner.record_mark", reason)); for(int i = 0; i < 13; i++) if(account[i] != physical_mark.account13[i]) return(DPCCPhysicalRefuse("owner.record_account13", reason)); } else if(m_profile.physical_native_required || StringLen(m_state.physical_mark_wire) > 0) return(DPCCPhysicalRefuse("owner.record_requires_mark", reason)); if(!ExecutionSettled(reason)) return(false); SDPCCOnlineOwnerState staged = m_state; staged.execution_transcript_wire = ""; staged.physical_mark_wire = physical_wire; staged.physical_next_wire = ""; if(ArrayCopy(staged.scenarios, scenarios) != ArraySize(scenarios)) return(Fail("online.owner.record_capacity", reason)); ArrayCopy(staged.account13, account); ArrayCopy(staged.behavior, behavior); ArrayInitialize(staged.next_scenarios, 0.0f); ArrayInitialize(staged.next_account13, 0.0f); staged.phase = 1; staged.decision_identity = decision_identity; staged.next_decision_identity = 0; staged.reward = 0; staged.bootstrap_mask = 0; staged.sample_u1 = 0; staged.sample_u2 = 0; m_state = staged; m_durable = false; m_checkpointable = false; ClearReplayObservation(); return(true); } bool ResolveBound(const float &next_scenarios[], const float &next_account[], const ulong next_decision_identity, const float reward, const float bootstrap_mask, const float u1, const float u2, const ulong sampler_cursor, string &reason, const string physical_next_wire = "") { reason = ""; if(!Boundary(reason)) return(false); if(!ExecutionSettled(reason)) return(false); if(m_state.phase != 1 || next_decision_identity <= m_state.decision_identity || next_decision_identity > (ulong)LONG_MAX || sampler_cursor != m_state.updates + 1 || !Observation(next_scenarios, next_account, reason)) { if(reason == "") reason = "online.owner.resolve_boundary"; return(false); } if(StringLen(m_state.physical_mark_wire) > 0) { SDPCCPhysicalOpenMark mark, next; float actual_reward = 0, actual_mask = 0; if(!DPCCPhysicalReadOpenMark(m_state.physical_mark_wire, mark, reason) || !DPCCPhysicalReadOpenMark(physical_next_wire, next, reason) || !DPCCPhysicalActualReward(mark, next, actual_reward, actual_mask, reason) || next_decision_identity != (ulong)next.clock.decision_bar_index + 1 || reward != actual_reward || bootstrap_mask != actual_mask) return(DPCCPhysicalRefuse("owner.resolve_mark", reason)); for(int i = 0; i < 13; i++) if(next_account[i] != next.account13[i]) return(DPCCPhysicalRefuse("owner.resolve_account13", reason)); } else if(physical_next_wire != "") return(DPCCPhysicalRefuse("owner.resolve_without_pending_mark", reason)); SDPCCOnlineOwnerState staged = m_state; staged.physical_next_wire = physical_next_wire; staged.phase = 2; staged.next_decision_identity = next_decision_identity; staged.sampler_cursor = sampler_cursor; staged.reward = reward; staged.bootstrap_mask = bootstrap_mask; staged.sample_u1 = u1; staged.sample_u2 = u2; if(ArrayCopy(staged.next_scenarios, next_scenarios) != ArraySize(next_scenarios)) return(Fail("online.owner.resolve_capacity", reason)); ArrayCopy(staged.next_account13, next_account); if(!ValidateState(staged, reason)) return(false); m_state = staged; m_durable = false; m_checkpointable = false; return(true); } public: bool Record(const float &scenarios[], const float &account[], const float &behavior[], const ulong decision_identity, string &reason) { return(RecordBound(scenarios, account, behavior, decision_identity, reason)); } bool Resolve(const float &next_scenarios[], const float &next_account[], const ulong next_decision_identity, const float reward, const float bootstrap_mask, const float u1, const float u2, const ulong sampler_cursor, string &reason) { return(ResolveBound(next_scenarios, next_account, next_decision_identity, reward, bootstrap_mask, u1, u2, sampler_cursor, reason)); } //--- Resolve only a successfully committed historical row that reached the native financial floor. bool ResolveHistoricalFloor(const SDPCCHistoricalCommittedBar &closed, string &reason) { reason = ""; if(!Boundary(reason)) return(false); if(!closed.committed || !closed.floor_reached || m_state.phase != 1 || m_profile.shape.K != 21 || m_profile.shape.H != 12 || closed.decision_identity == 0 || closed.decision_identity > (ulong)LONG_MAX || closed.decision_identity != m_state.decision_identity || ArraySize(closed.state) != 33 || ArraySize(closed.account13) != 13 || !MathIsValidNumber(closed.reward) || m_state.updates >= (ulong)(UINT_MAX - 1) || m_state.decision_identity >= (ulong)LONG_MAX) {reason = "online.owner.historical_floor_boundary"; return(false);} for(int i = 0; i < 33; i++) if(!MathIsValidNumber((double)closed.state[i])) {reason = "online.owner.historical_floor_state"; return(false);} const bool native_floor = DPCCTrainingFloorReached(closed.state[0], closed.state[1]); if(!native_floor || closed.floor_reached != native_floor) {reason = "online.owner.historical_floor_state_mismatch"; return(false);} for(int i = 0; i < 13; i++) if(!MathIsValidNumber((double)closed.account13[i])) {reason = "online.owner.historical_floor_account13"; return(false);} const ulong cursor = m_state.updates + 1; const ulong identity = m_state.decision_identity + 1; float next_scenarios[]; if(ArrayResize(next_scenarios, 21 * 12 * 4) != 21 * 12 * 4) {reason = "online.owner.historical_floor_scenarios"; return(false);} ArrayInitialize(next_scenarios, 0.0f); return(ResolveBound(next_scenarios, closed.account13, identity, (float)closed.reward, 0.0f, 0.0f, 0.0f, cursor, reason)); } //--- An open-gap floor consumes the preceding action, without inventing a committed current bar. bool ResolveHistoricalOpenFloor(const SDPCCHistoricalPreparedBar &previous, const SDPCCHistoricalCommittedBar &closed, const SDPCCHistoricalPreparedBar &prepared, string &reason) { if(!Boundary(reason)) return(false); if(m_state.phase != 1 || previous.decision_identity != m_state.decision_identity || m_profile.shape.K != 21 || m_profile.shape.H != 12 || m_state.updates >= (ulong)(UINT_MAX - 1) || CheckPointer(prepared.owner) == POINTER_INVALID || prepared.owner.LastStatus() != DPCC_OK || prepared.signature != prepared.owner.ContractSignature() || prepared.pending != (prepared.token != 0) || ArraySize(prepared.state) != 33 || ArraySize(prepared.account13) != 13 || !DPCCTrainingFloorReached(prepared.state[0], prepared.state[1])) {reason = "online.owner.prepared_floor_boundary"; return(false);} if(!DPCCValidateHistoricalPrepared(prepared, reason)) return(false); double reward = 0; if(!DPCCBuildOpenToOpenTDReward(previous, closed, prepared, reward, reason, true)) return(false); const float native_reward = (float)reward; if(!MathIsValidNumber((double)native_reward)) {reason = "online.owner.prepared_floor_reward_range"; return(false);} float next_scenarios[]; if(ArrayResize(next_scenarios, 21 * 12 * 4) != 21 * 12 * 4) {reason = "online.owner.prepared_floor_capacity"; return(false);} ArrayInitialize(next_scenarios, 0.0f); return(ResolveBound(next_scenarios, prepared.account13, prepared.decision_identity, native_reward, 0.0f, 0.0f, 0.0f, m_state.updates + 1, reason)); } bool RecordPhysical(const float &scenarios[], const float &behavior[], const SDPCCPhysicalOpenMark &mark, string &reason) { if(m_profile.physical_native_required || mark.clock.native_observed) return(DPCCPhysicalRefuse("owner.synthetic_record_only", reason)); string wire = ""; if(!DPCCPhysicalOpenMarkWire(mark, wire, reason)) return(false); return(RecordBound(scenarios, mark.account13, behavior, (ulong)mark.clock.decision_bar_index + 1, reason, wire)); } bool ResolvePhysical(const float &next_scenarios[], const ulong pending_decision_identity, const SDPCCPhysicalOpenMark &next, const float u1, const float u2, const ulong sampler_cursor, string &reason) { if(m_profile.physical_native_required || next.clock.native_observed) return(DPCCPhysicalRefuse("owner.synthetic_resolve_only", reason)); if(pending_decision_identity != m_state.decision_identity || StringLen(m_state.physical_mark_wire) == 0) return(DPCCPhysicalRefuse("owner.pending_decision", reason)); SDPCCPhysicalOpenMark mark; string next_wire = ""; float reward = 0, mask = 0; if(!DPCCPhysicalReadOpenMark(m_state.physical_mark_wire, mark, reason) || !DPCCPhysicalOpenMarkWire(next, next_wire, reason) || !DPCCPhysicalActualReward(mark, next, reward, mask, reason)) return(false); return(ResolveBound(next_scenarios, next.account13, (ulong)next.clock.decision_bar_index + 1, reward, mask, u1, u2, sampler_cursor, reason, next_wire)); } //--- Reuse the exact open already captured while resolving the prior transition. //--- This avoids sampling a second quote/account snapshot after network updates. bool RecordRetainedPhysical(const float &scenarios[], const float &behavior[], string &reason) { reason = ""; if(!Boundary(reason) || m_state.phase != 3 || m_state.bootstrap_mask == 0 || StringLen(m_state.physical_next_wire) == 0) return(DPCCPhysicalRefuse("owner.retained_record_boundary", reason)); SDPCCPhysicalOpenMark retained; if(!DPCCPhysicalReadOpenMark(m_state.physical_next_wire, retained, reason) || retained.ledger.terminal || retained.clock.native_observed != m_profile.physical_native_required || m_state.next_decision_identity != (ulong)retained.clock.decision_bar_index + 1) return(DPCCPhysicalRefuse("owner.retained_record_mark", reason)); return(RecordBound(scenarios, retained.account13, behavior, m_state.next_decision_identity, reason, m_state.physical_next_wire)); } bool RecordCapturedPhysical(const float &scenarios[], const float &behavior[], const SDPCCPhysicalPolicy &policy, const SDPCCPhysicalLedger &ledger, const SDPCCPhysicalConfirmation &confirmation, CDPCCMarketClock &clock, const double scale, string &reason) { if(!m_profile.physical_native_required || !Boundary(reason) || (m_state.phase != 0 && m_state.phase != 3)) return(DPCCPhysicalRefuse("owner.native_record_profile_or_boundary", reason)); SDPCCPhysicalOpenMark mark; string wire = ""; if(m_state.phase == 0) { if(!DPCCPhysicalCaptureOpen(policy, ledger, confirmation, clock, scale, mark, reason) || !DPCCPhysicalOpenMarkWire(mark, wire, reason)) return(false); } else { //--- Reconfirm the resolved pre-action mark using its original predecessor and day proof. SDPCCPhysicalOpenMark previous, retained; if(!DPCCPhysicalReadOpenMark(m_state.physical_mark_wire, previous, reason) || !DPCCPhysicalReadOpenMark(m_state.physical_next_wire, retained, reason)) return(false); string policy_sha = "", retained_policy_sha = ""; if(!DPCCPhysicalPolicySHA(policy, policy_sha, reason) || !DPCCPhysicalPolicySHA(retained.policy, retained_policy_sha, reason) || policy_sha != retained_policy_sha || scale != retained.scale || !DPCCPhysicalDaySamePrior(ledger, retained.ledger) || retained.ledger.terminal) return(DPCCPhysicalRefuse("owner.native_record_retained_identity", reason)); SDPCCPhysicalLedger captured_ledger = previous.ledger; if(StringLen(retained.day_boundary.capture_id) > 0) { if(!DPCCPhysicalImportDayBoundary(policy, previous.ledger, retained.day_boundary, retained.ledger.previous_mark, captured_ledger, reason)) return(false); } SDPCCPhysicalAccount account; if(!DPCCPhysicalCapture(policy, captured_ledger, confirmation, account, reason, GetPointer(clock))) return(false); SDPCCMarketClockCapture capture; if(!clock.ReadCapture(capture, reason)) return(false); if(!DPCCPhysicalSealOpen(policy, captured_ledger, confirmation, account, capture, scale, retained.day_boundary, mark, reason) || !DPCCPhysicalOpenMarkWire(mark, wire, reason) || wire != m_state.physical_next_wire) return(DPCCPhysicalRefuse("owner.native_record_retained_changed", reason)); } return(RecordBound(scenarios, mark.account13, behavior, (ulong)mark.clock.decision_bar_index + 1, reason, wire)); } bool ResolveCapturedPhysical(const float &next_scenarios[], const ulong pending_decision_identity, const SDPCCPhysicalConfirmation &confirmation, CDPCCMarketClock &clock, const float u1, const float u2, const ulong sampler_cursor, string &reason) { if(!m_profile.physical_native_required || !Boundary(reason) || pending_decision_identity != m_state.decision_identity || StringLen(m_state.physical_mark_wire) == 0) return(DPCCPhysicalRefuse("owner.native_resolve_profile_or_boundary", reason)); SDPCCPhysicalOpenMark mark, next; if(!DPCCPhysicalReadOpenMark(m_state.physical_mark_wire, mark, reason)) return(false); SDPCCPhysicalLedger captured_ledger = mark.ledger; SDPCCPhysicalDayBoundary day; ZeroMemory(day); if(DPCCPhysicalDayOpen(TimeCurrent()) != captured_ledger.day_open) { if(!DPCCPhysicalCaptureDayBoundary(mark.policy, captured_ledger, "ONLINE/" + (string)pending_decision_identity, day, reason)) return(false); if(!DPCCPhysicalImportDayBoundary(mark.policy, mark.ledger, day, TimeCurrent(), captured_ledger, reason)) return(false); } SDPCCPhysicalAccount account; if(!DPCCPhysicalCapture(mark.policy, captured_ledger, confirmation, account, reason, GetPointer(clock))) return(false); SDPCCMarketClockCapture capture; if(!clock.ReadCapture(capture, reason)) return(false); if(!DPCCPhysicalSealOpen(mark.policy, captured_ledger, confirmation, account, capture, mark.scale, day, next, reason)) return(false); string wire = ""; float reward = 0, mask = 0; if(!DPCCPhysicalOpenMarkWire(next, wire, reason) || !DPCCPhysicalActualReward(mark, next, reward, mask, reason)) return(false); return(ResolveBound(next_scenarios, next.account13, (ulong)next.clock.decision_bar_index + 1, reward, mask, u1, u2, sampler_cursor, reason, wire)); } bool Apply(SDPCCOnlinePolicyResult &result, string &reason) { ZeroMemory(result); reason = ""; if(!Boundary(reason)) return(false); if(m_state.phase != 2 || m_state.updates >= UINT_MAX - 1 || !ValidateState(m_state, reason)) { if(reason == "") reason = "online.owner.apply_boundary"; return(false); } const ulong next = m_state.updates + 1; SDPCCOnlinePolicyStep step; ZeroMemory(step); step.reward = m_state.reward; step.gamma = m_profile.gamma; step.bootstrap_mask = m_state.bootstrap_mask; step.sample_u1 = m_state.sample_u1; step.sample_u2 = m_state.sample_u2; step.probability_weight = m_profile.probability_weight; step.value_weight = m_profile.value_weight; step.actor_upstream = m_profile.actor_upstream; step.risk_type = m_profile.risk_type; step.risk_parameter = m_profile.risk_parameter; step.mean_weight = m_profile.mean_weight; step.target_tau = m_profile.target_tau; step.actor_critic = DPCCOnlineActorCritic(m_state.actor_updates, m_profile.actor_critic_selector, m_profile.selector_seed); step.update_actor = next % m_profile.actor_every == 0; step.refresh_actor_target = next % m_profile.actor_target_every == 0; step.refresh_critic_target = (next % m_profile.critic_target_every == 0 ? DPCCOnlineCrossedTarget(m_state.critic_target_updates, m_profile.selector_seed) : -1); //--- Pack only recorded first-six behavior into the existing native updater. CBufferFloat current, next_scenarios, behavior; SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(m_profile.shape, layout, reason)) return(false); float carrier[]; if(ArrayResize(carrier, (int)layout.plan_count) != (int)layout.plan_count) { reason = "online.owner.behavior_carrier_capacity"; return(false); } ArrayInitialize(carrier, 0.0f); ArrayCopy(carrier, m_state.behavior, 0, 0, 6); if(!current.AssignArray(m_state.scenarios) || !next_scenarios.AssignArray(m_state.next_scenarios) || !behavior.AssignArray(carrier)) { reason = "online.owner.update_capacity"; return(false); } if(!DPCCUpdateOnlinePolicy(*m_nets[0], *m_nets[1], *m_nets[2], *m_nets[3], *m_nets[4], *m_nets[5], m_profile.shape, GetPointer(current), m_state.account13, GetPointer(behavior), GetPointer(next_scenarios), m_state.next_account13, step, result, reason)) { if(result.mutation_started) return(Fail(reason, reason)); return(false); } m_state.updates = next; if(step.update_actor) m_state.actor_updates++; if(step.refresh_actor_target) m_state.actor_target_updates++; if(step.refresh_critic_target >= 0) m_state.critic_target_updates++; m_state.phase = 3; m_durable = false; m_checkpointable = true; return(true); } //--- Publish only model parameters and clocks; owner observations stay volatile. bool SaveCheckpoint(const string prefix, const bool common, string &manifest_sha, string &reason) { manifest_sha = ""; reason = ""; if(!m_initialized || m_failed) { reason = "online.owner.poisoned_or_unbound"; return(false); } if(m_replay_pending) { reason = "online.owner.replay_rebind_required"; return(false); } const bool initial = (m_state.phase == 0 && m_state.updates == 0 && m_resume_decision_identity == 0 && m_resume_server_open == 0); const bool applied = (m_state.phase == 3 && m_state.updates > 0 && m_resume_decision_identity == m_state.next_decision_identity && m_resume_server_open > 0); if(!m_checkpointable || (!initial && !applied)) { reason = "online.owner.checkpoint_requires_applied_model"; return(false); } if(m_profile.physical_native_required || m_state.physical_mark_wire != "" || m_state.physical_next_wire != "" || m_state.execution_transcript_wire != "") { reason = "online.owner.model_checkpoint_simulated_only"; return(false); } if(!CheckNets(reason) || !ValidateState(m_state, reason)) return(Fail(reason, reason)); string canonical = ""; if(!DPCCCheckpointCanonicalPath(prefix, canonical, reason) || canonical != prefix) return(Fail("online.owner.checkpoint_prefix", reason)); //--- Fingerprint mutable model state once at publication, not on each policy step. for(int i = 0; i < 6; i++) if(!m_nets[i].DPCCPolicyTrainingStateFingerprint(m_fingerprints[i])) return(Fail("online.owner.checkpoint_fingerprint", reason)); //--- Retain the exclusive writer lease before creating immutable model candidates. string lock_path = prefix + "_writer.lock"; if(m_checkpoint_lock != INVALID_HANDLE || !DPCCCheckpointCanonicalPath(lock_path, canonical, reason) || canonical != lock_path) return(Fail("online.owner.writer_lock_path", reason)); m_checkpoint_lock = FileOpen(lock_path, FILE_READ | FILE_WRITE | FILE_BIN | (common ? FILE_COMMON : 0)); if(m_checkpoint_lock == INVALID_HANDLE) return(Fail("online.owner.writer_lock", reason)); string paths[7], hashes[6]; const string names[] = {"actor.dppt", "q1.dppt", "q2.dppt", "target_actor.dppt", "target_q1.dppt", "target_q2.dppt", "manifest.txt" }; for(int i = 0; i < 7; i++) { paths[i] = prefix + "_" + names[i]; if(!DPCCCheckpointCanonicalPath(paths[i], canonical, reason) || canonical != paths[i] || FileIsExist(paths[i], common ? FILE_COMMON : 0)) return(Fail("online.owner.immutable_destination", reason)); } //--- Serialize exact policy weights and Adam state, excluding transient tensors. for(int i = 0; i < 6; i++) { const int handle = FileOpen(paths[i], FILE_WRITE | FILE_BIN | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) return(Fail("online.owner.model_save_open", reason)); ResetLastError(); bool saved = m_nets[i].SaveDPCCPolicyTrainingStateOnly(handle); FileFlush(handle); saved = saved && GetLastError() == 0; FileClose(handle); if(!saved || !DPCCCheckpointFileSHA(paths[i], common, 536870912, hashes[i], reason)) return(Fail("online.owner.model_save_or_hash", reason)); CNet *candidate = NULL; const bool verified = LoadCandidate(paths[i], common, i, m_nets[0].GetOpenCL(), m_fingerprints[i], candidate, reason); if(CheckPointer(candidate) == POINTER_DYNAMIC) delete candidate; if(!verified) return(Fail("online.owner.model_training_state_roundtrip", reason)); } //--- Bind graph hashes and the restart cursor to the authenticated manifest. string manifest = "DPCC_ONLINE_MODEL_CHECKPOINT_V2\n"; DPCCOfflineCheckpointField(manifest, m_profile_sha); DPCCOfflineCheckpointField(manifest, (string)m_resume_decision_identity); DPCCOfflineCheckpointField(manifest, (string)m_resume_server_open); DPCCOfflineCheckpointField(manifest, (string)m_state.updates); DPCCOfflineCheckpointField(manifest, (string)m_state.actor_updates); DPCCOfflineCheckpointField(manifest, (string)m_state.actor_target_updates); DPCCOfflineCheckpointField(manifest, (string)m_state.critic_target_updates); DPCCOfflineCheckpointField(manifest, (string)m_state.sampler_cursor); for(int i = 0; i < 6; i++) { string actual = ""; if(!DPCCCheckpointFileSHA(paths[i], common, 536870912, actual, reason) || actual != hashes[i]) return(Fail("online.owner.model_candidate_changed", reason)); DPCCOfflineCheckpointField(manifest, paths[i]); DPCCOfflineCheckpointField(manifest, hashes[i]); DPCCOfflineCheckpointField(manifest, StringFormat("%016I64X", m_fingerprints[i])); } DPCCOfflineCheckpointField(manifest, "END_DPCC_ONLINE_MODEL_CHECKPOINT_V2"); //--- Publish the immutable manifest last, then verify its exact bytes and digest. string expected_sha = "", actual_sha = "", readback = ""; if(!DPCCCheckpointTextSHA(manifest, expected_sha, reason) || !DPCCCheckpointWriteText(paths[6], common, manifest, reason) || !DPCCOnlineOwnerReadText(paths[6], common, readback, reason) || readback != manifest || !DPCCCheckpointFileSHA(paths[6], common, 1048576, actual_sha, reason) || actual_sha != expected_sha) return(Fail("online.owner.model_manifest_publication", reason)); m_last_manifest = paths[6]; m_last_sha = actual_sha; m_last_common = common; m_durable = true; m_checkpointable = false; m_trusted_restore = false; manifest_sha = actual_sha; CloseCheckpointLock(); return(true); } //--- Caller takes ownership of returned fresh networks; training clocks are restored separately. bool RestoreCheckpoint(const SDPCCOnlineOwnerProfile &profile, const string manifest_path, const string trusted_sha, const bool common, COpenCLMy *context, CNet *&nets_out[], string &reason) { reason = ""; if(m_replay_pending) { reason = "online.owner.replay_rebind_required"; return(false); } if(ArraySize(nets_out) != 6 || CheckPointer(context) == POINTER_INVALID) { reason = "online.owner.restore_destinations"; return(false); } for(int i = 0; i < 6; i++) if(nets_out[i] != NULL) { reason = "online.owner.restore_destinations"; return(false); } if(m_initialized && (manifest_path != m_last_manifest || trusted_sha != m_last_sha || common != m_last_common)) { reason = "online.owner.restore_requires_last_durable"; return(false); } if(profile.physical_native_required) { reason = "online.owner.restore_simulated_only"; return(false); } //--- Authenticate the exact compact clock and model manifest before graph allocation. CDPCCOnlineOwner staged; if(!DPCCOnlineOwnerProfileSHA(profile, staged.m_profile_sha, reason)) return(false); staged.m_profile = profile; string canonical = "", actual = "", manifest = ""; if(!DPCCResolverLowerSha256(trusted_sha) || !DPCCCheckpointCanonicalPath(manifest_path, canonical, reason) || canonical != manifest_path || !DPCCCheckpointFileSHA(manifest_path, common, 1048576, actual, reason) || actual != trusted_sha || !DPCCOnlineOwnerReadText(manifest_path, common, manifest, reason)) { reason = "online.owner.restore_manifest_identity"; return(false); } string fields[]; if(StringSplit(manifest, '\n', fields) != 29 || fields[0] != "DPCC_ONLINE_MODEL_CHECKPOINT_V2" || fields[1] != staged.m_profile_sha || fields[27] != "END_DPCC_ONLINE_MODEL_CHECKPOINT_V2" || fields[28] != "") { reason = "online.owner.restore_manifest_contract"; return(false); } ulong clock[7]; const ulong limits[] = { (ulong)LONG_MAX, (ulong)LONG_MAX, (ulong)UINT_MAX, (ulong)UINT_MAX, (ulong)UINT_MAX, (ulong)UINT_MAX, (ulong)UINT_MAX }; for(int i = 0; i < 7; i++) if(!DPCCCheckpointReadUnsigned(fields[2 + i], limits[i], clock[i])) { reason = "online.owner.restore_clock_integer"; return(false); } const ulong sampler_cursor = clock[6]; if(clock[2] >= UINT_MAX || ((clock[2] == 0) != (clock[0] == 0)) || ((clock[2] == 0) != (clock[1] == 0)) || (clock[2] > 0 && clock[6] != clock[2]) || clock[3] != clock[2] / profile.actor_every || clock[4] != clock[2] / profile.actor_target_every || clock[5] != clock[2] / profile.critic_target_every) { reason = "online.owner.restore_clock_contract"; return(false); } string paths[6], hashes[6]; ulong fingerprints[6]; for(int i = 0; i < 6; i++) { const int field = 9 + 3 * i; paths[i] = fields[field]; hashes[i] = fields[field + 1]; if(!DPCCCheckpointCanonicalPath(paths[i], canonical, reason) || canonical != paths[i] || paths[i] == manifest_path || !DPCCResolverLowerSha256(hashes[i]) || !DPCCOfflineCheckpointReadHex64(fields[field + 2], fingerprints[i])) { reason = "online.owner.restore_model_contract"; return(false); } for(int j = 0; j < i; j++) if(paths[i] == paths[j]) { reason = "online.owner.restore_model_alias"; return(false); } if(!DPCCCheckpointFileSHA(paths[i], common, 536870912, actual, reason) || actual != hashes[i]) { reason = "online.owner.restore_model_identity"; return(false); } } //--- Build the volatile phase-zero owner with counters but no account or transition snapshot. SDPCCPolicyLayout layout; if(!DPCCPolicyLayout(profile.shape, layout, reason) || ArrayResize(staged.m_state.scenarios, (int)layout.scenario_count) != (int)layout.scenario_count || ArrayResize(staged.m_state.next_scenarios, (int)layout.scenario_count) != (int)layout.scenario_count) return(false); ArrayInitialize(staged.m_state.scenarios, 0.0f); ArrayInitialize(staged.m_state.next_scenarios, 0.0f); staged.m_state.phase = 0; staged.m_state.updates = clock[2]; staged.m_state.actor_updates = clock[3]; staged.m_state.actor_target_updates = clock[4]; staged.m_state.critic_target_updates = clock[5]; staged.m_state.sampler_cursor = sampler_cursor; staged.m_resume_decision_identity = clock[0]; staged.m_resume_server_open = (long)clock[1]; CNet *candidates[6]; for(int i = 0; i < 6; i++) candidates[i] = NULL; for(int i = 0; i < 6; i++) { if(!staged.LoadCandidate(paths[i], common, i, context, fingerprints[i], candidates[i], reason)) { DeleteCandidates(candidates); return(false); } staged.m_nets[i] = candidates[i]; staged.m_fingerprints[i] = fingerprints[i]; } //--- Publish only after each graph, model digest, and owner profile has joined. bool verified = staged.CheckNets(reason) && staged.ValidateState(staged.m_state, reason); for(int i = 0; i < 6 && verified; i++) verified = DPCCCheckpointFileSHA(paths[i], common, 536870912, actual, reason) && actual == hashes[i]; if(verified) verified = DPCCCheckpointFileSHA(manifest_path, common, 1048576, actual, reason) && actual == trusted_sha; if(!verified) { DeleteCandidates(candidates); if(reason == "") reason = "online.owner.restore_final_rehash"; return(false); } m_profile = profile; m_profile_sha = staged.m_profile_sha; m_state = staged.m_state; for(int i = 0; i < 6; i++) { m_nets[i] = candidates[i]; m_fingerprints[i] = fingerprints[i]; nets_out[i] = candidates[i]; } m_resume_decision_identity = staged.m_resume_decision_identity; m_resume_server_open = staged.m_resume_server_open; m_last_manifest = manifest_path; m_last_sha = trusted_sha; m_last_common = common; m_initialized = true; m_failed = false; m_durable = true; m_trusted_restore = true; m_replay_pending = false; m_replay_identity = 0; m_checkpointable = false; ClearReplayObservation(); reason = ""; return(true); } }; //+------------------------------------------------------------------+ //| Bootstrap a durable online tuple from a completed Study result | //+------------------------------------------------------------------+ //--- On success the caller owns the owner and all six networks. Delete the owner //--- before the networks, and keep the supplied OpenCL context alive for their lifetime. //--- On failure outputs remain empty; immutable checkpoint candidates are retained. bool DPCCPromoteOfflineActor(const string result_path, const string trusted_result_sha, const bool common, const SDPCCOfflinePolicyProfile &offline_profile, const SDPCCOfflineFitKey &keys[], const SDPCCOfflinePolicyAdmission &admission, const SDPCCConfig &config, CDPCCMarketClock &clock, COpenCLMy *context, const SDPCCOnlineOwnerProfile &online_profile, const string online_prefix, CNet *&nets_out[], CDPCCOnlineOwner *&owner_out, string &manifest_sha, string &reason, const ulong maximum_bytes = 536870912) { //--- Reject occupied outputs and authenticate the completed Study result before allocating native graphs. manifest_sha = ""; reason = ""; if(ArraySize(nets_out) != 6 || owner_out != NULL || CheckPointer(context) == POINTER_INVALID || maximum_bytes == 0) { reason = "online.promotion.destinations_or_context"; return(false); } for(int i = 0; i < 6; i++) if(nets_out[i] != NULL) { reason = "online.promotion.destinations_or_context"; return(false); } string canonical = "", actual = "", wire = "", profile_sha = ""; const ulong text_limit = (maximum_bytes < 1048576 ? maximum_bytes : (ulong)1048576); if(!DPCCCheckpointCanonicalPath(result_path, canonical, reason) || canonical != result_path || !DPCCResolverLowerSha256(trusted_result_sha) || !DPCCCheckpointFileSHA(result_path, common, text_limit, actual, reason) || actual != trusted_result_sha || !DPCCOnlineOwnerReadText(result_path, common, wire, reason)) { reason = "online.promotion.trusted_result"; return(false); } string fields[]; ulong next = 0, total = 0; if(StringSplit(wire, '\n', fields) != 18 || fields[0] != "DPCC_STUDY_OFFLINE_RESULT_V2" || fields[15] != "NOT_A_READINESS_AUTHORIZATION" || fields[16] != "END_DPCC_STUDY_OFFLINE_RESULT_V2" || fields[17] != "" || !DPCCCheckpointReadUnsigned(fields[1], (ulong)ArraySize(keys), next) || !DPCCCheckpointReadUnsigned(fields[2], (ulong)ArraySize(keys), total) || total == 0 || next != total || total != (ulong)ArraySize(keys) || total != admission.rows || fields[3] != (string)false || fields[7] != offline_profile.decoder_sha256 || fields[12] != offline_profile.seed_sha256 || fields[13] != offline_profile.profile_sha256 || fields[14] != offline_profile.fit_plan_sha256) { reason = "online.promotion.completed_result_binding"; return(false); } const int paths[] = {4, 6, 8, 10}; const int hashes[] = {5, 7, 9, 11, 12, 13, 14}; for(int i = 0; i < ArraySize(paths); i++) if(!DPCCCheckpointCanonicalPath(fields[paths[i]], canonical, reason) || canonical != fields[paths[i]]) { reason = "online.promotion.result_path"; return(false); } for(int i = 0; i < ArraySize(hashes); i++) if(!DPCCResolverLowerSha256(fields[hashes[i]])) { reason = "online.promotion.result_digest"; return(false); } const SDPCCShape source = config.shape, target = online_profile.shape; const uint source_dims[] = {source.K, source.V, source.H, source.D, source.M, source.J, source.E, source.PlanBatch, source.NActions, source.StateSize, source.Embedding, source.Heads, source.KeyDimension, source.Depth }; const uint target_dims[] = {target.K, target.V, target.H, target.D, target.M, target.J, target.E, target.PlanBatch, target.NActions, target.StateSize, target.Embedding, target.Heads, target.KeyDimension, target.Depth }; for(int i = 0; i < ArraySize(source_dims); i++) if(source_dims[i] != target_dims[i]) { reason = "online.promotion.shape"; return(false); } if(!DPCCOnlineOwnerProfileSHA(online_profile, profile_sha, reason)) return(false); //--- Restore the authenticated Actor directly; allocate only its critic and frozen target companions. CNet *candidates[6]; for(int i = 0; i < 6; i++) candidates[i] = NULL; CDPCCOfflineRowOwner offline; CDPCCOnlineOwner *online = NULL; string actor_path = "", actor_sha = "", saved_sha = ""; bool completed = false; do { if(!offline.RestoreCheckpoint(fields[4], fields[5], common, offline_profile, keys, admission, config, clock, context, candidates[0], reason, maximum_bytes, fields[9], fields[11], fields[10])) break; if(offline.NextRow() != total || offline.FloorReached() || !offline.Durable() || !offline.RestoredActorArtifact(*candidates[0], actor_path, actor_sha, reason) || actor_sha != online_profile.offline_actor_sha256) { reason = "online.promotion.completed_actor_identity"; break; } bool companions = true; for(int i = 1; i < 6; i++) { candidates[i] = new CNet; if(CheckPointer(candidates[i]) == POINTER_INVALID) { reason = "online.promotion.companion_allocation"; companions = false; break; } if(!DPCCCreatePolicyNet(*candidates[i], online_profile.shape, i != 3, context, reason)) { companions = false; break; } } if(!companions) break; if(!DPCCInitializePolicyTargets(*candidates[0], *candidates[1], *candidates[2], *candidates[3], *candidates[4], *candidates[5], online_profile.shape, reason)) break; //--- The target copy must preserve the actual offline Actor's parameters. if(!offline.RestoredActorArtifact(*candidates[0], actor_path, actor_sha, reason)) break; online = new CDPCCOnlineOwner; if(CheckPointer(online) == POINTER_INVALID) { reason = "online.promotion.owner_allocation"; break; } if(!online.Initialize(online_profile, *candidates[0], *candidates[1], *candidates[2], *candidates[3], *candidates[4], *candidates[5], reason)) break; if(!DPCCCheckpointFileSHA(result_path, common, text_limit, actual, reason) || actual != trusted_result_sha || !DPCCCheckpointFileSHA(fields[4], common, text_limit, actual, reason) || actual != fields[5] || !DPCCCheckpointFileSHA(actor_path, common, maximum_bytes, actual, reason) || actual != actor_sha) { reason = "online.promotion.source_changed"; break; } if(!online.SaveCheckpoint(online_prefix, common, saved_sha, reason) || !online.Durable()) break; completed = true; } while(false); //--- Publish only the durable six-network boundary; failed candidates have no surviving borrowed owner. if(!completed) { if(CheckPointer(online) == POINTER_DYNAMIC) delete online; for(int i = 0; i < 6; i++) if(CheckPointer(candidates[i]) == POINTER_DYNAMIC) delete candidates[i]; return(false); } for(int i = 0; i < 6; i++) nets_out[i] = candidates[i]; owner_out = online; manifest_sha = saved_sha; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Factual one-bar continuation for the shared online training path | //+------------------------------------------------------------------+ class CDPCCOnlineHistoricalSteps { private: CNet m_factual; SDPCCConfig m_config; float m_state[]; SDPCCSimulatedPositionMetadata m_position; SDPCCHistoricalPreparedBar m_current, m_previous; SDPCCHistoricalCommittedBar m_closed; bool m_initialized, m_failed, m_have_closed; uint m_step, m_horizon; double m_starting_balance; uint m_currency_digits; string m_currency; float m_open_values[]; bool Fail(const string cause, string &reason) { m_failed = true; reason = cause; return(false); } public: CDPCCOnlineHistoricalSteps(void) { m_initialized = false; m_failed = false; m_have_closed = false; m_step = 0; m_horizon = 0; m_currency_digits = 0; ZeroMemory(m_current); ZeroMemory(m_previous); ZeroMemory(m_closed); ZeroMemory(m_position); } COpenCLMy *Context(void) { return(m_factual.GetOpenCL()); } bool Initialize(const SDPCCConfig &config, const uint rows, const double balance, const string currency, string &reason) { //--- The policy keeps H12; only the factual financial layer has H1. const long captured_currency_digits = AccountInfoInteger(ACCOUNT_CURRENCY_DIGITS); if(m_initialized || m_failed || rows < 2 || rows > 16777216 || !DPCCValidateFixedConfig(config.shape, config) || !MathIsValidNumber(balance) || balance < 50 || currency == "" || currency != AccountInfoString(ACCOUNT_CURRENCY) || captured_currency_digits < 0 || captured_currency_digits > 8 || (config.margin_fx_profile != DPCC_MARGIN_FOREX_SPOT_V1 && config.margin_fx_profile != DPCC_MARGIN_EXTERNAL_V2 && config.margin_fx_profile != DPCC_MARGIN_HISTORICAL_CLOSE_V1)) return(Fail("online.history.initialization", reason)); SDPCCConfig physical = config; physical.shape.K = 1; physical.shape.H = 1; physical.shape.M = 1; physical.shape.Depth = 0; physical.projection_lookback_steps = (uint)MathMin(physical.projection_lookback_steps, 1); if(!DPCCCreateHistoricalRolloutNet(m_factual, physical, NULL, reason)) return(Fail(reason, reason)); m_config = config; m_horizon = rows; m_starting_balance = balance; m_currency_digits = (uint)captured_currency_digits; m_currency = currency; m_initialized = true; return(true); } //+------------------------------------------------------------------+ //| Prepare only the first admitted factual bar before Actor action | //+------------------------------------------------------------------+ bool Prepare(CDPCCMarketClock &clock, const SDPCCOfflineFitWitness &witness, const float &row_context[], const float open, long &server_opens[], string &reason) { ArrayResize(server_opens, 0); if(!m_initialized || m_failed || m_current.pending || m_step >= m_horizon || !witness.admitted || witness.split != "fit" || witness.horizon != (long)m_config.shape.H || witness.decision_bar < 0 || witness.decision_bar >= LONG_MAX || ArraySize(witness.bar_indices) != (int)m_config.shape.H || ArraySize(witness.bar_open_times) != (int)m_config.shape.H || ArraySize(witness.bar_close_times) != (int)m_config.shape.H) return(Fail("online.history.prepare_boundary", reason)); //--- Resolve every witness endpoint through the existing admitted clock. long closes[], ordinals[]; const int count = 1; if(ArrayResize(server_opens, count) != count || ArrayResize(closes, count) != count || ArrayResize(ordinals, count) != count) return(Fail("online.history.clock_capacity", reason)); for(int h = 0; h < count; h++) { SDPCCClockBar bar; long utc_open = 0, utc_close = 0; if(!clock.IndexedBar(witness.decision_bar + h, bar, utc_open, utc_close, reason) || bar.ordinal != witness.bar_indices[h] || bar.ordinal != witness.decision_bar + h || utc_open != witness.bar_open_times[h] || utc_close != witness.bar_close_times[h] || (h == 0 && utc_open != witness.decision_time)) return(Fail("online.history.clock_join:" + reason, reason)); server_opens[h] = bar.server_open; closes[h] = bar.server_close; ordinals[h] = bar.ordinal; } //--- The bank owns execution/calendar/limits; only the continuous ledger is replaced. SDPCCShape one_shape = m_config.shape; one_shape.K = 1; one_shape.H = 1; one_shape.M = 1; SDPCCConfig seed_config = m_config; seed_config.shape = one_shape; seed_config.projection_lookback_steps = (uint)MathMin(seed_config.projection_lookback_steps, 1); float context[]; SDPCCLayout layout, source_layout, unused; uint tokens = 0; if(!DPCCCompositeLayouts(m_config.shape, source_layout, unused, tokens) || ArraySize(row_context) != (int)source_layout.offsets[10] || !DPCCCompositeLayouts(one_shape, layout, unused, tokens) || ArrayResize(context, (int)layout.offsets[10]) != (int)layout.offsets[10]) return(Fail("online.history.context_shape", reason)); //--- Copy only open-known financial fields. Decoder/target sections are unused by native rollout. ArrayInitialize(context, 0.0f); for(int section = 4; section <= 6; section++) for(int i = 0; i < (int)layout.lengths[section]; i++) context[(int)layout.offsets[section] + i] = row_context[(int)source_layout.offsets[section] + i]; const int execution = (int)layout.offsets[7]; const int source_execution = (int)source_layout.offsets[7]; for(int i = 0; i < 24; i++) if(i != 1 && i != 2 && i != 3 && i != 17 && i != 21) context[execution + i] = row_context[source_execution + i]; for(int i = 1; i <= 3; i++) context[execution + i] = context[execution]; context[execution + 17] = context[execution + 16]; //--- Native initial seed has no elapsed gap before this episode starts. if(m_step == 0) context[execution + 19] = 0.0f; int prefix = 0; //--- At the initial flat seed no pre-start event can affect an existing position. if(m_step > 0) for(uint event = 0; event < m_config.shape.E; event++) { const int source_event = (int)source_layout.offsets[9] + (int)event * 6; if(!MathIsValidNumber((double)row_context[source_event])) return(Fail("online.history.calendar_offset", reason)); if(row_context[source_event] >= 0) break; for(int channel = 0; channel < 6; channel++) context[(int)layout.offsets[9] + prefix * 6 + channel] = row_context[source_event + channel]; prefix++; } context[(int)layout.offsets[8]] = (float)prefix; context[execution + 21] = (float)prefix; float prices[4]; for(int channel = 0; channel < 4; channel++) prices[channel] = open; if(m_step == 0) { SDPCCSnapshotMeta meta; ZeroMemory(meta); meta.account_currency = m_currency; if(!DPCCBuildHistoricalFlatSeed(m_starting_balance, m_currency, m_currency_digits, meta, seed_config, one_shape, context, prices, m_state, m_position, reason)) return(Fail(reason, reason)); } else { if(!m_have_closed || m_closed.floor_reached || witness.decision_bar != m_previous.ordinal + 1) return(Fail("online.history.consecutive_ledger", reason)); for(int i = 0; i < 24; i++) context[(int)layout.offsets[4] + i] = m_state[i]; //--- Spot FX validates the reference conversion against the retained profile quote. if(!DPCCBindHistoricalFxReference(one_shape, context, m_state, reason)) return(Fail(reason, reason)); } float step_values[]; ENUM_SYMBOL_CALC_MODE calculation_mode = SYMBOL_CALC_MODE_FOREX; if(!DPCCBuildHistoricalBarStepValues(one_shape, context, prices, witness.decision_bar, server_opens, closes, ordinals, witness.bar_open_times[0], witness.bar_close_times[0], step_values, calculation_mode, reason)) return(Fail(reason, reason)); if(ArrayCopy(m_open_values, step_values) != ArraySize(step_values)) return(Fail("online.history.open_context_copy", reason)); CNeuronScenarioRolloutOCL *native = dynamic_cast(m_factual.Layer(1)); if(CheckPointer(native) == POINTER_INVALID) return(Fail("online.history.native_layer", reason)); CBufferFloat prior; if(!DPCCPrepareHistoricalBar(*native, step_values, witness.decision_bar, (ulong)witness.decision_bar + 1, server_opens[0], closes[0], witness.bar_open_times[0], witness.bar_close_times[0], calculation_mode, m_state, m_position, m_step, m_horizon, m_step == 0, prior, m_current, reason, true)) return(Fail(reason, reason)); return(true); } bool OpenFloor(void) const { return(ArraySize(m_current.state) == 33 && DPCCTrainingFloorReached(m_current.state[0], m_current.state[1])); } bool Account(float &account[], string &reason) const { if(!m_initialized || m_failed || !m_current.pending || ArrayCopy(account, m_current.account13) != 13) { reason = "online.history.account_unavailable"; return(false); } return(true); } //+-------------------------------------------------------------------+ //| Evaluate the full H12 Actor plan and extract the first six values | //+-------------------------------------------------------------------+ bool Action(CNet &actor, const float &scenarios[], float &behavior[], string &reason) { if(!m_initialized || m_failed || !m_current.pending || OpenFloor() || actor.GetOpenCL() != Context()) return(Fail("online.history.action_boundary", reason)); CBufferFloat values, policy_input; if(!values.AssignArray(scenarios)) return(Fail("online.history.scenario_copy", reason)); if(!DPCCPackPolicyInput(m_config.shape, GetPointer(values), m_current.account13, NULL, GetPointer(policy_input), reason)) return(Fail(reason, reason)); if(!actor.feedForward(GetPointer(policy_input), 1, false, (CBufferFloat*)NULL)) return(Fail("online.history.actor_forward", reason)); CBufferFloat *plan = actor.Layer(2).getOutput(); if(CheckPointer(plan) == POINTER_INVALID || !plan.BufferRead() || plan.Total() != (int)m_config.shape.H * 6 || ArrayResize(behavior, 6) != 6) return(Fail("online.history.actor_read", reason)); for(int i = 0; i < 6; i++) { behavior[i] = plan.At(i); if(!MathIsValidNumber(behavior[i]) || behavior[i] < 0 || behavior[i] > 1) return(Fail("online.history.actor_range", reason)); } return(true); } //+------------------------------------------------------------------+ //| Resolve the previous action at the prepared next H1 open | //+------------------------------------------------------------------+ bool UpdatePrevious(CDPCCOnlineOwner &owner, const SDPCCOnlineOwnerProfile &profile, const float &scenarios[], SDPCCOnlinePolicyResult &result, string &reason) { if(!m_initialized || m_failed || !m_have_closed || !m_current.pending) return(Fail("online.history.update_boundary", reason)); double reward = 0; if(!DPCCBuildOpenToOpenTDReward(m_previous, m_closed, m_current, reward, reason)) return(Fail(reason, reason)); SDPCCOnlineOwnerState state; if(!owner.Snapshot(state)) return(Fail("online.history.owner_snapshot", reason)); float u1 = 0, u2 = 0; const ulong cursor = state.updates + 1; if(!DPCCOnlineCriticSampleU(profile.sampler_sha256, cursor, 1, u1, reason)) return(Fail(reason, reason)); if(!DPCCOnlineCriticSampleU(profile.sampler_sha256, cursor, 2, u2, reason)) return(Fail(reason, reason)); if(!owner.Resolve(scenarios, m_current.account13, m_current.decision_identity, (float)reward, 1, u1, u2, cursor, reason)) return(Fail(reason, reason)); if(!owner.Apply(result, reason)) return(Fail(reason, reason)); if(!owner.SetResumeCursor(m_current.decision_identity, m_current.server_open, reason)) return(Fail(reason, reason)); return(true); } bool Commit(const float &behavior[], const float &row_context[], const float &prices[], bool &floor_reached, string &reason) { floor_reached = false; if(!m_initialized || m_failed || !m_current.pending || OpenFloor()) return(Fail("online.history.commit_boundary", reason)); //--- Materialize exactly this factual H1 suffix after the caller selected/recorded its action. SDPCCShape one_shape = m_config.shape; one_shape.K = 1; one_shape.H = 1; one_shape.M = 1; one_shape.Depth = 0; SDPCCLayout layout, source_layout, unused; uint tokens = 0; if(!DPCCBuildRolloutLayout(one_shape, layout) || !DPCCCompositeLayouts(m_config.shape, source_layout, unused, tokens) || ArraySize(row_context) != (int)source_layout.offsets[10] || ArraySize(prices) != 4 || ArraySize(m_open_values) != (int)layout.total) return(Fail("online.history.factual_shape", reason)); float factual[]; if(ArrayCopy(factual, m_open_values) != (int)layout.total) return(Fail("online.history.factual_capacity", reason)); for(int channel = 0; channel < 4; channel++) factual[(int)layout.offsets[0] + channel] = prices[channel]; for(int i = 0; i < 24; i++) factual[(int)layout.offsets[4] + i] = row_context[(int)source_layout.offsets[7] + i]; if(m_step == 0) factual[(int)layout.offsets[4] + 19] = 0.0f; const float count = row_context[(int)source_layout.offsets[8]]; if(!MathIsValidNumber((double)count) || count < 0 || count > (float)m_config.shape.E || MathFloor(count) != count) return(Fail("online.history.factual_calendar_count", reason)); for(int i = 0; i < (int)layout.lengths[6]; i++) factual[(int)layout.offsets[6] + i] = 0.0f; int retained = 0; for(int event = 0; event < (int)count; event++) { const int source_event = (int)source_layout.offsets[9] + event * 6; //--- Initial flat history excludes negative pre-start events; offset zero stays in the suffix. if(m_step == 0 && row_context[source_event] < 0) continue; for(int channel = 0; channel < 6; channel++) factual[(int)layout.offsets[6] + retained * 6 + channel] = row_context[source_event + channel]; retained++; } factual[(int)layout.offsets[5]] = (float)retained; factual[(int)layout.offsets[4] + 21] = (float)retained; CNeuronScenarioRolloutOCL *native = dynamic_cast(m_factual.Layer(1)); if(CheckPointer(native) == POINTER_INVALID) return(Fail("online.history.native_layer", reason)); CBufferFloat factual_buffer; if(!DPCCUploadHistoricalBuffer(factual_buffer, factual, Context())) return(Fail("online.history.factual_upload", reason)); if(!DPCCCommitHistoricalBar(*native, m_current, behavior, m_closed, reason, GetPointer(factual_buffer))) return(Fail(reason, reason)); if(ArrayCopy(m_state, m_closed.state) != 33) return(Fail("online.history.ledger_copy", reason)); m_position = m_closed.position; m_previous = m_current; m_have_closed = true; m_step++; floor_reached = m_closed.floor_reached; return(true); } bool UpdateOpenFloor(CDPCCOnlineOwner &owner, SDPCCOnlinePolicyResult &result, string &reason) { if(!m_initialized || m_failed || !m_have_closed || !OpenFloor()) return(Fail("online.history.open_floor_boundary", reason)); if(!owner.ResolveHistoricalOpenFloor(m_previous, m_closed, m_current, reason)) return(Fail(reason, reason)); if(!owner.Apply(result, reason)) return(Fail(reason, reason)); if(!owner.SetResumeCursor(m_current.decision_identity, m_current.server_open, reason)) return(Fail(reason, reason)); return(true); } bool UpdateFloor(CDPCCOnlineOwner &owner, SDPCCOnlinePolicyResult &result, string &reason) { if(!m_closed.committed || !m_closed.floor_reached) return(Fail("online.history.floor_boundary", reason)); SDPCCHistoricalCommittedBar terminal = m_closed; terminal.reward -= m_previous.preparation_reward; if(!owner.ResolveHistoricalFloor(terminal, reason)) return(Fail(reason, reason)); if(!owner.Apply(result, reason)) return(Fail(reason, reason)); if(!owner.SetResumeCursor(m_previous.decision_identity + 1, m_previous.server_close, reason)) return(Fail(reason, reason)); return(true); } ulong Identity(void) const { return(m_current.decision_identity); } long ServerOpen(void) const { return(m_current.server_open); } double Equity(void) const { if(OpenFloor()) return(m_current.state[1]); return(ArraySize(m_state) == 33 ? m_state[1] : m_starting_balance); } double Drawdown(void) const { return(ArraySize(m_state) == 33 && m_state[8] > 0 ? MathMax(0.0, 1.0 - (double)m_state[1] / (double)m_state[8]) : 0.0); } }; //+------------------------------------------------------------------+ //| DPCCValidateZPolicyNet binds native context and policy axes | //+------------------------------------------------------------------+ bool DPCCValidateZPolicyNet(CNet &net, const SDPCCShape &shape, const bool critic, string &reason) { const int query = (critic ? 19 : 13); const int layers = (critic ? 5 : 4); if(shape.K != 21 || shape.V != 9 || shape.H != 12 || shape.D != 16 || shape.NActions != 6 || net.Layer(layers) != NULL || !net.Layer(0) || !net.Layer(layers - 1) || net.Layer(0).getOutput().Total() != query || !dynamic_cast(net.Layer(1)) || net.Layer(1).getOutput().Total() != query || net.Layer(2).getOutput().Total() != 32 || net.Layer(3).getOutput().Total() != (critic ? 32 : 72) || (critic && !dynamic_cast(net.Layer(4)))) { reason = "policy.z.graph_or_axes"; return(false); } for(int i = 0; i < layers; i++) if(!net.Layer(i) || !net.Layer(i).getOutput() || !net.Layer(i).getGradient() || net.Layer(i).getOutput().GetOpenCL() != net.GetOpenCL() || net.Layer(i).getGradient().GetOpenCL() != net.GetOpenCL()) { reason = "policy.z.layer_context"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| DPCCCreateZPolicyNet uses Account13 and grouped contiguous Z | //| keys | //+------------------------------------------------------------------+ bool DPCCCreateZPolicyNet(CNet &net, const SDPCCShape &shape, const bool critic, COpenCLMy *context, string &reason, const bool legacy_actor = false) { if(net.Layer(0) != NULL || !context || shape.K != 21 || shape.V != 9 || shape.H != 12 || shape.D != 16) { reason = "policy.z.destination_or_axes"; return(false); } const int key_units = (int)(shape.K * shape.V); const int key_width = (int)(shape.H * shape.D); if(key_units > context.GetMaxLocalSize(1) || key_units > context.GetMaxWorkGroupSize()) { reason = "policy.z.device_key_limit"; return(false); } CArrayObj descriptions; descriptions.FreeMode(true); const int query = (critic ? 19 : 13); const int layers = (critic ? 5 : 4); for(int i = 0; i < layers; i++) { const uint type = (i == 0 || (critic && i == 3) ? defNeuronBaseOCL : (i == 1 ? defNeuronCrossAttenOCL : (critic && i == 4 ? defNeuronACSRM : (!critic && i == 3 && !legacy_actor ? defNeuronDPCCActorOCL : defNeuronConvOCL)))); CLayerDescription *description = new CLayerDescription; if(!description || !DPCCSetLayerBaseDefaults(description, type, (i == 0 ? query : (critic && i == 3 ? 32 : 1)), 1)) { delete description; reason = "policy.z.description"; return(false); } if(i == 1) { if(ArrayResize(description.windows, 2) < 2 || ArrayResize(description.units, 2) < 2) { delete description; reason = "policy.z.description"; return(false); } description.windows[0] = query; description.windows[1] = key_width; description.units[0] = 1; description.units[1] = key_units; description.window_out = 16; description.step = 1; } else if(type == defNeuronConvOCL || type == defNeuronDPCCActorOCL) { const int width = (i == 2 ? query : 32); description.window = width; description.step = width; description.window_out = (i == 2 ? 32 : 72); description.layers = 1; description.activation = (i == 2 ? GELU : SIGMOID); } else description.window_out = 32; if(!descriptions.Add(description)) { delete description; reason = "policy.z.description_add"; return(false); } } if(!net.Create(GetPointer(descriptions)) || !net.SetOpenCLChecked(context) || !DPCCValidateZPolicyNet(net, shape, critic, reason) || !net.TrainMode(true)) { if(reason == "") reason = "policy.z.create_or_context"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCForwardZPolicy consumes Z without confidence or | //| probabilities | //+------------------------------------------------------------------+ bool DPCCForwardZPolicy(CNet &net, const SDPCCShape &shape, CBufferFloat *z, const float &account[], const float &action[], const bool critic, string &reason) { if(!DPCCValidateZPolicyNet(net, shape, critic, reason) || !z || z.Total() != 36288 || z.GetIndex() < 0 || z.GetOpenCL() != net.GetOpenCL() || ArraySize(account) != 13 || (critic && ArraySize(action) != 6)) { if(reason == "") reason = "policy.z.context_or_query"; return(false); } CBufferFloat query_input; for(int i = 0; i < 13; i++) if(!MathIsValidNumber(account[i]) || !query_input.Add(account[i])) { reason = "policy.z.account"; return(false); } if(critic) for(int i = 0; i < 6; i++) if(!MathIsValidNumber(action[i]) || action[i] < 0 || action[i] > 1 || !query_input.Add(action[i])) { reason = "policy.z.action"; return(false); } if(!net.feedForward(GetPointer(query_input), 1, false, z)) { reason = "policy.z.forward"; return(false); } return(true); } //+------------------------------------------------------------------+ //| DPCCUpdateZCritic fits the AC-SRM distribution to factual return | //+------------------------------------------------------------------+ bool DPCCUpdateZCritic(CNet &critic, const SDPCCShape &shape, CBufferFloat *z, const float reward, const float probability_weight, const float value_weight, string &reason) { if(!DPCCValidateZPolicyNet(critic, shape, true, reason) || !z || z.GetOpenCL() != critic.GetOpenCL() || !MathIsValidNumber(reward) || !MathIsValidNumber(probability_weight) || probability_weight < 0 || !MathIsValidNumber(value_weight) || value_weight < 0 || !critic.WeightsUpdateEnabled()) { if(reason == "") reason = "policy.z.critic_target"; return(false); } CNeuronACSRM *head = dynamic_cast(critic.Layer(4)); if(!head.calcDistributionTargetGradients(critic.Layer(3), reward, probability_weight, value_weight)) { reason = "policy.z.critic_distribution"; return(false); } CBufferFloat detached_gradient; if(!detached_gradient.BufferInit(36288, 0) || !detached_gradient.BufferCreate(critic.GetOpenCL()) || !critic.backPropGradient(z, GetPointer(detached_gradient), -1, true)) { reason = "policy.z.critic_backward"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Native Z online helpers; historical scenario owners remain | //| legacy | //+------------------------------------------------------------------+ bool DPCCZAction(CNet &actor, const SDPCCShape &shape, CBufferFloat *z, const float &account[], float &action[], string &reason) { float unused[]; if(!DPCCForwardZPolicy(actor, shape, z, account, unused, false, reason)) return(false); CBufferFloat *plan = actor.Layer(3).getOutput(); if(!plan || plan.Total() != 72 || !plan.BufferRead() || ArrayResize(action, 6) != 6) {reason = "online.z.action_read"; return(false);} for(int i = 0; i < 6; i++) { action[i] = plan.At(i); if(!MathIsValidNumber(action[i]) || action[i] < 0 || action[i] > 1) {reason = "online.z.action_range"; return(false);} } return(true); } //+------------------------------------------------------------------+ //| Direct Action6 Critic forward and native sigmoid Actor backward | //+------------------------------------------------------------------+ bool DPCCZActorGradient(CNet &actor, CNet &critic, const SDPCCShape &shape, CBufferFloat *z, const SDPCCOnlinePolicyStep &step, const string gradient_mode, string &reason) { if(gradient_mode != DPCC_Z_GRADIENT_MODE) {reason = "BLOCKED.online.explicit_action_gradient_mode_required"; return(false);} if(!DPCCValidateZPolicyNet(actor, shape, false, reason) || !DPCCValidateZPolicyNet(critic, shape, true, reason) || actor.GetOpenCL() != critic.GetOpenCL() || critic.WeightsUpdateEnabled()) {reason = "online.z.actor_gradient_contract"; return(false);} COpenCLMy *context = actor.GetOpenCL(); CNeuronACSRM *head = dynamic_cast(critic.Layer(4)); CBufferFloat value, upstream, detached; if(!value.BufferInit(1, 0) || !value.BufferCreate(context) || !upstream.BufferInit(1, step.actor_upstream) || !upstream.BufferCreate(context) || !detached.BufferInit(36288, 0) || !detached.BufferCreate(context)) {reason = "online.z.actor_gradient_buffers"; return(false);} if(!head.SRMForward(GetPointer(value), 1, step.risk_type, step.risk_parameter, step.mean_weight) || !head.SRMGradient(GetPointer(upstream), 1, step.risk_type, step.risk_parameter, step.mean_weight) || !critic.backPropGradient(z, GetPointer(detached), -1, false)) {reason = "online.z.frozen_srm_backward"; return(false);} CBufferFloat *source = critic.Layer(0).getGradient(); CNeuronDPCCActorOCL *dpcc_head = dynamic_cast(actor.Layer(3)); if(CheckPointer(dpcc_head) != POINTER_INVALID) { if(!dpcc_head.PrepareActionGradient(source, 13, shape.NActions, reason)) return(false); return(actor.backPropGradient(z, GetPointer(detached), -1, true)); } CBufferFloat account_gradient, action_gradient, one, zero, factor; if(!account_gradient.BufferInit(13, 0) || !account_gradient.BufferCreate(context) || !action_gradient.BufferInit(72, 0) || !action_gradient.BufferCreate(context) || !one.BufferInit(72, 1) || !one.BufferCreate(context) || !zero.BufferInit(72, 0) || !zero.BufferCreate(context) || !factor.BufferInit(72, 0) || !factor.BufferCreate(context)) {reason = "online.z.actor_gradient_buffers"; return(false);} CBufferFloat *destination = actor.Layer(3).getGradient(); CBufferFloat *output = actor.Layer(3).getOutput(); //--- The Critic query gradient uses the Actor sigmoid derivative a*(1-a). //--- The query gradient is [Account13,Action6]; later plan slots stay zero. const int arguments[] = {account_gradient.GetIndex(), 13, action_gradient.GetIndex(), 6, account_gradient.GetIndex(), 0, destination.GetIndex(), 0, source.GetIndex() }; for(uint i = 0; i < 9; i++) if(!(i % 2 == 0 ? context.SetArgumentBuffer(def_k_DeconcatenateMatrix, i, arguments[i]) : context.SetArgument(def_k_DeconcatenateMatrix, i, arguments[i]))) {reason = "online.z.actor_gradient_split_argument"; return(false);} uint offset[2] = {0, 0}; uint global[2] = {1, 13}; if(!context.Execute(def_k_DeconcatenateMatrix, 2, offset, global) || !DPCCActorGradientDifferent(context, GetPointer(one), output, destination, 72, reason) || !DPCCActorGradientGate(context, output, GetPointer(zero), destination, GetPointer(factor), 72, reason) || !DPCCActorGradientGate(context, GetPointer(action_gradient), GetPointer(zero), GetPointer(factor), destination, 72, reason) || !actor.backPropGradient(z, GetPointer(detached), -1, true)) {if(reason == "")reason = "online.z.actor_backward"; return(false);} return(true); } //+------------------------------------------------------------------+ //| One-bar TD keeps the observed reward and detached current/next Z | //+------------------------------------------------------------------+ bool DPCCUpdateZOnlinePolicy(CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCShape &shape, CBufferFloat *z, const float &account[], const float &behavior[], CBufferFloat *next_z, const float &next_account[], const string gradient_mode, const SDPCCOnlinePolicyStep &step, SDPCCOnlinePolicyResult &result, string &reason) { ZeroMemory(result); if(gradient_mode != DPCC_Z_GRADIENT_MODE) {reason = "BLOCKED.online.explicit_action_profile_required"; return(false);} float prepared_behavior[]; if(!DPCCPreparePolicyActions(behavior, prepared_behavior, reason)) return(false); CNet *nets[] = {GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; if(!z || !next_z || z == next_z || ArraySize(behavior) != 6 || step.target_tau != 1.0f || step.actor_critic < 0 || step.actor_critic > 1 || step.refresh_critic_target < -1 || step.refresh_critic_target > 1 || !DPCCBuildOneBarTDTargets(step.reward, step.gamma, step.bootstrap_mask, 0, 0, result.y1, result.y2, reason)) {if(reason == "")reason = "online.z.tuple_contract"; return(false);} for(int i = 0; i < 6; i++) { if(!DPCCValidateZPolicyNet(nets[i], shape, i % 3 != 0, reason) || nets[i].GetOpenCL() != actor.GetOpenCL() || nets[i].WeightsUpdateEnabled() != (i < 3)) {reason = "online.z.tuple_graph_or_permission"; return(false);} for(int j = 0; j < i; j++) if(nets[i] == nets[j]) { reason = "online.z.aliased_net"; return(false); } } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(behavior[i]) || behavior[i] < 0 || behavior[i] > 1) {reason = "online.z.behavior_range"; return(false);} CNet *targets[] = {GetPointer(target_q1), GetPointer(target_q2)}; if(step.bootstrap_mask == 1) { float next_action[]; if(!DPCCZAction(target_actor, shape, next_z, next_account, next_action, reason)) return(false); const float random_values[] = {step.sample_u1, step.sample_u2}; float samples[2]; for(int i = 0; i < 2; i++) { if(!DPCCForwardZPolicy(targets[i], shape, next_z, next_account, next_action, true, reason)) return(false); CNeuronACSRM *head = dynamic_cast(targets[i].Layer(4)); CBufferFloat sample, random; const float u = random_values[i]; if(!MathIsValidNumber(u) || u <= 0 || u >= 1 || !sample.BufferInit(1, 0) || !sample.BufferCreate(actor.GetOpenCL()) || !random.BufferInit(1, u) || !random.BufferCreate(actor.GetOpenCL()) || !head.Sample(GetPointer(sample), GetPointer(random), 1) || !sample.BufferRead() || !MathIsValidNumber(sample.At(0))) {reason = "online.z.target_sample"; return(false);} samples[i] = sample.At(0); } result.z1 = samples[0]; result.z2 = samples[1]; } if(!DPCCBuildOneBarTDTargets(step.reward, step.gamma, step.bootstrap_mask, result.z1, result.z2, result.y1, result.y2, reason)) return(false); CNet *critics[] = {GetPointer(q1), GetPointer(q2)}; const float td[] = {result.y1, result.y2}; for(int i = 0; i < 2; i++) { if(!DPCCForwardZPolicy(critics[i], shape, z, account, prepared_behavior, true, reason)) return(false); result.mutation_started = true; if(!DPCCUpdateZCritic(critics[i], shape, z, td[i], step.probability_weight, step.value_weight, reason)) return(false); if(i == 0) result.q1_updated = true; else result.q2_updated = true; } if(step.update_actor) { float action[]; CNet *selected = critics[step.actor_critic]; if(!DPCCZAction(actor, shape, z, account, action, reason)) return(false); if(!DPCCForwardZPolicy(selected, shape, z, account, action, true, reason)) return(false); const bool previous_update = selected.WeightsUpdateEnabled(); if(!previous_update || !selected.SetWeightsUpdate(false)) { selected.SetWeightsUpdate(previous_update); reason = "online.z.critic_freeze"; return(false); } const bool actor_gradient = DPCCZActorGradient(actor, selected, shape, z, step, gradient_mode, reason); const bool critic_restored = selected.SetWeightsUpdate(previous_update); if(!actor_gradient || !critic_restored) { if(!critic_restored) reason = "online.z.critic_restore"; return(false); } result.actor_updated = true; } //--- Native ADAM shadow targets permit exact tau=1 copies, not Polyak. if(step.refresh_actor_target) { if(!target_actor.WeightsUpdate(GetPointer(actor), 1)) {reason = "online.z.target_actor_copy"; return(false);} result.actor_target_refreshed = true; } if(step.refresh_critic_target >= 0) { CNet *source = (step.refresh_critic_target == 0 ? GetPointer(q2) : GetPointer(q1)); if(!targets[step.refresh_critic_target].WeightsUpdate(source, 1)) {reason = "online.z.crossed_target_copy"; return(false);} result.critic_target_refreshed = true; } return(true); } //+------------------------------------------------------------------+ //| Read a bounded pinned model manifest and canonical framed fields | //+------------------------------------------------------------------+ bool DPCCZReadPinnedText(const string path, const string expected, string &source, string &reason, const bool common_files = false) { string canonical = "", observed = ""; uchar bytes[]; if(!DPCCResolverLowerSha256(expected) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path || !DPCCReadArtifactBytes(path, common_files, 65536, bytes, reason) || !DPCCSHA256Lower(bytes, observed, reason) || observed != expected || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 65536, source, reason)) {reason = "online.z.manifest_sha_or_encoding"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCZReadField | //+------------------------------------------------------------------+ bool DPCCZReadField(const string wire, int &cursor, string &value, string &reason) { const int colon = StringFind(wire, ":", cursor); ulong length = 0; if(colon < cursor || !DPCCCheckpointReadUnsigned(StringSubstr(wire, cursor, colon - cursor), 65536, length) || colon + 1 + (int)length >= StringLen(wire) || StringSubstr(wire, colon + 1 + (int)length, 1) != "\n") {reason = "online.z.manifest_field"; return(false);} value = StringSubstr(wire, colon + 1, (int)length); cursor = colon + 2 + (int)length; return(true); } //+------------------------------------------------------------------+ //| Recognize the compact policy artifact path suffix | //+------------------------------------------------------------------+ bool DPCCPolicyCompactPath(const string path) { const int length = StringLen(path); return(length > 5 && StringSubstr(path, length - 5) == ".dppt"); } //+------------------------------------------------------------------+ //| Save immutable compact CNet training state and hash file bytes | //+------------------------------------------------------------------+ bool DPCCSaveCompactPolicyTrainingState(CNet &net, const string path, const bool common, const ulong max_bytes, string &hash, string &reason, const bool overwrite = false) { //--- Admit the graph structure before creating an immutable artifact. hash = ""; reason = ""; string canonical = ""; if(max_bytes == 0 || max_bytes > (ulong)INT_MAX || !DPCCPolicyCompactPath(path) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path) { reason = "policy.compact.save_path_or_bound"; return(false); } ulong graph_fingerprint = 0; if(!net.DPCCPolicyTrainingStateFingerprint(graph_fingerprint)) { reason = "policy.compact.save_graph"; return(false); } if(!overwrite && FileIsExist(path, (common ? FILE_COMMON : 0))) { reason = "policy.compact.immutable_artifact"; return(false); } const int handle = FileOpen(path, FILE_WRITE | FILE_BIN | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "policy.compact.save_open"; return(false); } //--- CNet owns the complete state block; retain partial artifacts on failure. ResetLastError(); const bool written = net.SaveDPCCPolicyTrainingStateOnly(handle); FileFlush(handle); const int write_error = GetLastError(); const ulong size = FileSize(handle); const ulong end = FileTell(handle); FileClose(handle); if(!written || write_error != 0 || size == 0 || size > max_bytes || end != size) { reason = "policy.compact.save_block"; return(false); } if(!DPCCCheckpointFileSHA(path, common, max_bytes, hash, reason)) return(false); reason = ""; return(true); } //+------------------------------------------------------------------+ //| Load authenticated compact state into an admitted CNet graph | //+------------------------------------------------------------------+ bool DPCCLoadCompactPolicyTrainingState(CNet &net, const string path, const string hash, const bool common, const ulong max_bytes, string &reason) { //--- Admit the destination graph and authenticate the opened file before apply. reason = ""; string canonical = "", observed = ""; ulong fingerprint = 0; if(max_bytes == 0 || max_bytes > (ulong)INT_MAX || !DPCCPolicyCompactPath(path) || !DPCCResolverLowerSha256(hash) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path) { reason = "policy.compact.load_path_or_bound"; return(false); } if(!net.DPCCPolicyTrainingStateFingerprint(fingerprint)) { reason = "policy.compact.load_graph"; return(false); } const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "policy.compact.load_open"; return(false); } bool loaded = false; do { const ulong size = FileSize(handle); uchar bytes[]; if(size == 0 || size > max_bytes || ArrayResize(bytes, (int)size) != (int)size) { reason = "policy.compact.load_size"; break; } ResetLastError(); const uint read = FileReadArray(handle, bytes, 0, (uint)size); if(read != (uint)size || GetLastError() != 0 || FileTell(handle) != size || FileSize(handle) != size) { reason = "policy.compact.load_bytes"; break; } if(!DPCCSHA256Lower(bytes, observed, reason)) break; if(observed != hash) { reason = "policy.compact.load_sha"; break; } if(!FileSeek(handle, 0, SEEK_SET)) { reason = "policy.compact.load_seek"; break; } //--- CNet validates the full block and EOF before applying model state. if(!net.LoadDPCCPolicyTrainingStateOnly(handle)) { reason = "policy.compact.load_block"; break; } if(FileTell(handle) != size || FileSize(handle) != size) { reason = "policy.compact.load_eof"; break; } loaded = true; } while(false); FileClose(handle); return(loaded); } //+------------------------------------------------------------------+ //| Load compact Z policy state into a freshly constructed graph | //+------------------------------------------------------------------+ bool DPCCZLoadCompact(CNet &net, const SDPCCShape &shape, const bool critic, COpenCLMy *context, const string path, const string hash, const bool training, string &reason, const bool common_files = false) { const int header = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common_files ? FILE_COMMON : 0)); if(header == INVALID_HANDLE) { reason = "policy.compact.graph_header_open"; return(false); } ResetLastError(); const int magic = FileReadInteger(header, INT_VALUE); const int version = FileReadInteger(header, INT_VALUE); const int graph_kind = FileReadInteger(header, INT_VALUE); const int layer_count = FileReadInteger(header, INT_VALUE); const int read_error = GetLastError(); FileClose(header); if(read_error != 0 || magic != 0x44505054 || version != 1 || layer_count != (critic ? 5 : 4) || (critic ? graph_kind != 4 : (graph_kind != 3 && graph_kind != 5))) { reason = "policy.compact.graph_header"; return(false); } if(!DPCCCreateZPolicyNet(net, shape, critic, context, reason, !critic && graph_kind == 3)) return(false); if(!DPCCLoadCompactPolicyTrainingState(net, path, hash, common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, reason)) return(false); if(!DPCCValidateZPolicyNet(net, shape, critic, reason)) return(false); if(!net.Clear()) { reason = "online.z.compact_load_clear"; return(false); } if(!net.TrainMode(training)) { reason = "online.z.compact_load_train_mode"; return(false); } if(!net.SetWeightsUpdate(training)) { reason = "online.z.compact_load_weights_update"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Load native policy weights and Adam, then clear transient | //| buffers | //+------------------------------------------------------------------+ bool DPCCZLoadNative(CNet &net, const SDPCCShape &shape, const bool critic, COpenCLMy *context, const string path, const string hash, const bool training, string &reason, const bool common_files = false) { string canonical = "", observed = ""; if(!DPCCResolverLowerSha256(hash) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || canonical != path || !DPCCCheckpointFileSHA(path, common_files, 268435456, observed, reason) || observed != hash || !DPCCCreateZPolicyNet(net, shape, critic, context, reason)) return(false); const int handle = FileOpen(path, FILE_READ | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "online.z.native_load_open"; return(false); } //--- CNet::Load closes the checkpoint handle after consuming it; do not reuse it. const bool loaded = net.Load(handle); if(!loaded) {reason = "online.z.native_load"; return(false);} if(!net.SetOpenCLChecked(context)) {reason = "online.z.native_load_set_opencl"; return(false);} if(!DPCCValidateZPolicyNet(net, shape, critic, reason)) {if(reason == "")reason = "online.z.native_load_validate"; return(false);} if(!net.Clear()) {reason = "online.z.native_load_clear"; return(false);} if(!net.TrainMode(training)) {reason = "online.z.native_load_train_mode"; return(false);} if(!net.SetWeightsUpdate(training)) {reason = "online.z.native_load_weights_update"; return(false);} if(!DPCCCheckpointFileSHA(path, common_files, 268435456, observed, reason)) {if(reason == "")reason = "online.z.native_postload_sha_read"; return(false);} if(observed != hash) {reason = "online.z.native_postload_sha_mismatch expected=" + hash + " observed=" + observed; return(false);} return(true); } //+------------------------------------------------------------------+ //| Study export supplies models only; environment remains the | //| Tester | //+------------------------------------------------------------------+ bool DPCCZDecodeStudyModelsEx(const string wire, const string raw_sha, const string encoder_receipt_sha, const string encoder_sha, string &paths[], string &hashes[], bool &compact, string &reason) { ArrayResize(paths, 0); ArrayResize(hashes, 0); compact = false; const bool compact_wire = (StringFind(wire, "DPCC_Z_ORACLE_MODELS_V7\n") == 0); bool server_calendar = (StringFind(wire, "DPCC_Z_ORACLE_MODELS_V6\n") == 0); const string version = (compact_wire ? "V7" : (server_calendar ? "V6" : "V5")); const string header = "DPCC_Z_ORACLE_MODELS_" + version + "\n"; const string suffix = (compact_wire ? "COMPACT_CNET_POLICY_TRAINING_STATE_V1\n" : "NATIVE_CNET_MODELS_WITH_ADAM\n") + "REBUILD_TRANSIENT_STATE_ON_LOAD\n" + "FINANCIAL_STATE_AND_POSITION_NOT_SAVED\nMODELS_ONLY_NOT_RESUMABLE_LEDGER_CHECKPOINT\n" + "NOT_A_READINESS_AUTHORIZATION\nEND_DPCC_Z_ORACLE_MODELS_" + version + "\n"; if(StringSubstr(wire, 0, StringLen(header)) != header) {reason = "online.z.study_model_manifest_version"; return(false);} int field_count = (server_calendar ? 29 : 34); string fields[]; if(ArrayResize(fields, 34) != 34) {reason = "online.z.study_fields_capacity"; return(false);} int cursor = StringLen(header); if(compact_wire) { //--- V7 preserves the existing server-calendar and UTC provenance layouts. field_count = 0; while(field_count < 34 && StringSubstr(wire, cursor) != suffix) { if(!DPCCZReadField(wire, cursor, fields[field_count], reason)) return(false); field_count++; } if(field_count != 29 && field_count != 34) {reason = "online.z.study_compact_field_count"; return(false);} server_calendar = (field_count == 29); } else for(int i = 0; i < field_count; i++) if(!DPCCZReadField(wire, cursor, fields[i], reason)) return(false); const int receipt_sha_index = (server_calendar ? 11 : 17); const int encoder_sha_index = (server_calendar ? 13 : 19); const int action_index = (server_calendar ? 21 : 27); const int models_index = (server_calendar ? 23 : 28); if(StringSubstr(wire, cursor) != suffix || fields[0] != "provenance=ADMISSION_TIME_SOURCE_SHA256_LOADED_IN_MEMORY" || fields[1] != "Z[21,9,12,16];contiguous_group=[K*V=189,H*D=192];no_copy" || fields[receipt_sha_index] != "encoder_receipt.sha256=" + encoder_receipt_sha || fields[encoder_sha_index] != "trained_encoder_model.sha256=" + encoder_sha) {reason = "online.z.study_model_encoder_binding"; return(false);} if(server_calendar && fields[22] != "time_domain=SERVER_CALENDAR_SECONDS_V1") {reason = "online.z.study_server_calendar_domain"; return(false);} bool raw_bound = false; for(int i = 2; i <= encoder_sha_index; i += 2) { const int marker = StringFind(fields[i + 1], ".sha256="); if(marker <= 0 || !DPCCResolverLowerSha256(StringSubstr(fields[i + 1], marker + 8))) {reason = "online.z.study_source_pin_encoding"; return(false);} if(fields[i + 1] == "raw_history.sha256=" + raw_sha) raw_bound = true; } if(fields[action_index] != "action_contract=" + DPCC_Z_ACTION_CONTRACT) {reason = "BLOCKED.online.z.direct_lots_action_contract_required"; return(false);} if(!raw_bound || ArrayResize(paths, 3) != 3 || ArrayResize(hashes, 3) != 3) {reason = "online.z.study_raw_binding_or_capacity"; return(false);} for(int i = 0; i < 3; i++) { paths[i] = fields[models_index + i * 2]; hashes[i] = fields[models_index + 1 + i * 2]; string canonical = ""; const string extension = (compact_wire ? ".dppt" : ".nnw"); const int length = StringLen(paths[i]); if(length <= StringLen(extension) || StringSubstr(paths[i], length - StringLen(extension)) != extension || !DPCCCheckpointCanonicalPath(paths[i], canonical, reason) || canonical != paths[i] || !DPCCResolverLowerSha256(hashes[i])) { ArrayResize(paths, 0); ArrayResize(hashes, 0); reason = "online.z.study_model_format_or_pin"; return(false); } } compact = compact_wire; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCZReadStudyModelsEx | //+------------------------------------------------------------------+ bool DPCCZReadStudyModelsEx(const string path, const string expected, const string raw_sha, const string encoder_receipt_sha, const string encoder_sha, string &paths[], string &hashes[], bool &compact, string &reason, const bool common_files = false) { ArrayResize(paths, 0); ArrayResize(hashes, 0); compact = false; string wire = ""; if(!DPCCZReadPinnedText(path, expected, wire, reason, common_files)) return(false); return(DPCCZDecodeStudyModelsEx(wire, raw_sha, encoder_receipt_sha, encoder_sha, paths, hashes, compact, reason)); } //+------------------------------------------------------------------+ //| Legacy Study APIs admit only native V5/V6 model manifests | //+------------------------------------------------------------------+ bool DPCCZDecodeStudyModels(const string wire, const string raw_sha, const string encoder_receipt_sha, const string encoder_sha, string &paths[], string &hashes[], string &reason) { bool compact = false; if(!DPCCZDecodeStudyModelsEx(wire, raw_sha, encoder_receipt_sha, encoder_sha, paths, hashes, compact, reason)) return(false); if(compact) { ArrayResize(paths, 0); ArrayResize(hashes, 0); reason = "online.z.study_compact_dispatch_required"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCZReadStudyModels | //+------------------------------------------------------------------+ bool DPCCZReadStudyModels(const string path, const string expected, const string raw_sha, const string encoder_receipt_sha, const string encoder_sha, string &paths[], string &hashes[], string &reason) { ArrayResize(paths, 0); ArrayResize(hashes, 0); string wire = ""; if(!DPCCZReadPinnedText(path, expected, wire, reason)) return(false); return(DPCCZDecodeStudyModels(wire, raw_sha, encoder_receipt_sha, encoder_sha, paths, hashes, reason)); } //+------------------------------------------------------------------+ //| Tester observation keeps one owned direction and opening age | //+------------------------------------------------------------------+ bool DPCCTesterOnly(string &reason) { if(!MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION)) { reason = "BLOCKED.online.Strategy_Tester_only_no_optimization"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Admit recognized Tester account modes for one-direction | //| execution | //+------------------------------------------------------------------+ bool DPCCRequireTesterOneDirection(string &reason) { if(!DPCCTesterOnly(reason)) return(false); const long margin_mode = AccountInfoInteger(ACCOUNT_MARGIN_MODE); if(margin_mode != ACCOUNT_MARGIN_MODE_RETAIL_NETTING && margin_mode != ACCOUNT_MARGIN_MODE_RETAIL_HEDGING && margin_mode != ACCOUNT_MARGIN_MODE_EXCHANGE) { reason = "BLOCKED.online.unrecognized_account_margin_mode"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Collect stable owned tickets and their single-direction exposure | //+------------------------------------------------------------------+ bool DPCCCollectTesterPositions(const string symbol, const long magic, SDPCCPhysicalPosition &positions[], double &lots, bool &buy, string &reason, const bool common_protection = true) { //--- Capture only an isolated Tester account without outstanding orders. lots = 0; buy = false; ArrayFree(positions); if(!DPCCRequireTesterOneDirection(reason)) return(false); const int total = PositionsTotal(); const long mode = AccountInfoInteger(ACCOUNT_MARGIN_MODE); const datetime asof = TimeCurrent(); const double step = SymbolInfoDouble(symbol, SYMBOL_VOLUME_STEP); if(symbol == "" || magic <= 0 || total < 0 || OrdersTotal() != 0 || asof <= 0 || !MathIsValidNumber(step) || step <= 0 || (mode != ACCOUNT_MARGIN_MODE_RETAIL_HEDGING && total > 1)) { reason = "BLOCKED.online.tester.isolated_one_direction_account_required"; return(false); } if(ArrayResize(positions, total) != total) { reason = "online.tester.position_capacity"; return(false); } //--- Reuse whole-identifier ownership proof without physical financial admission. SDPCCPhysicalPolicy owner; ZeroMemory(owner); owner.symbol = symbol; owner.magic = magic; owner.max_history_deals = (uint)INT_MAX; owner.volume_atol = step * 0.0000001; for(int i = 0; i < total; i++) { const ulong ticket = PositionGetTicket(i); string position_symbol = ""; if(ticket == 0 || !PositionGetString(POSITION_SYMBOL, position_symbol) || position_symbol != symbol || !DPCCPhysicalReadPosition(owner, ticket, asof, positions[i], reason)) { if(reason == "") reason = "BLOCKED.online.tester.foreign_or_unconfirmed_position"; return(false); } const SDPCCPhysicalPosition item = positions[i]; const double values[] = {item.volume, item.open_price, item.sl, item.tp, item.profit, item.swap}; for(int j = 0; j < ArraySize(values); j++) if(!MathIsValidNumber(values[j])) { reason = "BLOCKED.online.tester.position_nonfinite"; return(false); } if(!item.history_owned || item.magic != magic || item.identifier == 0 || item.volume <= 0 || item.open_price <= 0 || item.sl < 0 || item.tp < 0 || item.opened_msc <= 0 || item.updated_msc < item.opened_msc || item.updated_msc / 1000 > (long)asof || (item.type != POSITION_TYPE_BUY && item.type != POSITION_TYPE_SELL)) { reason = "BLOCKED.online.tester.position_identity_or_geometry"; return(false); } for(int j = 0; j < i; j++) if(item.ticket == positions[j].ticket || item.identifier == positions[j].identifier) { reason = "BLOCKED.online.tester.duplicate_position"; return(false); } if(i > 0 && (item.type != positions[0].type || (common_protection && (item.sl != positions[0].sl || item.tp != positions[0].tp)))) { reason = "BLOCKED.online.tester.mixed_direction_or_protection"; return(false); } lots += item.volume; if(!MathIsValidNumber(lots)) { reason = "BLOCKED.online.tester.aggregate_volume"; return(false); } } //--- Restore complete Tester history after identifier-scoped ownership reads. if(!HistorySelect(0, asof)) { reason = "BLOCKED.online.tester.history_scope_restore"; return(false); } //--- Confirm the entire ticket set before publishing the aggregate. if(PositionsTotal() != total || OrdersTotal() != 0 || TimeCurrent() != asof || AccountInfoInteger(ACCOUNT_MARGIN_MODE) != mode) { reason = "BLOCKED.online.tester.concurrent_account_change"; return(false); } for(int i = 0; i < total; i++) { const SDPCCPhysicalPosition item = positions[i]; if(!PositionSelectByTicket(item.ticket) || PositionGetString(POSITION_SYMBOL) != symbol || (ulong)PositionGetInteger(POSITION_IDENTIFIER) != item.identifier || PositionGetInteger(POSITION_MAGIC) != item.magic || PositionGetInteger(POSITION_TYPE) != item.type || PositionGetInteger(POSITION_TIME_MSC) != item.opened_msc || PositionGetInteger(POSITION_TIME_UPDATE_MSC) != item.updated_msc || PositionGetDouble(POSITION_VOLUME) != item.volume || PositionGetDouble(POSITION_PRICE_OPEN) != item.open_price || PositionGetDouble(POSITION_SL) != item.sl || PositionGetDouble(POSITION_TP) != item.tp || PositionGetDouble(POSITION_PROFIT) != item.profit || PositionGetDouble(POSITION_SWAP) != item.swap) { reason = "BLOCKED.online.tester.concurrent_position_change"; return(false); } } //--- Preserve millisecond opening order without inventing a tie-break priority. for(int i = 1; i < total; i++) { const SDPCCPhysicalPosition item = positions[i]; int at = i; while(at > 0 && positions[at - 1].opened_msc > item.opened_msc) { positions[at] = positions[at - 1]; at--; } positions[at] = item; } buy = (total > 0 && positions[0].type == POSITION_TYPE_BUY); reason = ""; return(true); } //+------------------------------------------------------------------+ //| Refuse a close when millisecond FIFO priority cannot be | //| confirmed | //+------------------------------------------------------------------+ bool DPCCRequireTesterFIFO(const SDPCCPhysicalPosition &positions[], string &reason) { for(int i = 1; i < ArraySize(positions); i++) if(positions[i - 1].opened_msc >= positions[i].opened_msc) { reason = "BLOCKED.online.tester.FIFO_open_time_tie_or_order"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Recognize observed no-fill feasibility results | //+------------------------------------------------------------------+ bool DPCCExpectedTesterNoFill(const uint retcode) { return(retcode == TRADE_RETCODE_NO_MONEY || retcode == TRADE_RETCODE_MARKET_CLOSED || retcode == TRADE_RETCODE_INVALID_STOPS || retcode == TRADE_RETCODE_INVALID_VOLUME); } //+------------------------------------------------------------------+ //| Transient first-open age of one-direction owned exposure | //+------------------------------------------------------------------+ struct SDPCCTesterExposureAge { int direction; long first_open; }; //+------------------------------------------------------------------+ //| Stage exposure age and aggregate-profit discount | //+------------------------------------------------------------------+ bool DPCCStageTesterExposureAge(const SDPCCTesterExposureAge &previous, const int direction, const long earliest_open, const long current, const double profit, SDPCCTesterExposureAge &next, double &discount, string &reason) { //--- Reject invalid observations before staging a new age. discount = 0; if(current <= 0 || !MathIsValidNumber(profit) || direction < -1 || direction > 1 || previous.direction < -1 || previous.direction > 1 || (previous.direction == 0 && previous.first_open != 0) || (previous.direction != 0 && (previous.first_open <= 0 || previous.first_open > current)) || (direction != 0 && (earliest_open <= 0 || earliest_open > current))) { reason = "BLOCKED.online.position_exposure_age"; return(false); } //--- Keep the initial open through additions and partial reductions. next = previous; if(direction == 0) ZeroMemory(next); else if(previous.direction != direction) { next.direction = direction; next.first_open = earliest_open; } if(direction != 0) discount = (double)(current - next.first_open) / 36000.0 * MathAbs(profit); if(!MathIsValidNumber(discount)) { reason = "BLOCKED.online.position_exposure_discount"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Build Account13 and publish age only after successful | //| observation | //+------------------------------------------------------------------+ bool DPCCObserveTesterAccount(const string symbol, const long magic, const double previous_balance, const double previous_equity, SDPCCTesterExposureAge &age, float &account[], string &reason) { //--- Collect the same owned positions used for execution. SDPCCPhysicalPosition positions[]; double lots = 0; bool direction = false; if(!DPCCCollectTesterPositions(symbol, magic, positions, lots, direction, reason)) return(false); double buy = 0, sell = 0, buy_profit = 0, sell_profit = 0, discount = 0; const long current = (long)TimeCurrent(); long earliest_open = 0; for(int i = 0; i < ArraySize(positions); i++) { const SDPCCPhysicalPosition item = positions[i]; const int shift = iBarShift(symbol, PERIOD_H1, (datetime)(item.opened_msc / 1000), false); const long first_open = (long)iTime(symbol, PERIOD_H1, shift); if(shift < 0 || first_open <= 0 || first_open > current) { reason = "BLOCKED.online.position_first_open_bar"; return(false); } if(earliest_open == 0 || first_open < earliest_open) earliest_open = first_open; if(item.type == POSITION_TYPE_BUY) { buy += item.volume; buy_profit += item.profit; } else { sell += item.volume; sell_profit += item.profit; } } //--- Stage the exposure age without mutating the caller on failure. SDPCCTesterExposureAge next; const int observed_direction = (ArraySize(positions) == 0 ? 0 : (direction ? 1 : -1)); if(!DPCCStageTesterExposureAge(age, observed_direction, earliest_open, current, buy_profit + sell_profit, next, discount, reason)) return(false); if(!DPCCBuildAccount13Features(AccountInfoDouble(ACCOUNT_BALANCE), previous_balance, AccountInfoDouble(ACCOUNT_EQUITY), previous_equity, buy, sell, buy_profit, sell_profit, discount, current, account, reason)) return(false); //--- Publish only a successfully built Account13 observation. age = next; return(true); } //+------------------------------------------------------------------+ //| Capture one CLOSED bar using the same Forecast feature channels | //+------------------------------------------------------------------+ bool DPCCCaptureTesterClosedRow(const string symbol, const int &handles[], const int shift, SDPCCClosedHistoryRow &row, string &reason) { ZeroMemory(row); if(!DPCCTesterOnly(reason) || shift < 1 || ArraySize(handles) != 4) return(false); MqlRates rates[]; if(CopyRates(symbol, PERIOD_H1, shift, 1, rates) != 1) {reason = "online.z.tester_closed_rates"; return(false);} double values[5]; for(int i = 0; i < 5; i++) { double value[]; const int handle = (i < 3 ? handles[i] : handles[3]); if(handle == INVALID_HANDLE || CopyBuffer(handle, (i == 4 ? 1 : 0), shift, 1, value) != 1 || !MathIsValidNumber(value[0]) || value[0] == EMPTY_VALUE) {reason = "online.z.tester_closed_indicator"; return(false);} values[i] = value[0]; } row.ordinal = 0; row.successor_open_server = (long)iTime(symbol, PERIOD_H1, shift - 1); row.spread = rates[0].spread; row.real_volume = rates[0].real_volume; row.bar.server_open = (long)rates[0].time; row.bar.tick_volume = rates[0].tick_volume; row.bar.spread = rates[0].spread; row.bar.ohlc[0] = rates[0].open; row.bar.ohlc[1] = rates[0].high; row.bar.ohlc[2] = rates[0].low; row.bar.ohlc[3] = rates[0].close; for(int i = 0; i < 5; i++) row.bar.indicators[i] = (float)values[i]; float checked[]; if(row.successor_open_server <= row.bar.server_open || !DPCCStudyForecastFeature(row.bar, checked)) {reason = "online.z.tester_closed_geometry"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Capture a chronological closed-H1 replay, linked to decision | //| open | //+------------------------------------------------------------------+ bool DPCCCaptureTesterClosedHistory(const string symbol, const int &handles[], const long decision_open, const bool warmup, SDPCCStudyForecastBar &bars[], string &reason) { const int count = (warmup ? DPCC_ENCODER_WARMUP_HISTORY : DPCC_ENCODER_INPUT_WINDOW); ArrayResize(bars, 0); if(!DPCCTesterOnly(reason) || symbol == "" || decision_open <= 0 || ArrayResize(bars, count) != count || iTime(symbol, PERIOD_H1, 0) != decision_open) {reason = "online.z.tester_history.arguments_or_decision_open"; return(false);} long previous_successor = 0; for(int i = 0; i < count; i++) { SDPCCClosedHistoryRow row; const int shift = count - i; if(!DPCCCaptureTesterClosedRow(symbol, handles, shift, row, reason)) {ArrayResize(bars, 0); return(false);} const long expected_successor = (i + 1 < count ? (long)iTime(symbol, PERIOD_H1, shift - 1) : decision_open); if(row.successor_open_server != expected_successor || row.bar.server_open >= decision_open || (i > 0 && (bars[i - 1].server_open >= row.bar.server_open || previous_successor != row.bar.server_open))) {ArrayResize(bars, 0); reason = StringFormat("online.z.tester_history.chronology:%d", i); return(false);} previous_successor = row.successor_open_server; bars[i] = row.bar; } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Keep partial fills distinct from complete or ambiguous requests | //+------------------------------------------------------------------+ bool DPCCSendTesterRequest(MqlTradeRequest &request, bool &accepted, bool &partial, string &reason) { accepted = false; partial = false; if(!DPCCRequireTesterOneDirection(reason)) return(false); MqlTradeCheckResult check; MqlTradeResult result; ZeroMemory(check); ZeroMemory(result); ResetLastError(); const bool checked = OrderCheck(request, check); if(!checked || check.retcode != 0) { Print("DPCC_TESTER_ORDER_REJECT check=", check.retcode, " ", check.comment); if(DPCCExpectedTesterNoFill(check.retcode)) return(true); reason = "online.tester.order_check:" + (string)check.retcode + ":" + (string)GetLastError(); return(false); } if(!DPCCRequireTesterOneDirection(reason)) return(false); ResetLastError(); const bool sent = OrderSend(request, result); if(sent && (result.retcode == TRADE_RETCODE_DONE || (request.action == TRADE_ACTION_SLTP && result.retcode == TRADE_RETCODE_NO_CHANGES))) { accepted = true; return(true); } if(sent && request.action == TRADE_ACTION_DEAL && result.retcode == TRADE_RETCODE_DONE_PARTIAL) { Print("DPCC_TESTER_ORDER_PARTIAL action=", (int)request.action, " requested_volume=", request.volume, " filled_volume=", result.volume, " retcode=", result.retcode, " deal=", result.deal, " order=", result.order); accepted = true; partial = true; return(true); } if(DPCCExpectedTesterNoFill(result.retcode)) return(true); reason = "BLOCKED.online.tester.ambiguous_order_result:" + (string)result.retcode + ":" + (string)GetLastError(); return(false); } //+------------------------------------------------------------------+ //| Preserve the existing diagnostic call without a partial retry | //+------------------------------------------------------------------+ bool DPCCSendTesterRequest(MqlTradeRequest &request, bool &accepted, string &reason) { bool partial = false; if(!DPCCSendTesterRequest(request, accepted, partial, reason)) return(false); if(partial) { reason = "BLOCKED.online.tester.partial_requires_ticket_reconciliation"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Limit one Tester deal without changing desired net exposure | //+------------------------------------------------------------------+ double DPCCCapTesterDealVolume(const double requested, const double minimum, const double maximum, const double step) { if(!MathIsValidNumber(requested) || !MathIsValidNumber(minimum) || !MathIsValidNumber(maximum) || !MathIsValidNumber(step) || minimum <= 0 || maximum < minimum || step <= 0) return(0); //--- Permit only floating-point noise at min/max and grid boundaries. const double scale = MathMax(MathMax(MathAbs(requested), MathAbs(minimum)), MathMax(MathAbs(maximum), MathAbs(step))); const double tolerance = MathMin(16.0 * 2.2204460492503131e-16 * scale, step * 0.0000001); if(requested < minimum && minimum - requested > tolerance) return(0); const double normalized = (requested < minimum ? minimum : requested); const double limited = MathMin(normalized, maximum); const double raw_steps = (limited - minimum) / step; const double nearest_steps = MathRound(raw_steps); const double step_tolerance = tolerance / step; const double steps = (MathAbs(raw_steps - nearest_steps) <= step_tolerance ? nearest_steps : MathFloor(raw_steps)); if(steps < 0) return(0); double capped = minimum + steps * step; if(capped < minimum || capped - maximum > tolerance) return(0); return(MathMin(capped, maximum)); } //+------------------------------------------------------------------+ //| Broker grid maps the direct signed sigmoid lots to a net target | //+------------------------------------------------------------------+ bool DPCCBuildTesterPositionTarget(const double desired, const double minimum, const double step, double &signed_target, string &reason) { signed_target = 0; reason = ""; if(!MathIsValidNumber(desired) || MathAbs(desired) > 1 || !MathIsValidNumber(minimum) || minimum <= 0 || !MathIsValidNumber(step) || step <= 0) {reason = "BLOCKED.online.tester.direct_lots_and_grid_required"; return(false);} const double magnitude = MathAbs(desired); if(magnitude < minimum) return(true); const double rounded = minimum + MathRound((magnitude - minimum) / step) * step; if(!MathIsValidNumber(rounded)) {reason = "BLOCKED.online.tester.position_target_numeric"; return(false);} //--- Preserve nearest-grid targeting inside the sigmoid representable range. const double target = DPCCCapTesterDealVolume(rounded, minimum, 1.0, step); signed_target = (desired > 0 ? target : -target); return(true); } //+------------------------------------------------------------------+ //| Map Action6 protection prices before any Tester request | //+------------------------------------------------------------------+ bool DPCCBuildTesterActionProtection(const float &action[], const bool buy, const double bid, const double ask, const double point, const double grid, const int digits, const double stop, const double tp_cap_points, const double sl_cap_points, const double target, double &sl, double &tp, string &reason) { //--- Validate request geometry before deriving any active protection price. sl = 0; tp = 0; reason = ""; if(ArraySize(action) != 6 || !MathIsValidNumber(target) || target < 0 || !MathIsValidNumber(bid) || !MathIsValidNumber(ask) || bid <= 0 || ask < bid || !MathIsValidNumber(point) || point <= 0 || !MathIsValidNumber(grid) || grid <= 0 || digits < 0 || digits > 8 || !MathIsValidNumber(stop) || stop < 0 || !MathIsValidNumber(tp_cap_points) || tp_cap_points <= 0 || !MathIsValidNumber(sl_cap_points) || sl_cap_points <= 0) { reason = "BLOCKED.online.tester.protection_properties"; return(false); } if(target == 0) return(true); const double tp_coefficient = (double)action[buy ? 1 : 4]; const double sl_coefficient = (double)action[buy ? 2 : 5]; if(!MathIsValidNumber(tp_coefficient) || tp_coefficient < 0 || tp_coefficient > 1 || !MathIsValidNumber(sl_coefficient) || sl_coefficient < 0 || sl_coefficient > 1) { reason = "online.tester.protection_coefficient"; return(false); } //--- Zero coefficients disable protection; active prices use the broker tick grid. const double reference = (buy ? ask : bid); const double trigger = (buy ? bid : ask); double candidate_sl = 0; double candidate_tp = 0; if(sl_coefficient > 0) { const double distance = sl_coefficient * sl_cap_points * point; candidate_sl = NormalizeDouble(MathRound((reference + (buy ? -distance : distance)) / grid) * grid, digits); if(!MathIsValidNumber(distance) || !MathIsValidNumber(candidate_sl) || candidate_sl <= 0 || !MathIsValidNumber(candidate_sl / grid) || MathAbs(candidate_sl - MathRound(candidate_sl / grid)*grid) > grid * 0.0000001 || (buy ? trigger - candidate_sl : candidate_sl - trigger) < stop) { reason = "online.tester.invalid_active_sl"; return(false); } } if(tp_coefficient > 0) { const double distance = tp_coefficient * tp_cap_points * point; candidate_tp = NormalizeDouble(MathRound((reference + (buy ? distance : -distance)) / grid) * grid, digits); if(!MathIsValidNumber(distance) || !MathIsValidNumber(candidate_tp) || candidate_tp <= 0 || !MathIsValidNumber(candidate_tp / grid) || MathAbs(candidate_tp - MathRound(candidate_tp / grid)*grid) > grid * 0.0000001 || (buy ? candidate_tp - trigger : trigger - candidate_tp) < stop) { reason = "online.tester.invalid_active_tp"; return(false); } } //--- Publish both request prices only after the active protections are admissible. sl = candidate_sl; tp = candidate_tp; return(true); } //+------------------------------------------------------------------+ //| Confirm only the requested ticket changed during a close | //+------------------------------------------------------------------+ bool DPCCConfirmTesterClose(const SDPCCPhysicalPosition &before[], const SDPCCPhysicalPosition &after[], const ulong ticket, const double requested, const bool partial, string &reason) { double closed = 0; bool found = false; for(int i = 0; i < ArraySize(before); i++) { int at = -1; for(int j = 0; j < ArraySize(after); j++) if(before[i].ticket == after[j].ticket) { at = j; break; } if(before[i].ticket != ticket && at < 0) { reason = "BLOCKED.online.tester.unrequested_ticket_closed"; return(false); } if(at >= 0 && (before[i].identifier != after[at].identifier || before[i].type != after[at].type || before[i].opened_msc != after[at].opened_msc || before[i].updated_msc > after[at].updated_msc || before[i].open_price != after[at].open_price || before[i].sl != after[at].sl || before[i].tp != after[at].tp)) { reason = "BLOCKED.online.tester.close_position_binding_changed"; return(false); } if(before[i].ticket == ticket) { closed = before[i].volume - (at >= 0 ? after[at].volume : 0); found = true; } else if(before[i].volume != after[at].volume || before[i].updated_msc != after[at].updated_msc) { reason = "BLOCKED.online.tester.unrequested_ticket_changed"; return(false); } } for(int i = 0; i < ArraySize(after); i++) { bool known = false; for(int j = 0; j < ArraySize(before); j++) if(after[i].ticket == before[j].ticket) { known = true; break; } if(!known) { reason = "BLOCKED.online.tester.close_created_position"; return(false); } } const double tolerance = DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, requested); if(!found || !MathIsValidNumber(closed) || closed <= 0 || closed > requested + tolerance || (!partial && MathAbs(closed - requested) > tolerance)) { reason = "BLOCKED.online.tester.close_fill_not_confirmed"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Confirm one ticket protection update with unchanged exposure | //+------------------------------------------------------------------+ bool DPCCConfirmTesterProtection(const SDPCCPhysicalPosition &before[], const SDPCCPhysicalPosition &after[], const ulong ticket, const double sl, const double tp, string &reason) { if(ArraySize(before) != ArraySize(after)) { reason = "BLOCKED.online.tester.protection_position_count_changed"; return(false); } bool changed = false; for(int i = 0; i < ArraySize(before); i++) { int at = -1; for(int j = 0; j < ArraySize(after); j++) if(before[i].ticket == after[j].ticket) { at = j; break; } if(at < 0 || before[i].identifier != after[at].identifier || before[i].type != after[at].type || before[i].opened_msc != after[at].opened_msc || before[i].volume != after[at].volume || before[i].open_price != after[at].open_price) { reason = "BLOCKED.online.tester.protection_exposure_changed"; return(false); } if(before[i].ticket == ticket) { if(after[at].sl != sl || after[at].tp != tp) { reason = "BLOCKED.online.tester.protection_update_not_confirmed"; return(false); } changed = true; } else if(before[i].sl != after[at].sl || before[i].tp != after[at].tp || before[i].updated_msc != after[at].updated_msc) { reason = "BLOCKED.online.tester.unrequested_protection_changed"; return(false); } } if(!changed) { reason = "BLOCKED.online.tester.protection_ticket_missing"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Confirm an opening fill preserves every owned same-side ticket | //+------------------------------------------------------------------+ bool DPCCConfirmTesterOpen(const SDPCCPhysicalPosition &before[], const SDPCCPhysicalPosition &after[], const long margin_mode, const double old_lots, const double new_lots, const bool buy, const double requested, const bool partial, const double sl, const double tp, string &reason) { const double filled = new_lots - old_lots; const double tolerance = DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, MathMax(new_lots, requested)); if(!MathIsValidNumber(filled) || filled <= 0 || filled > requested + tolerance || (!partial && MathAbs(filled - requested) > tolerance) || ArraySize(after) > ArraySize(before) + 1) { reason = "BLOCKED.online.tester.open_fill_not_confirmed"; return(false); } for(int i = 0; i < ArraySize(after); i++) if(after[i].type != (buy ? POSITION_TYPE_BUY : POSITION_TYPE_SELL) || after[i].sl != sl || after[i].tp != tp) { reason = "BLOCKED.online.tester.open_direction_or_protection"; return(false); } for(int i = 0; i < ArraySize(before); i++) { int at = -1; for(int j = 0; j < ArraySize(after); j++) if(before[i].ticket == after[j].ticket) { at = j; break; } if(at < 0 || before[i].identifier != after[at].identifier || before[i].opened_msc != after[at].opened_msc || before[i].type != after[at].type || (margin_mode == ACCOUNT_MARGIN_MODE_RETAIL_HEDGING && (before[i].volume != after[at].volume || before[i].open_price != after[at].open_price || before[i].updated_msc != after[at].updated_msc))) { reason = "BLOCKED.online.tester.open_existing_ticket_changed"; return(false); } } return(true); } //+------------------------------------------------------------------+ //| Action6 reconciles owned FIFO closes before any opposite opening | //+------------------------------------------------------------------+ bool DPCCExecuteTesterAction(const string symbol, const long magic, const float &action[], const double tp_cap_points, const double sl_cap_points, string &reason) { //--- Admit direct lots and a stable one-direction ticket snapshot. if(!DPCCRequireTesterOneDirection(reason)) return(false); if(ArraySize(action) != 6 || !MathIsValidNumber(tp_cap_points) || !MathIsValidNumber(sl_cap_points) || tp_cap_points <= 0 || sl_cap_points <= 0) { reason = "BLOCKED.online.tester.action_scale"; return(false); } for(int i = 0; i < 6; i++) if(!MathIsValidNumber(action[i]) || action[i] < 0 || action[i] > 1) { reason = "online.tester.action_range"; return(false); } SDPCCPhysicalPosition positions[]; double old = 0; bool old_buy = false; if(!DPCCCollectTesterPositions(symbol, magic, positions, old, old_buy, reason)) return(false); const long margin_mode = AccountInfoInteger(ACCOUNT_MARGIN_MODE); MqlTick tick; if(!SymbolInfoTick(symbol, tick) || !MathIsValidNumber(tick.bid) || !MathIsValidNumber(tick.ask) || tick.bid <= 0 || tick.ask < tick.bid) { reason = "online.tester.quote"; return(false); } const double point = SymbolInfoDouble(symbol, SYMBOL_POINT); const double grid = SymbolInfoDouble(symbol, SYMBOL_TRADE_TICK_SIZE); const double minimum = SymbolInfoDouble(symbol, SYMBOL_VOLUME_MIN); const double maximum = SymbolInfoDouble(symbol, SYMBOL_VOLUME_MAX); const double step = SymbolInfoDouble(symbol, SYMBOL_VOLUME_STEP); const double stop = SymbolInfoInteger(symbol, SYMBOL_TRADE_STOPS_LEVEL) * point; const int digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); const long fills = SymbolInfoInteger(symbol, SYMBOL_FILLING_MODE); const long execution = SymbolInfoInteger(symbol, SYMBOL_TRADE_EXEMODE); if(!MathIsValidNumber(point) || !MathIsValidNumber(grid) || !MathIsValidNumber(minimum) || !MathIsValidNumber(maximum) || !MathIsValidNumber(step) || point <= 0 || grid <= 0 || minimum <= 0 || maximum < minimum || step <= 0) { reason = "BLOCKED.online.tester.symbol_execution_properties"; return(false); } ENUM_ORDER_TYPE_FILLING filling = ORDER_FILLING_FOK; if((fills & SYMBOL_FILLING_FOK) != 0) filling = ORDER_FILLING_FOK; else if((fills & SYMBOL_FILLING_IOC) != 0) filling = ORDER_FILLING_IOC; else if(execution != SYMBOL_TRADE_EXECUTION_MARKET) filling = ORDER_FILLING_RETURN; else { reason = "BLOCKED.online.tester.filling_mode"; return(false); } double signed_target = 0; if(!DPCCBuildTesterPositionTarget((double)action[0] - (double)action[3], minimum, step, signed_target, reason)) return(false); const bool buy = signed_target > 0; double target = MathAbs(signed_target); double sl = 0; double tp = 0; if(!DPCCBuildTesterActionProtection(action, buy, tick.bid, tick.ask, point, grid, digits, stop, tp_cap_points, sl_cap_points, target, sl, tp, reason)) { if(reason == "online.tester.invalid_active_sl" || reason == "online.tester.invalid_active_tp") { reason = ""; return(true); } return(false); } const bool reversal = old > 0 && target > 0 && old_buy != buy; const double retained = (reversal ? 0 : target); //--- Close FIFO tickets, reconciling every completed deal before another request. while(old > retained + DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, old)) { if(!DPCCRequireTesterFIFO(positions, reason)) return(false); const SDPCCPhysicalPosition first = positions[0]; const double close_requested = MathMin(first.volume, old - retained); double close_volume = DPCCCapTesterDealVolume(close_requested, minimum, maximum, step); const double tolerance = DPCC_EXECUTION_LOT_EPSILON * MathMax(1.0, first.volume); if(close_volume > 0 && close_requested == first.volume && first.volume >= minimum && first.volume <= maximum && MathAbs(close_volume - first.volume) <= tolerance) close_volume = first.volume; if(close_volume <= 0) { if(reversal || target == 0) { reason = "BLOCKED.online.tester.full_close_volume_unexecutable"; return(false); } break; } if(!SymbolInfoTick(symbol, tick) || !MathIsValidNumber(tick.bid) || !MathIsValidNumber(tick.ask) || tick.bid <= 0 || tick.ask < tick.bid) { reason = "online.tester.close_quote"; return(false); } MqlTradeRequest request; ZeroMemory(request); request.action = TRADE_ACTION_DEAL; request.symbol = symbol; request.magic = (ulong)magic; request.position = first.ticket; request.volume = close_volume; request.type = (old_buy ? ORDER_TYPE_SELL : ORDER_TYPE_BUY); request.price = (old_buy ? tick.bid : tick.ask); request.type_filling = filling; bool accepted = false; bool partial = false; if(!DPCCSendTesterRequest(request, accepted, partial, reason)) return(false); if(!accepted) return(true); SDPCCPhysicalPosition after[]; double remaining = 0; bool remaining_buy = false; if(!DPCCCollectTesterPositions(symbol, magic, after, remaining, remaining_buy, reason)) return(false); if(!DPCCConfirmTesterClose(positions, after, first.ticket, close_volume, partial, reason)) return(false); const int remaining_count = ArraySize(after); if(ArrayResize(positions, remaining_count) != remaining_count) { reason = "online.tester.close_reconciliation_capacity"; return(false); } if(remaining_count > 0 && ArrayCopy(positions, after) != remaining_count) { reason = "online.tester.close_reconciliation_copy"; return(false); } old = remaining; old_buy = remaining_buy; if(partial) return(true); } //--- A fresh flat snapshot is mandatory before the opposite opening leg. if(!DPCCCollectTesterPositions(symbol, magic, positions, old, old_buy, reason)) return(false); if(reversal && (ArraySize(positions) != 0 || old != 0 || OrdersTotal() != 0)) { reason = "BLOCKED.online.tester.reversal_requires_confirmed_flat"; return(false); } if(target == 0) return(true); if(old > 0 && old_buy != buy) { reason = "BLOCKED.online.tester.opposite_exposure_before_open"; return(false); } //--- Converge all owned tickets to one confirmed protection before an add-on. bool protection_changed = false; for(int i = 0; i < ArraySize(positions); i++) { if(positions[i].sl == sl && positions[i].tp == tp) continue; const ulong ticket = positions[i].ticket; MqlTradeRequest modify; ZeroMemory(modify); modify.action = TRADE_ACTION_SLTP; modify.symbol = symbol; modify.magic = (ulong)magic; modify.position = ticket; modify.sl = sl; modify.tp = tp; bool accepted = false; bool partial = false; if(!DPCCSendTesterRequest(modify, accepted, partial, reason)) return(false); if(!accepted) { if(protection_changed) { reason = "BLOCKED.online.tester.incomplete_common_protection"; return(false); } return(true); } SDPCCPhysicalPosition after[]; double remaining = 0; bool remaining_buy = false; if(!DPCCCollectTesterPositions(symbol, magic, after, remaining, remaining_buy, reason, false)) return(false); if(!DPCCConfirmTesterProtection(positions, after, ticket, sl, tp, reason)) return(false); if(ArrayCopy(positions, after) != ArraySize(after)) { reason = "online.tester.protection_reconciliation_capacity"; return(false); } old = remaining; old_buy = remaining_buy; protection_changed = true; } //--- Reconcile again before one feasible same-side opening deal. if(!DPCCCollectTesterPositions(symbol, magic, positions, old, old_buy, reason)) return(false); if(AccountInfoInteger(ACCOUNT_MARGIN_MODE) != margin_mode || (old > 0 && old_buy != buy) || (reversal && (old != 0 || ArraySize(positions) != 0))) { reason = "BLOCKED.online.tester.open_state_changed"; return(false); } const double open_volume = DPCCCapTesterDealVolume(target - old, minimum, maximum, step); if(open_volume <= 0) return(true); if(!SymbolInfoTick(symbol, tick) || !MathIsValidNumber(tick.bid) || !MathIsValidNumber(tick.ask) || tick.bid <= 0 || tick.ask < tick.bid) { reason = "online.tester.open_quote"; return(false); } MqlTradeRequest request; ZeroMemory(request); request.action = TRADE_ACTION_DEAL; request.symbol = symbol; request.magic = (ulong)magic; request.volume = open_volume; request.type = (buy ? ORDER_TYPE_BUY : ORDER_TYPE_SELL); request.price = (buy ? tick.ask : tick.bid); request.sl = sl; request.tp = tp; request.type_filling = filling; bool accepted = false; bool partial = false; if(!DPCCSendTesterRequest(request, accepted, partial, reason)) return(false); if(!accepted) return(true); SDPCCPhysicalPosition after[]; double remaining = 0; bool remaining_buy = false; if(!DPCCCollectTesterPositions(symbol, magic, after, remaining, remaining_buy, reason)) return(false); return(DPCCConfirmTesterOpen(positions, after, margin_mode, old, remaining, buy, open_volume, partial, sl, tp, reason)); } //+------------------------------------------------------------------+ //| Chronological checkpoint metadata contains no authoritative | //| ledger | //+------------------------------------------------------------------+ struct SDPCCZOnlineCheckpoint { string source_sha, encoder_receipt_sha, encoder_sha, raw_sha, profile_sha; string paths[], hashes[]; long first_open, resume_open; ulong updates, actor_updates, target_updates; }; //+------------------------------------------------------------------+ //| Function DPCCZReadOnlineCheckpoint | //+------------------------------------------------------------------+ bool DPCCZReadOnlineCheckpoint(const string path, const string expected, SDPCCZOnlineCheckpoint &state, string &reason, const bool common_files = false) { string wire = ""; const string header = "DPCC_Z_ONLINE_MODELS_V3\n"; const string suffix = "NATIVE_CNET_WEIGHTS_ADAM_MOMENTS_COUNTERS\nREBUILD_TRANSIENTS_BY_TESTER_REPLAY\n" + "NO_AUTHORITATIVE_FINANCIAL_STATE\nEND_DPCC_Z_ONLINE_MODELS_V3\n"; if(!DPCCZReadPinnedText(path, expected, wire, reason, common_files) || StringSubstr(wire, 0, StringLen(header)) != header) {reason = "online.z.checkpoint_version"; return(false);} string fields[23]; int cursor = StringLen(header); for(int i = 0; i < 23; i++) if(!DPCCZReadField(wire, cursor, fields[i], reason)) return(false); ulong first = 0, resume = 0; if(StringSubstr(wire, cursor) != suffix || fields[0] != "Z[21,9,12,16];contiguous_group=[K*V=189,H*D=192];no_copy;first_Action6;H1_TD" || !DPCCCheckpointReadUnsigned(fields[6], LONG_MAX, first) || first == 0 || !DPCCCheckpointReadUnsigned(fields[7], LONG_MAX, resume) || resume <= first || !DPCCCheckpointReadUnsigned(fields[8], UINT_MAX, state.updates) || state.updates == 0 || !DPCCCheckpointReadUnsigned(fields[9], UINT_MAX, state.actor_updates) || state.actor_updates > state.updates || !DPCCCheckpointReadUnsigned(fields[10], UINT_MAX, state.target_updates) || state.target_updates > state.updates || ArrayResize(state.paths, 6) != 6 || ArrayResize(state.hashes, 6) != 6) {reason = "online.z.checkpoint_metadata"; return(false);} for(int i = 1; i <= 5; i++) if(!DPCCResolverLowerSha256(fields[i])) { reason = "online.z.checkpoint_pin"; return(false); } state.source_sha = fields[1]; state.encoder_receipt_sha = fields[2]; state.encoder_sha = fields[3]; state.raw_sha = fields[4]; state.profile_sha = fields[5]; state.first_open = (long)first; state.resume_open = (long)resume; for(int i = 0; i < 6; i++) { state.paths[i] = fields[11 + 2 * i]; state.hashes[i] = fields[12 + 2 * i]; } return(true); } //+------------------------------------------------------------------+ //| Dispatch native V3 and compact V4 chronological model | //| checkpoints | //+------------------------------------------------------------------+ bool DPCCZReadOnlineCheckpointEx(const string path, const string expected, SDPCCZOnlineCheckpoint &state, bool &compact, string &reason, const bool common_files = false) { //--- Authenticate the manifest before selecting its artifact format. compact = false; string wire = ""; if(!DPCCZReadPinnedText(path, expected, wire, reason, common_files)) return(false); const bool compact_wire = (StringFind(wire, "DPCC_Z_ONLINE_MODELS_V4\n") == 0); SDPCCZOnlineCheckpoint staged; if(!compact_wire) { if(StringFind(wire, "DPCC_Z_ONLINE_MODELS_V3\n") != 0) { reason = "online.z.checkpoint_version"; return(false); } if(!DPCCZReadOnlineCheckpoint(path, expected, staged, reason, common_files)) return(false); } else { const string header = "DPCC_Z_ONLINE_MODELS_V4\n"; const string suffix = "COMPACT_CNET_POLICY_TRAINING_STATE_V1\nREBUILD_TRANSIENTS_BY_TESTER_REPLAY\n" + "NO_AUTHORITATIVE_FINANCIAL_STATE\nEND_DPCC_Z_ONLINE_MODELS_V4\n"; string fields[23]; int cursor = StringLen(header); for(int i = 0; i < 23; i++) if(!DPCCZReadField(wire, cursor, fields[i], reason)) return(false); ulong first = 0, resume = 0; if(StringSubstr(wire, cursor) != suffix || fields[0] != "Z[21,9,12,16];contiguous_group=[K*V=189,H*D=192];no_copy;first_Action6;H1_TD" || !DPCCCheckpointReadUnsigned(fields[6], LONG_MAX, first) || first == 0 || !DPCCCheckpointReadUnsigned(fields[7], LONG_MAX, resume) || resume <= first || !DPCCCheckpointReadUnsigned(fields[8], UINT_MAX, staged.updates) || staged.updates == 0 || !DPCCCheckpointReadUnsigned(fields[9], UINT_MAX, staged.actor_updates) || staged.actor_updates > staged.updates || !DPCCCheckpointReadUnsigned(fields[10], UINT_MAX, staged.target_updates) || staged.target_updates > staged.updates) { reason = "online.z.checkpoint_metadata"; return(false); } for(int i = 1; i <= 5; i++) if(!DPCCResolverLowerSha256(fields[i])) { reason = "online.z.checkpoint_pin"; return(false); } if(ArrayResize(staged.paths, 6) != 6 || ArrayResize(staged.hashes, 6) != 6) { reason = "online.z.checkpoint_capacity"; return(false); } staged.source_sha = fields[1]; staged.encoder_receipt_sha = fields[2]; staged.encoder_sha = fields[3]; staged.raw_sha = fields[4]; staged.profile_sha = fields[5]; staged.first_open = (long)first; staged.resume_open = (long)resume; for(int i = 0; i < 6; i++) { staged.paths[i] = fields[11 + 2 * i]; staged.hashes[i] = fields[12 + 2 * i]; } } //--- All six artifacts must match the authenticated manifest generation. const string extension = (compact_wire ? ".dppt" : ".nnw"); for(int i = 0; i < 6; i++) { string canonical = ""; const int length = StringLen(staged.paths[i]); if(length <= StringLen(extension) || StringSubstr(staged.paths[i], length - StringLen(extension)) != extension || !DPCCCheckpointCanonicalPath(staged.paths[i], canonical, reason) || canonical != staged.paths[i] || !DPCCResolverLowerSha256(staged.hashes[i])) { reason = "online.z.checkpoint_model_format_or_pin"; return(false); } } state = staged; compact = compact_wire; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Write the latest compact models and chronological metadata | //+------------------------------------------------------------------+ bool DPCCZSaveOnlineCheckpoint(CNet *&nets[], const SDPCCZOnlineCheckpoint &state, const string stem, string &manifest, string &sha, string &reason) { manifest = ""; sha = ""; string canonical = ""; if(!DPCCTesterOnly(reason) || ArraySize(nets) != 6 || state.updates == 0 || state.updates > UINT_MAX || state.first_open <= 0 || state.resume_open <= state.first_open || state.actor_updates > state.updates || state.target_updates > state.updates || !DPCCResolverLowerSha256(state.source_sha) || !DPCCResolverLowerSha256(state.encoder_receipt_sha) || !DPCCResolverLowerSha256(state.encoder_sha) || !DPCCResolverLowerSha256(state.raw_sha) || !DPCCResolverLowerSha256(state.profile_sha) || !DPCCCheckpointCanonicalPath(stem, canonical, reason) || canonical != stem) {if(reason == "")reason = "online.z.save_boundary"; return(false);} const string prefix = stem + ".v4.latest"; const string receipt = prefix + ".manifest"; const int writer = FileOpen(stem + ".writer.lock", FILE_READ | FILE_WRITE | FILE_BIN | FILE_COMMON); if(writer == INVALID_HANDLE) { reason = "online.z.writer_lock"; return(false); } bool saved = false; do { string wire = "DPCC_Z_ONLINE_MODELS_V4\n"; const string fields[] = {"Z[21,9,12,16];contiguous_group=[K*V=189,H*D=192];no_copy;first_Action6;H1_TD", state.source_sha, state.encoder_receipt_sha, state.encoder_sha, state.raw_sha, state.profile_sha, (string)state.first_open, ( string)state.resume_open, (string)state.updates, (string)state.actor_updates, (string)state.target_updates }; for(int i = 0; i < 11; i++) DPCCOfflineField(wire, fields[i]); bool models = true; for(int i = 0; i < 6; i++) { const string path = prefix + ".net." + (string)i + ".dppt"; if(!nets[i]) { reason = "online.z.model_pointer"; models = false; break; } string hash = ""; if(!DPCCSaveCompactPolicyTrainingState(*nets[i], path, true, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, hash, reason, true)) { models = false; break; } DPCCOfflineField(wire, path); DPCCOfflineField(wire, hash); } if(!models) break; wire += "COMPACT_CNET_POLICY_TRAINING_STATE_V1\nREBUILD_TRANSIENTS_BY_TESTER_REPLAY\n"; wire += "NO_AUTHORITATIVE_FINANCIAL_STATE\nEND_DPCC_Z_ONLINE_MODELS_V4\n"; string expected = "", observed = ""; if(!DPCCCheckpointTextSHA(wire, expected, reason) || !DPCCCheckpointWriteText(receipt, true, wire, reason, true) || !DPCCCheckpointFileSHA(receipt, true, 65536, observed, reason) || observed != expected) break; manifest = receipt; sha = observed; saved = true; } while(false); FileClose(writer); return(saved); } #ifdef DPCC_TRAJECTORY_ENTRY_STUDY_FORECAST #define DPCC_TRAJECTORY_INCLUDE_STUDY_FORECAST_CORE #endif #ifdef DPCC_TRAJECTORY_ENTRY_STUDY #define DPCC_TRAJECTORY_INCLUDE_STUDY_FORECAST_CORE #endif #ifdef DPCC_TRAJECTORY_INCLUDE_STUDY_FORECAST_CORE //+------------------------------------------------------------------+ //| StudyForecastCore.mqh | //+------------------------------------------------------------------+ #ifndef DPCC_STUDY_FORECAST_CORE_MQH #define DPCC_STUDY_FORECAST_CORE_MQH //--- These existing device operations are protected on CNeuronBaseOCL. class CDPCCStudyForecastGradientOps : public CNeuronBaseOCL { private: CBufferFloat m_finite_flags, m_finite_status; public: bool Bind(COpenCLMy *context) { OpenCL = context; if(CheckPointer(OpenCL) == POINTER_INVALID) return(false); if(!m_finite_flags.BufferInit(36288, 0.0f)) return(false); if(!m_finite_status.BufferInit(1, 0.0f)) return(false); if(!m_finite_flags.BufferCreate(OpenCL)) return(false); if(!m_finite_status.BufferCreate(OpenCL)) return(false); return(true); } bool BuildTargetDelta(CBufferFloat *zero, CBufferFloat *prediction, CBufferFloat *negative, CBufferFloat *future, CBufferFloat *delta, const uint dimension, const uint variables) { return(Different(zero, prediction, negative, (int)dimension) && SumVecMatrix(negative, future, delta, dimension, variables)); } bool Copy(CBufferFloat *source, CBufferFloat *destination, const uint count) {return(CopyBufferRaw(source, destination, count));} bool CopyFinite(CBufferFloat *source, CBufferFloat *destination, const uint count) { if(!source || !destination || CheckPointer(OpenCL) == POINTER_INVALID || count == 0 || count != 36288 || source == destination || source.GetOpenCL() != OpenCL || destination.GetOpenCL() != OpenCL || source.GetIndex() < 0 || destination.GetIndex() < 0 || (uint)source.Total() != count || (uint)destination.Total() != count || OpenCL.GetBufferSize(source.GetIndex()) != (ulong)count * sizeof(float) || OpenCL.GetBufferSize(destination.GetIndex()) != (ulong)count * sizeof(float) || m_finite_flags.GetOpenCL() != OpenCL || m_finite_status.GetOpenCL() != OpenCL || m_finite_flags.GetIndex() < 0 || m_finite_status.GetIndex() < 0 || m_finite_flags.Total() != (int)count || m_finite_status.Total() != 1 || OpenCL.GetBufferSize(m_finite_flags.GetIndex()) != (ulong)count * sizeof(float) || OpenCL.GetBufferSize(m_finite_status.GetIndex()) != sizeof(float)) return(false); if(source.GetIndex() == destination.GetIndex() || source.GetIndex() == m_finite_flags.GetIndex() || source.GetIndex() == m_finite_status.GetIndex() || destination.GetIndex() == m_finite_flags.GetIndex() || destination.GetIndex() == m_finite_status.GetIndex() || m_finite_flags.GetIndex() == m_finite_status.GetIndex()) return(false); if(!OpenCL.SetArgumentBuffer(def_k_DPCCActivationForward, 0, source.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_DPCCActivationForward, 1, destination.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_DPCCActivationForward, 2, m_finite_flags.GetIndex()) || !OpenCL.SetArgument(def_k_DPCCActivationForward, 3, (int)None) || !OpenCL.SetArgument(def_k_DPCCActivationForward, 4, count)) return(false); const uint offset[] = {0}, global[] = {count}; if(!OpenCL.Execute(def_k_DPCCActivationForward, 1, offset, global)) return(false); if(!OpenCL.SetArgumentBuffer(def_k_DPCCOperatorReduce, 0, m_finite_flags.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_DPCCOperatorReduce, 1, m_finite_status.GetIndex()) || !OpenCL.SetArgument(def_k_DPCCOperatorReduce, 2, count)) return(false); const uint reduce_global[] = {1}; if(!OpenCL.Execute(def_k_DPCCOperatorReduce, 1, offset, reduce_global)) return(false); if(!m_finite_status.BufferRead() || m_finite_status[0] != 0.0f) return(false); return(true); } }; //+------------------------------------------------------------------+ //| Adapter for the existing AC-SRM Stage 02 calibration inputs | //+------------------------------------------------------------------+ //--- VERIFY limits are independent of donor OMPB calibration parameters. struct SDPCCM05VerifyProfile { double atol, rtol, max_price_error, min_scenario_coverage; uint min_distinct_scenarios; bool has_max_price_error, has_min_scenario_coverage, has_min_distinct_scenarios; }; struct SDPCCM05OMPBProfile { string profile_sha256, raw_sha256; string assigned_profile_sha256, market_profile_sha256; string calibration_preset_path, calibration_preset_sha256; ulong noise_seed; string market_preset_path, market_preset_sha256; datetime latent_start, latent_end; uint latent_epochs; datetime reference_start, reference_split, reference_end; datetime calibration_start, calibration_split, calibration_end; int reference_first_bar, reference_end_bar; int reference_split_bar; int calibration_first_bar, calibration_end_bar; int calibration_split_bar; uint epochs, source_anchor_period; uint price_joint_epochs, price_adapt_epochs; ulong price_joint_max_updates, price_adapt_max_updates; uint price_joint_early_stop_patience, price_adapt_early_stop_patience; float tau, lambda_dis, lambda_kl, lambda_alpha, alpha_prior, max_kl; ENUM_APPLIED_PRICE rsi_price, cci_price, macd_price; }; //+------------------------------------------------------------------+ //| M05 owner snapshot at a completed call boundary | //+------------------------------------------------------------------+ struct SDPCCM05CompletedBoundary { string stage, subphase, profile_sha256, market_profile_sha256, source_preset_sha256, raw_sha256; ulong noise_seed, completed_updates, price_joint_updates, price_adapt_updates; ulong ompb_target_updates, ompb_source_updates; uint rng_state[5]; uint latent_epochs_completed, price_joint_epochs_completed, price_adapt_epochs_completed; uint ompb_epoch, ompb_epoch_source_updates; int next_record, ompb_reference_cursor; bool epoch_started, ompb_reference_complete, ompb_epoch_started, ompb_source_pending; }; //--- Read-only chart snapshot. This is not part of checkpoint serialization. struct SDPCCM05ChartMetrics { string subphase; uint epoch, epoch_limit; int evaluation_stage; float disagreement, kl, alpha_prior; uint invalid_fallbacks, kl_rejects; bool diagnostics_ready; }; //+------------------------------------------------------------------+ //| Strict DTO for the section-200 M05 resume record | //+------------------------------------------------------------------+ struct SDPCCM05ResumeRecord { string stage, subphase, record_order, profile_sha256, market_profile_sha256; string source_preset_sha256, raw_sha256; string time_basis, runtime_restore, cause, terminal_status, verify_status; ulong epoch, next_record, cursor, fit_limit, holdout_cursor, noise_seed, replay_from_bar, next_bar; ulong completed_transitions, policy_updates, target_updates; ulong rng_state[5]; bool target_order_flag, journal_path_null, journal_sha256_null, pending_null, finished; ulong completed_updates, price_joint_updates, price_adapt_updates, ompb_target_updates, ompb_source_updates; ulong latent_epochs_completed, price_joint_epochs_completed, price_adapt_epochs_completed; bool epoch_started, ompb_epoch_started; ulong ompb_epoch, ompb_epoch_source_updates, ompb_reference_cursor, ompb_reference_capacity; ulong ompb_current_capacity, ompb_reference_count, ompb_current_count; bool ompb_reference_complete, ompb_source_pending; ulong ompb_evaluation_stage, ompb_evaluation_cursor; double ompb_evaluation_loss[6]; ulong ompb_evaluation_valid[6], ompb_evaluation_invalid[6]; bool calibration_completed, stage02_quality_pass; double price_joint_loss_sum, price_adapt_loss_sum; ulong teacher_initial_fingerprint, price_joint_ompb_fingerprint, latent_decoder_fingerprint; ulong ompb_frozen_fingerprint, ompb_initial_fingerprint, ompb_evaluation_posterior; ulong price_adapt_frozen_fingerprint, verify_market_fingerprint, verify_target_fingerprint; ulong ompb_decoder_step_baseline, price_adapt_decoder_step_baseline; ulong ompb_forecast_step_baselines[3]; bool verify_started; ulong verify_snapshots, verify_covered, verify_min_distinct; double verify_max_price_error, verify_coverage, verify_error_sum[48]; }; //--- The caller resolves these counts from the admitted bars and the donor preset. struct SDPCCM05ResumeBounds { ulong fit_limit, holdout_count, reference_count, reference_capacity, current_capacity; ulong source_evaluation_count, target_evaluation_count, calibration_count; }; //+------------------------------------------------------------------+ //| Function DPCCM05ResumeUnsigned | //+------------------------------------------------------------------+ bool DPCCM05ResumeUnsigned(CDPCCJsonDocument &document, const int object, const string name, const ulong maximum, ulong &value, string &reason) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_NUMBER || !DPCCCheckpointReadUnsigned(document.Text(node), ULONG_MAX, value) || value > maximum) { reason = "M05_RESUME.unsigned." + name; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCM05ResumeNumber | //+------------------------------------------------------------------+ bool DPCCM05ResumeNumber(CDPCCJsonDocument &document, const int object, const string name, double &value, string &reason) { const int node = document.FindMember(object, name); if(node < 0 || !document.TryNumber(node, value) || value < 0 || !MathIsValidNumber(value)) { reason = "M05_RESUME.number." + name; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCM05ResumeBoolean | //+------------------------------------------------------------------+ bool DPCCM05ResumeBoolean(CDPCCJsonDocument &document, const int object, const string name, bool &value, string &reason) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_BOOLEAN) { reason = "M05_RESUME.boolean." + name; return(false); } value = (document.Text(node) == "true"); return(true); } //+------------------------------------------------------------------+ //| Function DPCCM05ResumeString | //+------------------------------------------------------------------+ bool DPCCM05ResumeString(CDPCCJsonDocument &document, const int object, const string name, string &value, string &reason) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_STRING) { reason = "M05_RESUME.string." + name; return(false); } value = document.Text(node); return(true); } //+------------------------------------------------------------------+ //| Function DPCCM05ResumeNull | //+------------------------------------------------------------------+ bool DPCCM05ResumeNull(CDPCCJsonDocument &document, const int object, const string name, bool &is_null, string &reason) { const int node = document.FindMember(object, name); if(node < 0 || document.Type(node) != DPCC_JSON_NULL) { reason = "M05_RESUME.null." + name; return(false); } is_null = true; return(true); } //+------------------------------------------------------------------+ //| Function DPCCParseM05ResumeRecord | //+------------------------------------------------------------------+ bool DPCCParseM05ResumeRecord(const string text, const SDPCCM05ResumeBounds &bounds, const string expected_market_profile_sha256, const string expected_source_preset_sha256, const string expected_raw_sha256, SDPCCM05ResumeRecord &record, string &reason) { ZeroMemory(record); reason = ""; if(!DPCCResolverLowerSha256(expected_market_profile_sha256) || !DPCCResolverLowerSha256(expected_source_preset_sha256) || !DPCCResolverLowerSha256(expected_raw_sha256) || bounds.fit_limit == 0 || bounds.fit_limit > (ulong)INT_MAX || bounds.holdout_count > (ulong)INT_MAX || bounds.reference_count > (ulong)INT_MAX || bounds.source_evaluation_count > (ulong)INT_MAX || bounds.target_evaluation_count > (ulong)INT_MAX || bounds.calibration_count > (ulong)INT_MAX || bounds.reference_capacity > (ulong)UINT_MAX || bounds.current_capacity > (ulong)UINT_MAX) { reason = "M05_RESUME.expected_identity_or_bounds"; return(false); } CDPCCJsonDocument document; if(!document.Parse(text, reason)) return(false); if(document.Type(document.Root()) != DPCC_JSON_OBJECT) { reason = "M05_RESUME.object_required"; return(false); } if(!DPCCM05KnownMembers(document, document.Root(), "|stage|subphase|epoch|next_record|cursor|fit_limit|holdout_cursor|record_order|completed_transitions|policy_updates|target_updates|target_order_flag|rng_state|noise_seed|profile_sha256|market_profile_sha256|source_preset_sha256|raw_sha256|journal_path|journal_sha256|time_basis|replay_from_bar|next_bar|pending|runtime_restore|cause|finished|terminal_status|completed_updates|price_joint_updates|price_adapt_updates|ompb_target_updates|ompb_source_updates|latent_epochs_completed|price_joint_epochs_completed|price_adapt_epochs_completed|epoch_started|ompb_epoch_started|ompb_epoch|ompb_epoch_source_updates|ompb_reference_cursor|ompb_reference_capacity|ompb_current_capacity|ompb_reference_count|ompb_current_count|ompb_reference_complete|ompb_source_pending|ompb_evaluation_stage|ompb_evaluation_cursor|ompb_evaluation|calibration_completed|stage02_quality_pass|price_joint_loss_sum|price_adapt_loss_sum|teacher_initial_fingerprint|price_joint_ompb_fingerprint|latent_decoder_fingerprint|ompb_frozen_fingerprint|ompb_initial_fingerprint|ompb_evaluation_posterior|price_adapt_frozen_fingerprint|verify_market_fingerprint|verify_target_fingerprint|ompb_decoder_step_baseline|price_adapt_decoder_step_baseline|ompb_forecast_step_baselines|verify_started|verify_status|verify_snapshots|verify_covered|verify_min_distinct|verify_max_price_error|verify_coverage|verify_error_sum|", "M05_RESUME", reason)) return(false); const int root = document.Root(); if(!DPCCM05ResumeString(document, root, "stage", record.stage, reason) || !DPCCM05ResumeString(document, root, "subphase", record.subphase, reason) || !DPCCM05ResumeUnsigned(document, root, "epoch", UINT_MAX, record.epoch, reason) || !DPCCM05ResumeUnsigned(document, root, "next_record", (ulong)INT_MAX, record.next_record, reason) || !DPCCM05ResumeUnsigned(document, root, "cursor", (ulong)INT_MAX, record.cursor, reason) || !DPCCM05ResumeUnsigned(document, root, "fit_limit", (ulong)INT_MAX, record.fit_limit, reason) || !DPCCM05ResumeUnsigned(document, root, "holdout_cursor", (ulong)INT_MAX, record.holdout_cursor, reason) || !DPCCM05ResumeString(document, root, "record_order", record.record_order, reason) || !DPCCM05ResumeUnsigned(document, root, "completed_transitions", ULONG_MAX, record.completed_transitions, reason) || !DPCCM05ResumeUnsigned(document, root, "policy_updates", ULONG_MAX, record.policy_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "target_updates", ULONG_MAX, record.target_updates, reason) || !DPCCM05ResumeBoolean(document, root, "target_order_flag", record.target_order_flag, reason)) return(false); if(record.record_order != "CHRONOLOGICAL_TERMINAL_BARS" || record.completed_transitions != 0 || record.policy_updates != 0 || record.target_updates != 0 || record.target_order_flag) { reason = "M05_RESUME.fixed_order_invariant"; return(false); } const int rng = document.FindMember(root, "rng_state"); if(rng < 0 || document.Type(rng) != DPCC_JSON_ARRAY || document.ChildCount(rng) != 5) { reason = "M05_RESUME.rng_state.shape"; return(false); } for(int i = 0; i < 5; i++) if(!DPCCCheckpointReadUnsigned(document.Text(document.ChildAt(rng, i)), UINT_MAX, record.rng_state[i]) || document.Type(document.ChildAt(rng, i)) != DPCC_JSON_NUMBER) { reason = "M05_RESUME.rng_state.value"; return(false); } if(!DPCCM05ResumeUnsigned(document, root, "noise_seed", ULONG_MAX, record.noise_seed, reason) || !DPCCM05ResumeString(document, root, "profile_sha256", record.profile_sha256, reason) || !DPCCM05ResumeString(document, root, "market_profile_sha256", record.market_profile_sha256, reason) || !DPCCM05ResumeString(document, root, "source_preset_sha256", record.source_preset_sha256, reason) || !DPCCM05ResumeString(document, root, "raw_sha256", record.raw_sha256, reason) || !DPCCM05ResumeNull(document, root, "journal_path", record.journal_path_null, reason) || !DPCCM05ResumeNull(document, root, "journal_sha256", record.journal_sha256_null, reason) || !DPCCM05ResumeString(document, root, "time_basis", record.time_basis, reason) || !DPCCM05ResumeUnsigned(document, root, "replay_from_bar", (ulong)LONG_MAX, record.replay_from_bar, reason) || !DPCCM05ResumeUnsigned(document, root, "next_bar", (ulong)LONG_MAX, record.next_bar, reason) || !DPCCM05ResumeNull(document, root, "pending", record.pending_null, reason) || !DPCCM05ResumeString(document, root, "runtime_restore", record.runtime_restore, reason) || !DPCCM05ResumeString(document, root, "cause", record.cause, reason) || !DPCCM05ResumeBoolean(document, root, "finished", record.finished, reason) || !DPCCM05ResumeString(document, root, "terminal_status", record.terminal_status, reason)) return(false); if(!DPCCResolverLowerSha256(record.profile_sha256) || !DPCCResolverLowerSha256(record.market_profile_sha256) || record.market_profile_sha256 != expected_market_profile_sha256 || !DPCCResolverLowerSha256(record.source_preset_sha256) || record.source_preset_sha256 != expected_source_preset_sha256 || !DPCCResolverLowerSha256(record.raw_sha256) || record.raw_sha256 != expected_raw_sha256 || record.time_basis != "TERMINAL_SERVER" || record.runtime_restore != "WARMUP_FROM_LATEST_WEIGHTS" || (record.cause != "PHASE_COMPLETED" && record.cause != "CONTROLLED_STOP" && record.cause != "VERIFY_COMPLETED" && record.cause != "USER_ACCEPTED_LATENT_PREPARE" && record.cause != "EPOCH_COMPLETED" && record.cause != "USER_ACCEPTED_PARTIAL_OMPB" && record.cause != "USER_SELECTED_PHASE") || (record.terminal_status != "" && record.terminal_status != "STOPPED_COMPLETE_UPDATE" && record.terminal_status != "PHASE_COMPLETED" && record.terminal_status != "PASS" && record.terminal_status != "FAIL" && record.terminal_status != "BLOCKED_MISSING_LIMITS" && record.terminal_status != "INSUFFICIENT_DATA")) { reason = "M05_RESUME.identity_or_fixed_invariant"; return(false); } if(!DPCCM05ResumeUnsigned(document, root, "completed_updates", ULONG_MAX, record.completed_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "price_joint_updates", ULONG_MAX, record.price_joint_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_updates", ULONG_MAX, record.price_adapt_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_target_updates", ULONG_MAX, record.ompb_target_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_source_updates", ULONG_MAX, record.ompb_source_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "latent_epochs_completed", UINT_MAX, record.latent_epochs_completed, reason) || !DPCCM05ResumeUnsigned(document, root, "price_joint_epochs_completed", UINT_MAX, record.price_joint_epochs_completed, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_epochs_completed", UINT_MAX, record.price_adapt_epochs_completed, reason) || !DPCCM05ResumeBoolean(document, root, "epoch_started", record.epoch_started, reason) || !DPCCM05ResumeBoolean(document, root, "ompb_epoch_started", record.ompb_epoch_started, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_epoch", UINT_MAX, record.ompb_epoch, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_epoch_source_updates", UINT_MAX, record.ompb_epoch_source_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_reference_cursor", ULONG_MAX, record.ompb_reference_cursor, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_reference_capacity", ULONG_MAX, record.ompb_reference_capacity, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_current_capacity", ULONG_MAX, record.ompb_current_capacity, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_reference_count", ULONG_MAX, record.ompb_reference_count, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_current_count", ULONG_MAX, record.ompb_current_count, reason) || !DPCCM05ResumeBoolean(document, root, "ompb_reference_complete", record.ompb_reference_complete, reason) || !DPCCM05ResumeBoolean(document, root, "ompb_source_pending", record.ompb_source_pending, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_evaluation_stage", 6, record.ompb_evaluation_stage, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_evaluation_cursor", ULONG_MAX, record.ompb_evaluation_cursor, reason)) return(false); const int evaluation = document.FindMember(root, "ompb_evaluation"); if(evaluation < 0 || document.Type(evaluation) != DPCC_JSON_ARRAY || document.ChildCount(evaluation) != 6) { reason = "M05_RESUME.ompb_evaluation.shape"; return(false); } for(int i = 0; i < 6; i++) { const int item = document.ChildAt(evaluation, i); if(!DPCCM05KnownMembers(document, item, "|loss|valid|invalid|", "M05_RESUME.ompb_evaluation", reason) || document.ChildCount(item) != 3 || !DPCCM05ResumeNumber(document, item, "loss", record.ompb_evaluation_loss[i], reason) || !DPCCM05ResumeUnsigned(document, item, "valid", ULONG_MAX, record.ompb_evaluation_valid[i], reason) || !DPCCM05ResumeUnsigned(document, item, "invalid", ULONG_MAX, record.ompb_evaluation_invalid[i], reason)) { if(reason == "") reason = "M05_RESUME.ompb_evaluation.item"; return(false); } } if(!DPCCM05ResumeBoolean(document, root, "calibration_completed", record.calibration_completed, reason) || !DPCCM05ResumeBoolean(document, root, "stage02_quality_pass", record.stage02_quality_pass, reason) || !DPCCM05ResumeNumber(document, root, "price_joint_loss_sum", record.price_joint_loss_sum, reason) || !DPCCM05ResumeNumber(document, root, "price_adapt_loss_sum", record.price_adapt_loss_sum, reason) || !DPCCM05ResumeUnsigned(document, root, "teacher_initial_fingerprint", ULONG_MAX, record.teacher_initial_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "price_joint_ompb_fingerprint", ULONG_MAX, record.price_joint_ompb_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "latent_decoder_fingerprint", ULONG_MAX, record.latent_decoder_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_frozen_fingerprint", ULONG_MAX, record.ompb_frozen_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_initial_fingerprint", ULONG_MAX, record.ompb_initial_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_evaluation_posterior", ULONG_MAX, record.ompb_evaluation_posterior, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_frozen_fingerprint", ULONG_MAX, record.price_adapt_frozen_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "verify_market_fingerprint", ULONG_MAX, record.verify_market_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "verify_target_fingerprint", ULONG_MAX, record.verify_target_fingerprint, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_decoder_step_baseline", UINT_MAX, record.ompb_decoder_step_baseline, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_decoder_step_baseline", UINT_MAX, record.price_adapt_decoder_step_baseline, reason)) return(false); const int forecast = document.FindMember(root, "ompb_forecast_step_baselines"); if(forecast < 0 || document.Type(forecast) != DPCC_JSON_ARRAY || document.ChildCount(forecast) != 3) { reason = "M05_RESUME.ompb_forecast_step_baselines.shape"; return(false); } for(int i = 0; i < 3; i++) if(document.Type(document.ChildAt(forecast, i)) != DPCC_JSON_NUMBER || !DPCCCheckpointReadUnsigned(document.Text(document.ChildAt(forecast, i)), UINT_MAX, record.ompb_forecast_step_baselines[i])) { reason = "M05_RESUME.ompb_forecast_step_baselines.value"; return(false); } if(!DPCCM05ResumeBoolean(document, root, "verify_started", record.verify_started, reason) || !DPCCM05ResumeString(document, root, "verify_status", record.verify_status, reason) || !DPCCM05ResumeUnsigned(document, root, "verify_snapshots", UINT_MAX, record.verify_snapshots, reason) || !DPCCM05ResumeUnsigned(document, root, "verify_covered", UINT_MAX, record.verify_covered, reason) || !DPCCM05ResumeUnsigned(document, root, "verify_min_distinct", 21, record.verify_min_distinct, reason) || !DPCCM05ResumeNumber(document, root, "verify_max_price_error", record.verify_max_price_error, reason) || !DPCCM05ResumeNumber(document, root, "verify_coverage", record.verify_coverage, reason) || record.verify_coverage > 1.0) return(false); const int errors = document.FindMember(root, "verify_error_sum"); if(errors < 0 || document.Type(errors) != DPCC_JSON_ARRAY || document.ChildCount(errors) != 48) { reason = "M05_RESUME.verify_error_sum.shape"; return(false); } for(int i = 0; i < 48; i++) if(!document.TryNumber(document.ChildAt(errors, i), record.verify_error_sum[i]) || record.verify_error_sum[i] < 0 || !MathIsValidNumber(record.verify_error_sum[i])) { reason = "M05_RESUME.verify_error_sum.value"; return(false); } if(record.verify_status != "" && record.verify_status != "NOT_RUN" && record.verify_status != "RUNNING" && record.verify_status != "PASS" && record.verify_status != "FAIL" && record.verify_status != "BLOCKED_MISSING_LIMITS" && record.verify_status != "INSUFFICIENT_DATA") { reason = "M05_RESUME.verify_status.domain"; return(false); } if((record.verify_started && (record.verify_status == "" || record.verify_status == "NOT_RUN")) || (!record.verify_started && record.verify_status != "" && record.verify_status != "NOT_RUN")) { reason = "M05_RESUME.verify_lifecycle"; return(false); } if(record.fit_limit != bounds.fit_limit || (record.cursor > bounds.fit_limit && record.cursor > bounds.holdout_count) || record.holdout_cursor > bounds.holdout_count || record.ompb_reference_cursor > bounds.reference_count || record.ompb_reference_capacity != bounds.reference_capacity || record.ompb_current_capacity != bounds.current_capacity || record.ompb_reference_count > record.ompb_reference_capacity || record.ompb_reference_count > bounds.reference_count || record.ompb_current_count > record.ompb_current_capacity || record.verify_snapshots > bounds.holdout_count || record.ompb_epoch_source_updates > record.ompb_source_updates || record.verify_covered > record.verify_snapshots || record.completed_updates < record.price_joint_updates || record.completed_updates < record.price_adapt_updates || record.completed_updates < record.ompb_target_updates || record.completed_updates < record.ompb_source_updates || record.ompb_target_updates > record.completed_updates - record.ompb_source_updates || (record.replay_from_bar > 0 && record.next_bar > 0 && record.next_bar < record.replay_from_bar)) { reason = "M05_RESUME.count_or_cursor_bounds"; return(false); } ulong evaluation_cursor_limit = bounds.source_evaluation_count; if(bounds.target_evaluation_count > evaluation_cursor_limit) evaluation_cursor_limit = bounds.target_evaluation_count; if(record.ompb_evaluation_cursor > evaluation_cursor_limit || (record.ompb_reference_complete && record.ompb_reference_count != record.ompb_reference_capacity)) { reason = "M05_RESUME.stage02_or_evaluation_state"; return(false); } const ulong expected_epoch = (record.stage == "LATENT_PREPARE" ? record.latent_epochs_completed : (record.stage == "PRICE_JOINT" ? record.price_joint_epochs_completed : (record.stage == "OMPB_CALIBRATE" ? record.ompb_epoch : record.price_adapt_epochs_completed))); if(record.epoch != expected_epoch) { reason = "M05_RESUME.epoch_phase_mismatch"; return(false); } if(record.stage == "OMPB_CALIBRATE") { ulong next_limit = 0; bool phase_ok = false; if(record.subphase == "REFERENCE") { phase_ok = !record.ompb_reference_complete && record.ompb_reference_cursor < bounds.reference_count && record.next_record == record.ompb_reference_cursor; next_limit = bounds.reference_count; } else if(record.subphase == "SOURCE_ANCHOR") { phase_ok = record.ompb_reference_complete && record.ompb_source_pending; next_limit = bounds.calibration_count; } else if(record.subphase == "CALIBRATION") { phase_ok = record.ompb_reference_complete && !record.ompb_source_pending && record.ompb_evaluation_stage >= 4 && !record.calibration_completed; next_limit = bounds.calibration_count; } else if(StringFind(record.subphase, "EVALUATION_") == 0) { const string expected_subphase = "EVALUATION_" + (string)record.ompb_evaluation_stage; next_limit = (record.ompb_evaluation_stage % 2 == 0 ? bounds.source_evaluation_count : bounds.target_evaluation_count); phase_ok = record.subphase == expected_subphase && !record.ompb_source_pending && (record.ompb_evaluation_stage < 4 || record.calibration_completed) && next_limit > 0 && record.ompb_evaluation_cursor < next_limit; phase_ok = phase_ok && record.next_record == record.ompb_evaluation_cursor; } if(!phase_ok || record.next_record > next_limit || record.cursor > bounds.calibration_count || (record.ompb_source_pending && (!record.ompb_reference_complete || record.calibration_completed || record.ompb_evaluation_stage < 4)) || (record.ompb_epoch_started && (record.subphase != "CALIBRATION" && record.subphase != "SOURCE_ANCHOR")) || (record.calibration_completed && record.ompb_evaluation_stage < 4) || (record.stage02_quality_pass && !record.calibration_completed)) { reason = "M05_RESUME.ompb_phase_state"; return(false); } } else { ulong phase_limit = 0; if(record.stage == "LATENT_PREPARE" || record.stage == "PRICE_JOINT" || record.stage == "PRICE_ADAPT") phase_limit = bounds.fit_limit; else if(record.stage == "VERIFY") phase_limit = bounds.holdout_count; else { reason = "M05_RESUME.stage.domain"; return(false); } if(record.subphase != record.stage || record.next_record != record.cursor || record.next_record > phase_limit) { reason = "M05_RESUME.phase_cursor"; return(false); } } if(record.verify_coverage > 0 && record.verify_snapshots == 0) { reason = "M05_RESUME.verify_without_snapshots"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Bind model-only loading to the source generation, without | //| cursors | //+------------------------------------------------------------------+ bool DPCCParseM05ModelBoundary(const string text, SDPCCM05ResumeRecord &record, string &reason) { ZeroMemory(record); reason = ""; CDPCCJsonDocument document; if(!document.Parse(text, reason)) return(false); const int root = document.Root(); if(document.Type(root) != DPCC_JSON_OBJECT || !DPCCM05ResumeString(document, root, "stage", record.stage, reason) || !DPCCM05ResumeString(document, root, "profile_sha256", record.profile_sha256, reason) || !DPCCM05ResumeString(document, root, "market_profile_sha256", record.market_profile_sha256, reason) || !DPCCM05ResumeString(document, root, "source_preset_sha256", record.source_preset_sha256, reason) || !DPCCM05ResumeString(document, root, "raw_sha256", record.raw_sha256, reason) || !DPCCM05ResumeString(document, root, "time_basis", record.time_basis, reason) || !DPCCM05ResumeString(document, root, "runtime_restore", record.runtime_restore, reason) || !DPCCM05ResumeUnsigned(document, root, "noise_seed", ULONG_MAX, record.noise_seed, reason) || !DPCCM05ResumeUnsigned(document, root, "completed_updates", ULONG_MAX, record.completed_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_reference_capacity", UINT_MAX, record.ompb_reference_capacity, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_current_capacity", UINT_MAX, record.ompb_current_capacity, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_reference_count", UINT_MAX, record.ompb_reference_count, reason) || !DPCCM05ResumeUnsigned(document, root, "ompb_current_count", UINT_MAX, record.ompb_current_count, reason) || !DPCCM05ResumeBoolean(document, root, "finished", record.finished, reason) || !DPCCM05ResumeBoolean(document, root, "calibration_completed", record.calibration_completed, reason) || !DPCCM05ResumeBoolean(document, root, "stage02_quality_pass", record.stage02_quality_pass, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_updates", ULONG_MAX, record.price_adapt_updates, reason) || !DPCCM05ResumeUnsigned(document, root, "price_adapt_epochs_completed", UINT_MAX, record.price_adapt_epochs_completed, reason) || !DPCCM05ResumeString(document, root, "verify_status", record.verify_status, reason)) return(false); if((record.stage != "LATENT_PREPARE" && record.stage != "PRICE_JOINT" && record.stage != "OMPB_CALIBRATE" && record.stage != "PRICE_ADAPT" && record.stage != "VERIFY") || !DPCCResolverLowerSha256(record.profile_sha256) || !DPCCResolverLowerSha256(record.market_profile_sha256) || !DPCCResolverLowerSha256(record.source_preset_sha256) || !DPCCResolverLowerSha256(record.raw_sha256) || record.time_basis != "TERMINAL_SERVER" || record.runtime_restore != "WARMUP_FROM_LATEST_WEIGHTS" || record.price_adapt_updates > record.completed_updates || record.ompb_reference_capacity == 0 || record.ompb_current_capacity == 0 || record.ompb_reference_count > record.ompb_reference_capacity || record.ompb_current_count > record.ompb_current_capacity) { reason = "M05_SELECT.source_model_identity"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCM05JsonEqual | //+------------------------------------------------------------------+ bool DPCCM05JsonEqual(CDPCCJsonDocument &left, const int left_node, CDPCCJsonDocument &right, const int right_node) { if(left_node < 0 || right_node < 0 || left.Type(left_node) != right.Type(right_node) || left.ChildCount(left_node) != right.ChildCount(right_node)) return(false); const ENUM_DPCC_JSON_TYPE type = left.Type(left_node); if(type == DPCC_JSON_OBJECT) { for(int child = left.FirstChild(left_node); child >= 0; child = left.NextSibling(child)) { const int other = right.FindMember(right_node, left.Key(child)); if(other < 0 || !DPCCM05JsonEqual(left, child, right, other)) return(false); } return(true); } if(type == DPCC_JSON_ARRAY) { for(int i = 0; i < left.ChildCount(left_node); i++) if(!DPCCM05JsonEqual(left, left.ChildAt(left_node, i), right, right.ChildAt(right_node, i))) return(false); return(true); } return(left.Text(left_node) == right.Text(right_node)); } //+------------------------------------------------------------------+ //| Read the profile-owned cap for one new Market price phase | //+------------------------------------------------------------------+ bool DPCCReadM05PricePhaseBudget(CDPCCJsonDocument &document, const int phases, const string phase, uint &epochs, ulong &max_updates, uint &early_stop_patience, string &reason) { epochs = 0; max_updates = 0; early_stop_patience = 0; const string path = "profile.market_phases." + phase; const int node = document.FindMember(phases, phase); if(node < 0 || !DPCCM05KnownMembers(document, node, "|epochs|max_updates|early_stop_patience|", path, reason)) { if(reason == "") reason = path + ".required"; return(false); } long epoch_value = 0, update_value = 0; if(!DPCCResourceReadLong(document, node, "epochs", epoch_value, reason, path + ".epochs") || !DPCCResourceReadLong(document, node, "max_updates", update_value, reason, path + ".max_updates")) return(false); if(epoch_value <= 0 || epoch_value > UINT_MAX || update_value < 0) { reason = path + ".positive_epochs_and_nonnegative_max_updates_required"; return(false); } const int patience_node = document.FindMember(node, "early_stop_patience"); if(patience_node >= 0 && document.Type(patience_node) != DPCC_JSON_NULL) { long patience_value = 0; if(!DPCCResourceReadLong(document, node, "early_stop_patience", patience_value, reason, path + ".early_stop_patience") || patience_value <= 0 || patience_value > UINT_MAX) { if(reason == "") reason = path + ".early_stop_patience_positive_or_null"; return(false); } early_stop_patience = (uint)patience_value; reason = path + ".early_stop_monitor_undefined"; return(false); } epochs = (uint)epoch_value; max_updates = (ulong)update_value; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Resolve zero cap from the admitted FIT rows and configured | //| epochs | //+------------------------------------------------------------------+ bool DPCCResolveM05PricePhaseBudget(const int fit_count, const uint epochs, const ulong requested_cap, ulong &resolved_cap, string &reason) { resolved_cap = 0; if(fit_count <= 0 || epochs == 0) { reason = "profile.market_phase_budget.positive_fit_count_and_epochs_required"; return(false); } if((ulong)fit_count > ULONG_MAX / (ulong)epochs) { reason = "profile.market_phase_budget.fit_count_times_epochs_overflow"; return(false); } resolved_cap = (requested_cap == 0 ? (ulong)fit_count * (ulong)epochs : requested_cap); reason = ""; return(true); } //+------------------------------------------------------------------+ //| Reject names outside the consumed execution-profile vocabulary | //+------------------------------------------------------------------+ bool DPCCM05KnownMembers(CDPCCJsonDocument &document, const int node, const string vocabulary, const string path, string &reason) { if(document.Type(node) != DPCC_JSON_OBJECT) { reason = path + ".object_required"; return(false); } for(int i = 0; i < document.ChildCount(node); i++) { const string key = document.Key(document.ChildAt(node, i)); if(key == "" || StringFind(key, "|") >= 0 || StringFind(vocabulary, "|" + key + "|") < 0) { reason = path + ".unknown_key." + key; return(false); } } return(true); } //+------------------------------------------------------------------+ //| Parse required Stage01 or Stage02 keys without input fallbacks | //+------------------------------------------------------------------+ bool DPCCParseM05DonorPreset(const string source, SDPCCM05OMPBProfile &profile, string &reason, const uint expected_stage = 2) { ZeroMemory(profile); const string names[] = {"TimeFrame", "RSIPeriod", "RSIPrice", "CCIPeriod", "CCIPrice", "ATRPeriod", "FastPeriod", "SlowPeriod", "SignalPeriod", "MACDPrice", "InpStart", "InpEnd", "InpEpochs", "InpOMPBStage", "InpOMPBSmoke", "InpOMPBEpochs", "InpOMPBReferenceStart", "InpOMPBReferenceEnd", "InpOMPBCalibrationStart", "InpOMPBCalibrationEnd", "InpOMPBSourcePeriod", "InpOMPBTau", "InpOMPBLambdaDis", "InpOMPBLambdaKL", "InpOMPBLambdaAlpha", "InpOMPBAlphaPrior", "InpOMPBMaxKL", "InpRecoverySmoke", "InpRecoverySmokeBatches", "InpRecoverySmokeAge" }; string values[]; if(ArrayResize(values, ArraySize(names)) != ArraySize(names)) { reason = "DONOR_PRESET_INVALID.capacity"; return(false); } for(int i = 0; i < ArraySize(values); i++) values[i] = ""; string lines[]; const int count = StringSplit(source, '\n', lines); for(int i = 0; i < count; i++) { string line = lines[i]; StringTrimLeft(line); StringTrimRight(line); if(line == "" || StringGetCharacter(line, 0) == ';') continue; const int separator = StringFind(line, "="); if(separator <= 0 || StringFind(line, "=", separator + 1) >= 0) { reason = "DONOR_PRESET_INVALID.line." + (string)(i + 1); return(false); } string key = StringSubstr(line, 0, separator); string value = StringSubstr(line, separator + 1); StringTrimRight(key); StringTrimLeft(value); int index = -1; for(int j = 0; j < ArraySize(names); j++) if(names[j] == key) { index = j; break; } if(index < 0 || value == "" || values[index] != "") { reason = "DONOR_PRESET_INVALID." + (index < 0 ? "unknown_key." : "duplicate_or_empty_key.") + key; return(false); } values[index] = value; } //--- Check all donor types before adapting the calibration fields. double numbers[]; if(ArrayResize(numbers, ArraySize(names)) != ArraySize(names)) { reason = "DONOR_PRESET_INVALID.capacity"; return(false); } ArrayInitialize(numbers, 0.0); for(int i = 0; i < ArraySize(names); i++) { if(values[i] == "") { reason = "DONOR_PRESET_INVALID.missing_key." + names[i]; return(false); } if(i == 0) continue; if(i == 14 || i == 27) { if(values[i] != "true" && values[i] != "false") { reason = "DONOR_PRESET_INVALID.boolean." + names[i]; return(false); } continue; } if(i >= 21 && i <= 26) { CDPCCJsonDocument number_document; if(!number_document.Parse(values[i], reason) || !number_document.TryNumber(number_document.Root(), numbers[i]) || !MathIsValidNumber((float)numbers[i])) { reason = "DONOR_PRESET_INVALID.number." + names[i]; return(false); } } else { ulong number = 0; if(!DPCCCheckpointReadUnsigned(values[i], 32535215999, number) || ((i < 10 || i == 12 || i == 13 || i == 15 || i == 20 || i >= 28) && number > (ulong)UINT_MAX)) { reason = "DONOR_PRESET_INVALID.integer." + names[i]; return(false); } numbers[i] = (double)number; } } //--- Donor TimeFrame is provenance; DPCC history follows the selected chart period. if((values[0] != "PERIOD_CURRENT" && !DPCCStudyForecastTimeframeName(values[0])) || numbers[1] <= 0 || numbers[3] <= 0 || numbers[5] <= 0 || numbers[6] <= 0 || numbers[7] <= 0 || numbers[8] <= 0 || numbers[12] <= 0) { reason = "MARKET_INPUT_PROFILE_MISMATCH.donor_timeframe_or_capture_preprocessing"; return(false); } if(!DPCCStudyForecastAppliedPriceValid((long)numbers[2]) || !DPCCStudyForecastAppliedPriceValid((long)numbers[4]) || !DPCCStudyForecastAppliedPriceValid((long)numbers[9])) { reason = "DONOR_PRESET_INVALID.applied_price"; return(false); } if(numbers[10] <= 0 || numbers[11] <= numbers[10] || numbers[12] <= 0 || (expected_stage != 1 && expected_stage != 2) || numbers[13] != (double)expected_stage || values[14] != "false" || values[27] != "false") { reason = "DONOR_PRESET_INVALID.production_stage_required." + (string)expected_stage; return(false); } if(numbers[20] == 0) { reason = "DONOR_PRESET_INVALID.InpOMPBSourcePeriod_positive_required"; return(false); } profile.latent_start = (datetime)(long)numbers[10]; profile.latent_end = (datetime)(long)numbers[11]; profile.latent_epochs = (uint)numbers[12]; profile.reference_start = (datetime)(long)numbers[16]; profile.reference_end = (datetime)(long)numbers[17]; profile.calibration_start = (datetime)(long)numbers[18]; profile.calibration_end = (datetime)(long)numbers[19]; profile.epochs = (uint)numbers[15]; profile.source_anchor_period = (uint)numbers[20]; profile.rsi_price = (ENUM_APPLIED_PRICE)(long)numbers[2]; profile.cci_price = (ENUM_APPLIED_PRICE)(long)numbers[4]; profile.macd_price = (ENUM_APPLIED_PRICE)(long)numbers[9]; profile.tau = (float)numbers[21]; profile.lambda_dis = (float)numbers[22]; profile.lambda_kl = (float)numbers[23]; profile.lambda_alpha = (float)numbers[24]; profile.alpha_prior = (float)numbers[25]; profile.max_kl = (float)numbers[26]; if(profile.epochs == 0 || profile.reference_start < (datetime)(long)numbers[10] || profile.reference_end <= profile.reference_start || profile.reference_end > profile.calibration_start || profile.calibration_end <= profile.calibration_start || profile.calibration_end > (datetime)(long)numbers[11] || profile.tau <= 0 || profile.tau > 1 || profile.lambda_dis < 0 || profile.lambda_kl < 0 || profile.lambda_alpha < 0 || profile.max_kl < 0) { reason = "DONOR_PRESET_INVALID.calibration_domain"; return(false); } reason = ""; return(true); } //--- Direct M05 settings come from advisor inputs; AC-SRM calibration is adapted in memory. struct SDPCCM05DirectInputs { string task_id, task_directory; ENUM_TIMEFRAMES timeframe; ulong noise_seed, max_file_bytes; datetime latent_start, latent_end, reference_start, reference_end, calibration_start, calibration_end; uint latent_epochs, ompb_epochs, source_anchor_period; uint rsi_period, cci_period, price_atr_period, fast_period, slow_period, signal_period; ENUM_APPLIED_PRICE rsi_price, cci_price, macd_price; float tau, lambda_dis, lambda_kl, lambda_alpha, alpha_prior, max_kl; uint price_joint_epochs, price_adapt_epochs; ulong price_joint_max_updates, price_adapt_max_updates; float lambda_price, huber_delta, rho_tolerance; double verify_atol, verify_rtol, verify_max_price_error, verify_min_scenario_coverage; uint verify_min_distinct_scenarios; }; //+------------------------------------------------------------------+ //| Function DPCCM05IdentityVectorJSON | //+------------------------------------------------------------------+ string DPCCM05IdentityVectorJSON(const float value) { string json = "["; for(int i = 0; i < 109; i++) json += (i == 0 ? "" : ",") + StringFormat("%.9g", (double)value); return(json + "]"); } //+------------------------------------------------------------------+ //| Function DPCCBuildM05ExecutionProfileFromInputs | //+------------------------------------------------------------------+ bool DPCCBuildM05ExecutionProfileFromInputs(SDPCCExecutionControl &control, const SDPCCM05DirectInputs &direct, const bool verify_enabled, SDPCCM05OMPBProfile &profile, SDPCCM05VerifyProfile &verify_profile, float &lambda_price, float &huber_delta, float &rho_tolerance, string &reason) { ZeroMemory(profile); ZeroMemory(verify_profile); lambda_price = 0; huber_delta = 0; rho_tolerance = 0; reason = ""; const double finite_values[] = {(double)direct.tau, (double)direct.lambda_dis, (double)direct.lambda_kl, ( double)direct.lambda_alpha, (double)direct.alpha_prior, (double)direct.max_kl, (double)direct.lambda_price, ( double)direct.huber_delta, (double)direct.rho_tolerance, direct.verify_atol, direct.verify_rtol, direct.verify_max_price_error, direct.verify_min_scenario_coverage }; for(int i = 0; i < ArraySize(finite_values); i++) if(!MathIsValidNumber(finite_values[i]) || !MathIsValidNumber((float)finite_values[i])) {reason = "M05_INPUT.nonfinite"; return(false);} if(control.stage != DPCC_STAGE_STUDYFORECAST || direct.task_id == "" || direct.task_directory == "" || direct.timeframe <= PERIOD_CURRENT || PeriodSeconds(direct.timeframe) <= 0 || direct.noise_seed == 0 || direct.max_file_bytes == 0 || direct.latent_start <= 0 || direct.latent_end <= direct.latent_start || direct.reference_start < direct.latent_start || direct.reference_end <= direct.reference_start || direct.reference_end > direct.calibration_start || direct.calibration_end <= direct.calibration_start || direct.calibration_end > direct.latent_end || direct.latent_epochs == 0 || direct.ompb_epochs == 0 || direct.source_anchor_period == 0 || direct.rsi_period == 0 || direct.cci_period == 0 || direct.price_atr_period == 0 || direct.fast_period == 0 || direct.slow_period == 0 || direct.signal_period == 0 || !DPCCStudyForecastAppliedPriceValid((long)direct.rsi_price) || !DPCCStudyForecastAppliedPriceValid((long)direct.cci_price) || !DPCCStudyForecastAppliedPriceValid((long)direct.macd_price) || direct.price_joint_epochs == 0 || direct.price_adapt_epochs == 0 || direct.lambda_price <= 0 || direct.huber_delta <= 0 || direct.rho_tolerance <= 0 || direct.rho_tolerance >= 1 || direct.verify_atol <= 0 || direct.verify_rtol <= 0 || direct.verify_max_price_error <= 0 || direct.verify_min_scenario_coverage <= 0 || direct.verify_min_scenario_coverage > 1 || direct.verify_min_distinct_scenarios == 0 || direct.verify_min_distinct_scenarios > 21) {reason = "M05_INPUT.domain"; return(false);} //--- The shared AC-SRM parser validates an in-memory projection of its calibration preset. const string donor_prefix = "TimeFrame=" + EnumToString(direct.timeframe) + "\n" + "RSIPeriod=" + (string)direct.rsi_period + "\nRSIPrice=" + (string)direct.rsi_price + "\n" + "CCIPeriod=" + (string)direct.cci_period + "\nCCIPrice=" + (string)direct.cci_price + "\n" + "ATRPeriod=" + (string)direct.price_atr_period + "\nFastPeriod=" + ( string)direct.fast_period + "\n" + "SlowPeriod=" + (string)direct.slow_period + "\nSignalPeriod=" + ( string)direct.signal_period + "\n" + "MACDPrice=" + (string)direct.macd_price + "\nInpStart=" + (string)( long)direct.latent_start + "\n" + "InpEnd=" + (string)(long)direct.latent_end + "\nInpEpochs=" + ( string)direct.latent_epochs + "\n"; const string donor_tail = "InpOMPBSmoke=false\nInpOMPBEpochs=" + (string)direct.ompb_epochs + "\n" + "InpOMPBReferenceStart=" + (string)(long)direct.reference_start + "\n" + "InpOMPBReferenceEnd=" + (string)(long)direct.reference_end + "\n" + "InpOMPBCalibrationStart=" + (string)(long)direct.calibration_start + "\n" + "InpOMPBCalibrationEnd=" + (string)(long)direct.calibration_end + "\n" + "InpOMPBSourcePeriod=" + (string)direct.source_anchor_period + "\n" + "InpOMPBTau=" + StringFormat("%.9g", (double)direct.tau) + "\n" + "InpOMPBLambdaDis=" + StringFormat("%.9g", (double)direct.lambda_dis) + "\n" + "InpOMPBLambdaKL=" + StringFormat("%.9g", (double)direct.lambda_kl) + "\n" + "InpOMPBLambdaAlpha=" + StringFormat("%.9g", (double)direct.lambda_alpha) + "\n" + "InpOMPBAlphaPrior=" + StringFormat("%.9g", (double)direct.alpha_prior) + "\n" + "InpOMPBMaxKL=" + StringFormat("%.9g", (double)direct.max_kl) + "\n" + "InpRecoverySmoke=false\nInpRecoverySmokeBatches=2000\nInpRecoverySmokeAge=512\n"; const string market_source = donor_prefix + "InpOMPBStage=1\n" + donor_tail; const string calibration_source = donor_prefix + "InpOMPBStage=2\n" + donor_tail; SDPCCM05OMPBProfile market_donor, calibration_donor; if(!DPCCParseM05DonorPreset(market_source, market_donor, reason, 1) || !DPCCParseM05DonorPreset(calibration_source, calibration_donor, reason, 2)) {reason = "M05_INPUT.AC_SRM_adapter:" + reason; return(false);} if(market_donor.rsi_price != calibration_donor.rsi_price || market_donor.cci_price != calibration_donor.cci_price || market_donor.macd_price != calibration_donor.macd_price) {reason = "M05_INPUT.AC_SRM_stage_mismatch"; return(false);} string market_donor_sha = "", calibration_donor_sha = ""; if(!DPCCCheckpointTextSHA(market_source, market_donor_sha, reason) || !DPCCCheckpointTextSHA(calibration_source, calibration_donor_sha, reason)) return(false); profile = calibration_donor; profile.noise_seed = direct.noise_seed; profile.price_joint_epochs = direct.price_joint_epochs; profile.price_joint_max_updates = direct.price_joint_max_updates; profile.price_adapt_epochs = direct.price_adapt_epochs; profile.price_adapt_max_updates = direct.price_adapt_max_updates; profile.market_preset_path = "EA_INPUTS/AC-SRM/StudyForecast"; profile.market_preset_sha256 = market_donor_sha; profile.calibration_preset_path = "EA_INPUTS/AC-SRM/02Calibration"; profile.calibration_preset_sha256 = calibration_donor_sha; profile.raw_sha256 = ""; lambda_price = direct.lambda_price; huber_delta = direct.huber_delta; rho_tolerance = direct.rho_tolerance; verify_profile.atol = direct.verify_atol; verify_profile.rtol = direct.verify_rtol; verify_profile.max_price_error = direct.verify_max_price_error; verify_profile.min_scenario_coverage = direct.verify_min_scenario_coverage; verify_profile.min_distinct_scenarios = direct.verify_min_distinct_scenarios; verify_profile.has_max_price_error = verify_enabled; verify_profile.has_min_scenario_coverage = verify_enabled; verify_profile.has_min_distinct_scenarios = verify_enabled; const string profile_text = "{\"contract_id\":" + DPCCExecutionJSONString(DPCC_AP_CONTRACT) + ",\"revision\":\"CNET_ACTION6_20261014\",\"profile_kind\":\"USER_ASSIGNED\"" + ",\"task_id\":" + DPCCExecutionJSONString(direct.task_id) + ",\"symbol_source\":\"CHART_SYMBOL\",\"timeframe\":" + DPCCExecutionJSONString( EnumToString(direct.timeframe)) + ",\"shape\":{},\"diffusion\":{\"seed\":" + (string)direct.noise_seed + "}" + ",\"price\":{\"atr_period\":" + (string)direct.price_atr_period + ",\"lambda_price\":" + StringFormat("%.9g", (double)direct.lambda_price) + ",\"huber_delta\":" + StringFormat("%.9g", (double)direct.huber_delta) + ",\"rho_tolerance\":" + StringFormat("%.9g", (double)direct.rho_tolerance) + "}" + ",\"action\":{},\"projection\":{},\"numeric\":{\"atol\":" + DoubleToString( direct.verify_atol, 9) + ",\"rtol\":" + DoubleToString(direct.verify_rtol, 9) + "},\"data\":{},\"finance\":{}" + ",\"templates\":{},\"limits\":{\"max_price_error\":" + DoubleToString( direct.verify_max_price_error, 9) + ",\"min_scenario_coverage\":" + DoubleToString(direct.verify_min_scenario_coverage, 9) + ",\"min_distinct_scenarios\":" + (string)direct.verify_min_distinct_scenarios + "}" + ",\"persistence\":{\"task_directory\":" + DPCCExecutionJSONString( direct.task_directory) + ",\"max_file_bytes\":" + (string)direct.max_file_bytes + "},\"acceptance\":{}" + ",\"condition\":{\"normalization_source\":\"DIRECT_Z_ACCOUNT_IDENTITY_V1\"" + ",\"scale\":" + DPCCM05IdentityVectorJSON(1.0f) + ",\"bias\":" + DPCCM05IdentityVectorJSON(0.0f) + "}" + ",\"donor\":{\"market_preset\":{\"file\":\"EA_INPUTS/AC-SRM/StudyForecast\",\"sha256\":\"" + market_donor_sha + "\"}" + ",\"ompb_calibration_preset\":{\"file\":\"EA_INPUTS/AC-SRM/02Calibration\",\"sha256\":\"" + calibration_donor_sha + "\"}}" + ",\"market_phases\":{\"latent_prepare\":{\"start\":" + (string)( long)direct.latent_start + ",\"end\":" + (string)(long)direct.latent_end + ",\"epochs\":" + ( string)direct.latent_epochs + "}" + ",\"price_joint\":{\"epochs\":" + (string)direct.price_joint_epochs + ",\"max_updates\":" + (string)direct.price_joint_max_updates + "}" + ",\"ompb_calibrate\":{\"epochs\":" + (string)direct.ompb_epochs + ",\"source_anchor_period\":" + (string)direct.source_anchor_period + "}" + ",\"price_adapt\":{\"epochs\":" + (string)direct.price_adapt_epochs + ",\"max_updates\":" + (string)direct.price_adapt_max_updates + "}}" + ",\"resource_target\":{}}"; string assigned_sha = "", market_sha = ""; CDPCCJsonDocument document; if(!DPCCCheckpointTextSHA(profile_text, assigned_sha, reason) || !document.Parse(profile_text, reason) || !DPCCExecutionMarketProfileSHA(document, document.Root(), market_sha, reason)) {reason = "M05_INPUT.profile:" + reason; return(false);} control.profile_text = profile_text; control.profile_sha256 = assigned_sha; control.market_profile_sha256 = market_sha; control.task_id = direct.task_id; profile.assigned_profile_sha256 = assigned_sha; profile.market_profile_sha256 = market_sha; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Encode metadata strings without changing their UTF-8 contents | //+------------------------------------------------------------------+ string DPCCAPJSONString(const string value) { return(DPCCExecutionJSONString(value)); } //+------------------------------------------------------------------+ //| Describe the actual native graph used by this Market writer | //+------------------------------------------------------------------+ bool DPCCAPM05Graph(CNet &model, const int count, string &graph, string &reason) { graph = "["; for(int i = 0; i < count; i++) { CNeuronBaseOCL *layer = model.Layer(i); if(!layer) { reason = "M05_SAVE.graph.layer_missing"; return(false); } uint abi = 0; if(layer.Type() == defNeuronScenarioForecastPriceOCL) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(layer); if(!price) return(false); abi = price.ABI(); } else if(layer.Type() == defNeuronScenarioDecoderOCL) { CNeuronScenarioDecoderOCL *decoder = dynamic_cast(layer); if(!decoder) return(false); abi = decoder.ABI(); } if(i > 0) graph += ","; CLayerDescription *description = layer.GetLayerInfo(); if(!description) { reason = "M05_SAVE.graph.description_missing"; return(false); } graph += "{\"index\":" + (string)i + ",\"type\":" + (string)layer.Type() + ",\"abi\":" + (string)abi + ",\"neurons\":" + (string)layer.Neurons() + ",\"count\":" + (string)description.count + ",\"window\":" + (string)description.window + ",\"window_out\":" + (string)description.window_out + ",\"step\":" + (string)description.step + ",\"layers\":" + (string)description.layers + ",\"batch\":" + (string)description.batch + ",\"variables\":" + (string)description.variables + ",\"activation\":" + (string)description.activation + ",\"optimization\":" + (string)description.optimization + ",\"probability\":" + StringFormat("%.9g", (double)description.probability) + ",\"units\":["; for(int j = 0; j < ArraySize(description.units); j++) graph += (j == 0 ? "" : ",") + (string)description.units[j]; graph += "],\"windows\":["; for(int j = 0; j < ArraySize(description.windows); j++) graph += (j == 0 ? "" : ",") + (string)description.windows[j]; graph += "],\"heads\":["; for(int j = 0; j < ArraySize(description.heads); j++) graph += (j == 0 ? "" : ",") + (string)description.heads[j]; graph += "]}"; delete description; } graph += "]"; if(model.Layer(count) != NULL) { reason = "M05_SAVE.graph.extra_layer"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Describe one native Trading CNet graph and its published layer | //| ABI | //+------------------------------------------------------------------+ bool DPCCAPTradingGraph(CNet &model, const int count, string &graph, string &reason) { graph = "["; if(count < 2 || count > 16) { reason = "TRADING_SAVE.graph.count"; return(false); } for(int i = 0; i < count; i++) { CNeuronBaseOCL *layer = model.Layer(i); CLayerDescription *description = (CheckPointer(layer) == POINTER_INVALID ? NULL : layer.GetLayerInfo()); uint abi = 0; if(CheckPointer(layer) == POINTER_INVALID || CheckPointer(description) == POINTER_INVALID || !DPCCAPNativeLayerABI(layer, description, abi)) { if(CheckPointer(description) != POINTER_INVALID) delete description; reason = "TRADING_SAVE.graph.layer_or_abi"; return(false); } if(i > 0) graph += ","; graph += "{\"index\":" + (string)i + ",\"type\":" + (string)layer.Type() + ",\"abi\":" + (string)abi + ",\"neurons\":" + (string)layer.Neurons() + ",\"count\":" + (string)description.count + ",\"window\":" + (string)description.window + ",\"window_out\":" + (string)description.window_out + ",\"step\":" + (string)description.step + ",\"layers\":" + (string)description.layers + ",\"batch\":" + (string)description.batch + ",\"variables\":" + (string)description.variables + ",\"activation\":" + (string)description.activation + ",\"optimization\":" + (string)description.optimization + ",\"probability\":" + StringFormat("%.9g", (double)description.probability) + ",\"units\":["; for(int j = 0; j < ArraySize(description.units); j++) graph += (j == 0 ? "" : ",") + (string)description.units[j]; graph += "],\"windows\":["; for(int j = 0; j < ArraySize(description.windows); j++) graph += (j == 0 ? "" : ",") + (string)description.windows[j]; graph += "],\"heads\":["; for(int j = 0; j < ArraySize(description.heads); j++) graph += (j == 0 ? "" : ",") + (string)description.heads[j]; graph += "]}"; delete description; } graph += "]"; if(model.Layer(count) != NULL) { reason = "TRADING_SAVE.graph.extra_layer"; return(false); } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Hard-copy the same ABI3 Action6 roles into fresh frozen targets | //+------------------------------------------------------------------+ bool DPCCSyncAction6TradingTargets(CNet &market, CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, string &reason) { reason = ""; CNet *nets[] = {GetPointer(market), GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; const int layer_counts[] = {8, 2, 5, 5, 2, 5, 5}; COpenCLMy *context = market.GetOpenCL(); if(CheckPointer(context) == POINTER_INVALID || !market.IsOpenCLTransferReady() || market.WeightsUpdateEnabled() || market.Layer(8) != NULL) { reason = "TRADING_SAVE.action6.market_context_or_frozen"; return(false); } for(int i = 0; i < ArraySize(nets); i++) { if(CheckPointer(nets[i]) == POINTER_INVALID || !nets[i].IsOpenCLTransferReady() || nets[i].GetOpenCL() != context) { reason = StringFormat("TRADING_SAVE.action6.net_context.%d", i); return(false); } for(int a = 0; a < layer_counts[i]; a++) if(CheckPointer(nets[i].Layer(a)) == POINTER_INVALID || nets[i].Layer(a).GetOpenCL() != context || (i == 0 && nets[i].Layer(a).TrainMode())) { reason = StringFormat("TRADING_SAVE.action6.layer_context_or_market_train.%d", i); return(false); } for(int j = 0; j < i; j++) { if(nets[i] == nets[j]) { reason = "TRADING_SAVE.action6.net_alias"; return(false); } for(int a = 0; a < layer_counts[i]; a++) for(int b = 0; b < layer_counts[j]; b++) if(nets[i].Layer(a) == nets[j].Layer(b)) { reason = "TRADING_SAVE.action6.layer_alias"; return(false); } } } CNeuronDPCCOCL *online_top = dynamic_cast(actor.Layer(1)); CNeuronDPCCOCL *target_top = dynamic_cast(target_actor.Layer(1)); if(actor.Layer(2) != NULL || target_actor.Layer(2) != NULL || !online_top || !target_top || online_top.Type() != defNeuronDPCCOCL || target_top.Type() != defNeuronDPCCOCL || online_top.ActorABI() != 3 || target_top.ActorABI() != 3 || online_top.GetOpenCL() != context || target_top.GetOpenCL() != context || (online_top.LastStatus() != DPCC_OK && online_top.LastStatus() != DPCC_NO_FEASIBLE_CANDIDATE) || (target_top.LastStatus() != DPCC_OK && target_top.LastStatus() != DPCC_NO_FEASIBLE_CANDIDATE) || !NNQCriticGraph(GetPointer(q1)) || !NNQCriticGraph(GetPointer(q2)) || !NNQCriticGraph(GetPointer(target_q1)) || !NNQCriticGraph(GetPointer(target_q2))) { reason = "TRADING_SAVE.action6.graph_contract"; return(false); } //--- Bind each Actor to the exact same Market and its corresponding Q pair. //--- This public ABI3 check also clears any stale, unpublished decision state. if(!online_top.BindPolicyNetworks(GetPointer(market), GetPointer(q1), GetPointer(q2)) || !target_top.BindPolicyNetworks(GetPointer(market), GetPointer(target_q1), GetPointer(target_q2))) { reason = "TRADING_SAVE.action6.policy_binding"; return(false); } string actor_graph = "", target_actor_graph = "", q1_graph = "", target_q1_graph = "", q2_graph = "", target_q2_graph = ""; if(!DPCCAPTradingGraph(actor, 2, actor_graph, reason) || !DPCCAPTradingGraph(target_actor, 2, target_actor_graph, reason) || !DPCCAPTradingGraph(q1, 5, q1_graph, reason) || !DPCCAPTradingGraph(target_q1, 5, target_q1_graph, reason) || !DPCCAPTradingGraph(q2, 5, q2_graph, reason) || !DPCCAPTradingGraph(target_q2, 5, target_q2_graph, reason)) return(false); if(actor_graph != target_actor_graph || q1_graph != target_q1_graph || q2_graph != target_q2_graph) { reason = "TRADING_SAVE.action6.target_graph_mismatch"; return(false); } //--- Freeze every target before copying. CNet::WeightsUpdate(tau=1) copies //--- parameters without enabling training or modifying optimizer state. CNet *targets[] = {GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2)}; for(int i = 0; i < ArraySize(targets); i++) { const int target_layers = (i == 0 ? 2 : 5); if(!targets[i].TrainMode(false) || !targets[i].SetWeightsUpdate(false) || targets[i].WeightsUpdateEnabled()) { reason = StringFormat("TRADING_SAVE.action6.target_freeze.%d", i); return(false); } for(int l = 0; l < target_layers; l++) if(targets[i].Layer(l).TrainMode()) { reason = StringFormat("TRADING_SAVE.action6.target_layer_train.%d.%d", i, l); return(false); } } if(!target_q1.WeightsUpdate(GetPointer(q1), 1.0f)) { reason = "TRADING_SAVE.action6.target_q1_copy"; return(false); } if(!target_q2.WeightsUpdate(GetPointer(q2), 1.0f)) { reason = "TRADING_SAVE.action6.target_q2_copy"; return(false); } if(!target_actor.WeightsUpdate(GetPointer(actor), 1.0f)) { reason = "TRADING_SAVE.action6.target_actor_copy"; return(false); } if(target_top.LastStatus() != DPCC_OK || target_actor.WeightsUpdateEnabled() || target_q1.WeightsUpdateEnabled() || target_q2.WeightsUpdateEnabled()) { reason = "TRADING_SAVE.action6.target_post_copy_state"; return(false); } for(int i = 0; i < ArraySize(targets); i++) { const int target_layers = (i == 0 ? 2 : 5); for(int l = 0; l < target_layers; l++) if(targets[i].Layer(l).TrainMode()) { reason = StringFormat("TRADING_SAVE.action6.target_post_copy_train.%d.%d", i, l); return(false); } } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Stream a bounded metadata section directly into its role frame | //+------------------------------------------------------------------+ bool DPCCAPWriteJSON(CDPCCAPRoleWriter &writer, const uint section, const string text, string &reason) { CDPCCJsonDocument document; if(!document.Parse(text, reason)) return(false); uchar bytes[]; int length = StringToCharArray(text, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(length <= 1 || length > 1048577) { reason = "dpccap.metadata.size"; return(false); } if(bytes[length - 1] == 0) length--; int handle = INVALID_HANDLE; if(!writer.BeginSection(section, 1, handle, reason)) return(false); if(FileWriteArray(handle, bytes, 0, length) != (uint)length) { reason = "dpccap.metadata.write"; return(false); } return(writer.EndSection(reason)); } //+------------------------------------------------------------------+ //| Check an older Market generation using its recorded source ABI | //+------------------------------------------------------------------+ bool DPCCAPValidateMarketFile(const string path, const ulong maximum, string &reason) { const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_COMMON); if(handle == INVALID_HANDLE) { reason = "M05_SAVE.existing.open"; return(false); } uchar header[]; SDPCCAPSection sections[]; const bool valid = (DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason) && DPCCAPValidateHandle(handle, 1, DPCCAPReadSHA(header, 40), maximum, 64, sections, reason)); FileClose(handle); return(valid); } //+------------------------------------------------------------------+ //| Accept only a complete Trading frame with its six native roles | //+------------------------------------------------------------------+ bool DPCCAPValidateTradingFile(const string path, const bool common, const ulong maximum, string &reason) { reason = ""; const int common_flag = (common ? FILE_COMMON : 0); const int handle = FileOpen(path, FILE_READ | FILE_BIN | common_flag); if(handle == INVALID_HANDLE) { reason = "TRADING_SAVE.existing.open"; return(false); } uchar header[]; SDPCCAPSection sections[]; bool valid = DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason) && DPCCAPMagic(header, 0, "DPCCAP01") && DPCCAPReadUnsigned(header, 12, 4) == 2 && DPCCAPValidateHandle(handle, 2, DPCCAPReadSHA(header, 40), maximum, 8, sections, reason) && ArraySize(sections) == 8; if(valid) { const uint expected_ids[] = {1, 110, 111, 112, 113, 114, 115, 200}; for(int i = 0; i < 8; i++) if(sections[i].id != expected_ids[i] || sections[i].encoding != (i == 0 || i == 7 ? 1 : 2)) { valid = false; reason = "TRADING_SAVE.existing.sections"; break; } } if(valid) { uchar bytes[]; string text = ""; const int length = (int)sections[0].length; if(length <= 0 || length > 1048576 || ArrayResize(bytes, length) != length || !FileSeek(handle, (long)sections[0].offset, SEEK_SET) || FileReadArray(handle, bytes, 0, length) != (uint)length || !DPCCStrictUtf8Decode(bytes, length, 1048576, text, reason)) { valid = false; if(reason == "") reason = "TRADING_SAVE.existing.manifest_read"; } else { CDPCCJsonDocument document; if(!document.Parse(text, reason)) { valid = false; FileClose(handle); return(false); } const int root = document.Root(); string contract = "", revision = "", role = "", source_abi = "", market_dependency = ""; const string header_abi = DPCCAPReadSHA(header, 40); if(!DPCCResourceReadString(document, root, "contract_id", contract, reason, "trading.manifest.contract") || !DPCCResourceReadString(document, root, "revision", revision, reason, "trading.manifest.revision") || !DPCCResourceReadString(document, root, "role", role, reason, "trading.manifest.role") || !DPCCResourceReadString(document, root, "source_abi_sha256", source_abi, reason, "trading.manifest.abi") || !DPCCResourceReadString(document, root, "market_dependency_sha256", market_dependency, reason, "trading.manifest.market_dependency") || contract != DPCC_AP_CONTRACT || revision != DPCC_TRADING_REVISION || role != "Trading" || source_abi != header_abi || !DPCCResolverLowerSha256(market_dependency)) { valid = false; if(reason == "") reason = "TRADING_SAVE.existing.manifest_binding"; } } } FileClose(handle); return(valid); } //+------------------------------------------------------------------+ //| Publish six roles once under the shared current/backup writer | //| lock | //+------------------------------------------------------------------+ bool DPCCAPSaveTradingGeneration(const string directory, const bool common, const ulong maximum, const string source_abi, const string manifest_text, const string resume_text, CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const string source_path, const string source_sha, string &output_path, string &output_sha, string &reason) { output_path = ""; output_sha = ""; reason = ""; const string current = directory + "/Trading.current.dpc"; const string backup = directory + "/Trading.bak.dpc"; const string lock_path = directory + "/Trading.writer.lock"; string canonical = ""; if(!DPCCCheckpointCanonicalPath(current, canonical, reason) || canonical != current || !DPCCCheckpointCanonicalPath(backup, canonical, reason) || canonical != backup || !DPCCCheckpointCanonicalPath(lock_path, canonical, reason) || canonical != lock_path || (source_path != "" && source_path != current && source_path != backup) || (source_path == "" ? source_sha != "" : !DPCCResolverLowerSha256(source_sha))) { reason = "TRADING_SAVE.canonical_or_source_binding:" + reason; return(false); } CDPCCJsonDocument json; if(!json.Parse(manifest_text, reason) || !json.Parse(resume_text, reason)) return(false); const int common_flag = (common ? FILE_COMMON : 0); const int lock = FileOpen(lock_path, FILE_READ | FILE_WRITE | FILE_BIN | common_flag); if(lock == INVALID_HANDLE) { reason = "TRADING_SAVE.writer_lock"; return(false); } CDPCCAPRoleWriter writer; bool saved = false; do { //--- Recheck the loaded source after obtaining exclusive publication ownership. if(source_path != "") { string observed = ""; if(!DPCCCheckpointFileSHA(source_path, common, maximum, observed, reason) || observed != source_sha) { reason = "TRADING_SAVE.loaded_generation_changed:" + reason; break; } } if(FileIsExist(current, common_flag)) { string current_reason = ""; if(DPCCAPValidateTradingFile(current, common, maximum, current_reason)) { if(source_path == backup) { reason = "TRADING_SAVE.new_current_supersedes_loaded_backup"; break; } if(!FileMove(current, common_flag, backup, common_flag | FILE_REWRITE)) { reason = "TRADING_SAVE.current_to_bak"; break; } } else { if(!FileIsExist(backup, common_flag) || !DPCCAPValidateTradingFile(backup, common, maximum, reason)) { reason = "TRADING_SAVE.current_invalid_without_valid_bak:" + current_reason; break; } if(!FileDelete(current, common_flag)) { reason = "TRADING_SAVE.invalid_current_remove"; break; } } } CNet *roles[] = {GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; if(!writer.Open(current, common, 2, source_abi, 8, maximum, reason) || !DPCCAPWriteJSON(writer, 1, manifest_text, reason)) break; bool roles_written = true; for(int role = 0; role < 6 && roles_written; role++) roles_written = writer.WriteTradingNativeSection((uint)(110 + role), *roles[role], reason); if(!roles_written || !DPCCAPWriteJSON(writer, 200, resume_text, reason) || !writer.Finish(reason)) break; if(!DPCCCheckpointFileSHA(current, common, maximum, output_sha, reason)) break; output_path = current; saved = true; } while(false); writer.Close(); FileClose(lock); return(saved); } //+------------------------------------------------------------------+ //| Frozen Market prefix that preserves the loaded Decoder layer | //+------------------------------------------------------------------+ class CDPCCMarketCNet : public CNet { private: bool m_prefix_failed; public: CDPCCMarketCNet(void) : m_prefix_failed(false) {} //+------------------------------------------------------------------+ //| Run only the existing encoder layers on one admitted input | //+------------------------------------------------------------------+ bool ForwardEncoderPrefix(CBufferFloat *market_input, string &reason) { reason = ""; if(m_prefix_failed || IsStopped() || !IsOpenCLTransferReady() || bWeightsUpdate || CheckPointer(opencl) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID || layers.Total() != 8 || CheckPointer(market_input) == POINTER_INVALID) { reason = "market_prefix.owner_or_boundary"; return(false); } CNeuronBaseOCL *nodes[7]; const int types[] = {defNeuronBaseOCL, defNeuronBatchNormOCL, defNeuronDropoutOCL, defNeuronCogDriverData, defNeuronCogDriverRankTCM, defNeuronOMPBOCL, defNeuronScenarioForecastPriceOCL }; for(int i = 0; i < 7; i++) { CLayer *layer = dynamic_cast(layers.At(i)); if(!layer || layer.Total() != 1) { reason = "market_prefix.layer_shape." + (string)i; return(false); } nodes[i] = dynamic_cast(layer.At(0)); if(!nodes[i] || nodes[i].Type() != types[i] || nodes[i].GetOpenCL() != opencl || nodes[i].TrainMode() || (i > 0 && !nodes[i].CanStartForward())) { reason = "market_prefix.layer_policy." + (string)i; return(false); } if(nodes[i].OwnsBufferOrDeviceSlot(market_input)) { reason = "market_prefix.input_alias." + (string)i; return(false); } } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(nodes[6]); CNeuronOMPBOCL *ompb = dynamic_cast(nodes[5]); CBufferFloat *destination = nodes[0].getOutput(); const ulong count = (ulong)nodes[0].Neurons(); if(!forecast || forecast.ABI() != defScenarioForecastPriceABI || forecast.CodebookUpdate() || forecast.PricePhase() != defForecastPriceVerify || !ompb || ompb.Mode() != OMPB_INFERENCE || ompb.SourceAnchor() || !NNQValid(opencl, market_input, count) || market_input.Total() != (int)count || !NNQValid(opencl, destination, count) || destination.Total() != (int)count) { reason = "market_prefix.input_or_forecast_contract"; return(false); } //--- The phase setter rejects a still-live Forecast frame before trunk state can advance. if(!forecast.SetPricePhase(defForecastPriceVerify)) { reason = "market_prefix.forecast_frame_pending"; return(false); } const int copy_kernel = def_k_CopyBufferRaw; const uint copy_offset[] = {0}; const uint copy_size[] = {(uint)count}; if(!opencl.SetArgumentBuffer(copy_kernel, 0, market_input.GetIndex())) { m_prefix_failed = true; reason = "market_prefix.input_copy.source_argument"; return(false); } if(!opencl.SetArgumentBuffer(copy_kernel, 1, destination.GetIndex())) { m_prefix_failed = true; reason = "market_prefix.input_copy.destination_argument"; return(false); } if(!opencl.SetArgument(copy_kernel, 2, (int)count)) { m_prefix_failed = true; reason = "market_prefix.input_copy.count_argument"; return(false); } if(!opencl.Execute(copy_kernel, 1, copy_offset, copy_size)) { m_prefix_failed = true; reason = "market_prefix.input_copy.execute"; return(false); } for(int i = 1; i < 7; i++) { if(IsStopped() || !nodes[i].FeedForward(nodes[i - 1])) { m_prefix_failed = true; reason = "market_prefix.forward." + (string)i; return(false); } } return(true); } }; //+------------------------------------------------------------------+ //| Declaration CDPCCStudyForecastCore | //+------------------------------------------------------------------+ class CDPCCStudyForecastCore { private: CDPCCMarketCNet m_market; CNet m_target; CDPCCStudyForecastGradientOps m_gradient_ops; SDPCCStudyForecastBar m_bars[]; int m_fit[], m_holdout[], m_latent_fit[]; uint m_latent_epochs_completed; CBufferFloat m_state, m_future, m_future_latent, m_zero, m_negative, m_delta; ulong m_updates; int m_cursor, m_holdout_cursor, m_fit_limit; uint m_max_updates; string m_raw_sha, m_root; bool m_ready, m_finished, m_failed, m_epoch_started, m_full_run, m_holdout_started; double m_holdout_latent, m_holdout_valid, m_holdout_invalid; ulong m_holdout_market_fingerprint, m_holdout_target_fingerprint; ulong m_decoder_market_fingerprint, m_decoder_target_fingerprint; bool m_restore_only; bool m_encoder_only; ulong m_teacher_initial_fingerprint; bool m_z_available; int m_inference_split, m_inference_cursor; SDPCCStudyForecastBar m_live_bars[]; int m_live_count; long m_raw_successor, m_live_next_open; bool m_live_prepared; bool m_price_forward_only, m_price_quote_basis_resolved, m_price_h1_ohlc_targets; bool m_raw_identity_bound; bool m_ready_encoder_restored, m_ready_encoder_live; int m_ready_encoder_cursor; ulong m_ready_encoder_generation; string m_price_server, m_price_symbol; ENUM_TIMEFRAMES m_price_timeframe; ENUM_SYMBOL_CHART_MODE m_price_quote_mode; CBufferFloat m_price_native, m_price_target; bool m_price_joint; float m_price_lambda, m_price_huber, m_price_rho_tolerance; double m_price_loss_sum; ulong m_price_joint_ompb_fingerprint, m_price_latent_decoder_fingerprint; ulong m_price_joint_updates; uint m_price_study_phase; uint m_price_joint_epoch_limit, m_price_joint_epochs_completed; ulong m_price_joint_update_limit; uint m_price_adapt_epoch_limit, m_price_adapt_epochs_completed; ulong m_price_adapt_update_limit; string m_price_phase_terminal_status; SDPCCM05OMPBProfile m_ompb_profile; bool m_ompb_configured, m_ompb_reference_complete, m_ompb_epoch_started; bool m_ompb_source_pending; bool m_ompb_calibration_completed, m_ompb_stage02_quality_pass; //--- Lanes: BYPASS source/target, pre-INFERENCE source/target, post-INFERENCE source/target. int m_ompb_evaluation_stage, m_ompb_evaluation_cursor; double m_ompb_evaluation_loss[6], m_ompb_evaluation_valid[6], m_ompb_evaluation_invalid[6]; ulong m_ompb_evaluation_posterior; int m_ompb_source_evaluation[], m_ompb_target_evaluation[]; int m_ompb_reference[], m_ompb_calibration[], m_ompb_reference_cursor; uint m_ompb_epoch, m_ompb_reference_capacity, m_ompb_current_capacity; ulong m_ompb_target_updates, m_ompb_source_updates, m_ompb_frozen_fingerprint; ulong m_ompb_initial_fingerprint; uint m_ompb_forecast_steps[3], m_ompb_decoder_updates; uint m_ompb_epoch_source_updates; bool m_m05, m_m05_manifest_bound; ulong m_price_adapt_frozen_fingerprint, m_price_adapt_updates; uint m_price_adapt_decoder_updates; double m_price_adapt_loss_sum; SDPCCM05VerifyProfile m_verify_profile; bool m_verify_configured, m_verify_started; double m_verify_error_sum[48], m_verify_max_price_error, m_verify_coverage; uint m_verify_snapshots, m_verify_covered, m_verify_min_distinct; ulong m_verify_market_fingerprint, m_verify_target_fingerprint; string m_verify_status; SDPCCHistoricalPriceSnapshot m_price_observation; bool NetworkTrainingEnabled(CNet &network) { CNeuronBaseOCL *last = network.Layer(-1); if(CheckPointer(last) == POINTER_INVALID) return(true); for(int i = 0; i < 1024; i++) { CNeuronBaseOCL *layer = network.Layer(i); if(CheckPointer(layer) == POINTER_INVALID) return(true); if(layer.TrainMode()) return(true); if(layer == last) return(false); } return(true); } bool HasTeacherNetwork(void) { return(m_target.Layer(0) != NULL); } bool MissingTeacherAllowed(void) { return(m_price_forward_only || m_price_joint || m_m05); } bool TeacherFrozenPolicy(void) { if(!HasTeacherNetwork()) return(MissingTeacherAllowed()); return(!NetworkTrainingEnabled(m_target) && !m_target.WeightsUpdateEnabled()); } bool SetTeacherFrozenPolicy(void) { if(!HasTeacherNetwork()) return(MissingTeacherAllowed()); return(m_target.TrainMode(false) && m_target.SetWeightsUpdate(false) && TeacherFrozenPolicy()); } bool ClearTeacherState(void) { if(!HasTeacherNetwork()) return(MissingTeacherAllowed()); return(m_target.Clear()); } bool TeacherFingerprint(ulong &fingerprint) { if(!HasTeacherNetwork()) { fingerprint = 0; return(MissingTeacherAllowed()); } return(m_target.ParameterFingerprint(fingerprint)); } bool FrozenProducer(string &reason) { if(m_ready_encoder_restored) { CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronBaseOCL *ompb_base = ompb; if(!m_restore_only || !m_price_forward_only || m_encoder_only || !forecast || !ompb || m_market.WeightsUpdateEnabled() || m_target.Layer(0) != NULL || forecast.CodebookUpdate() || forecast.PricePhase() != defForecastPriceVerify || ompb_base.TrainMode() || ompb.Mode() != OMPB_INFERENCE || ompb.SourceAnchor()) return(Fail("ready_encoder.frozen_producer_state", reason)); for(int i = 0; i < 8; i++) if(!m_market.Layer(i) || m_market.Layer(i).TrainMode()) return(Fail("ready_encoder.trainable_layer", reason)); return(true); } if(m_encoder_only && m_restore_only) { CNeuronBaseOCL *encoder_input = m_market.Layer(0); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronBaseOCL *ompb_base = ompb; if(!encoder_input || encoder_input.TrainMode() || NetworkTrainingEnabled(m_market) || m_market.WeightsUpdateEnabled() || m_target.Layer(0) != NULL || !forecast || forecast.CodebookUpdate() || !ompb || ompb_base.TrainMode() || ompb.Mode() != OMPB_BYPASS || ompb.SourceAnchor()) return(Fail("study_encoder.restore_producer_state", reason)); return(true); } CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(NetworkTrainingEnabled(m_market) || m_market.WeightsUpdateEnabled() || !TeacherFrozenPolicy() || !forecast || forecast.CodebookUpdate()) return(Fail("study_decoder.frozen_producer_changed", reason)); return(true); } bool VerifyFrozenProducerFingerprints(string &reason) { if(!FrozenProducer(reason)) return(false); ulong market = 0, target = 0; if(!m_market.ParameterFingerprint(market) || market != m_decoder_market_fingerprint || !TeacherFingerprint(target) || target != m_decoder_target_fingerprint) return(Fail("study_decoder.frozen_producer_changed", reason)); return(true); } //+------------------------------------------------------------------+ //| PublishZ copies a finite latent snapshot owned by the caller | //+------------------------------------------------------------------+ bool PublishZ(float &z[], string &reason) { ArrayResize(z, 0); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); float candidate[]; if(!forecast || forecast.CodebookUpdate() || !forecast.GetZ() || forecast.GetZ().GetOpenCL() != m_market.GetOpenCL() || forecast.GetZ().Total() != 21 * 9 * 12 * 16 || forecast.GetZ().GetData(candidate) != 21 * 9 * 12 * 16) return(Fail("study_encoder.inference.z_shape_or_context", reason)); for(int i = 0; i < ArraySize(candidate); i++) if(!MathIsValidNumber(candidate[i])) return(Fail("study_encoder.inference.z_nonfinite", reason)); if(ArrayCopy(z, candidate) != 21 * 9 * 12 * 16) { ArrayResize(z, 0); return(Fail("study_encoder.inference.z_publish", reason)); } reason = ""; m_z_available = true; return(true); } //+------------------------------------------------------------------+ //| SaveEncoderCompleted publishes only the new two-network run | //+------------------------------------------------------------------+ bool SaveEncoderCompleted(string &reason) { if(!m_encoder_only || !Finished() || !m_full_run || m_restore_only || m_cursor != ArraySize(m_fit) || m_updates != (ulong)ArraySize(m_fit) || m_holdout_cursor != ArraySize(m_holdout) || m_holdout_valid != (double)ArraySize(m_holdout) || m_holdout_invalid != 0 || !MathIsValidNumber(m_holdout_latent) || m_holdout_latent < 0 || !VerifyFrozenProducerFingerprints(reason) || m_decoder_target_fingerprint != m_teacher_initial_fingerprint) return(Fail("study_encoder.save.before_successful_holdout", reason)); const string encoder_path = m_root + "/Encoder.trained.nnw"; const string teacher_path = m_root + "/Teacher.frozen_untrained.nnw"; const string receipt_path = m_root + "/EncoderTeacher.study.txt"; if(FileIsExist(encoder_path) || FileIsExist(teacher_path) || FileIsExist(receipt_path)) return(Fail("study_encoder.save.output_exists", reason)); const datetime now = TimeCurrent(); if(!m_market.Save(encoder_path, 0, 0, 0, now, false)) return(Fail("study_encoder.save.encoder", reason)); if(!m_target.Save(teacher_path, 0, 0, 0, now, false)) return(Fail("study_encoder.save.teacher", reason)); string encoder_sha = "", teacher_sha = "", profile_sha = "", receipt_sha = ""; if(!DPCCCheckpointFileSHA(encoder_path, false, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, encoder_sha, reason) || !DPCCCheckpointFileSHA(teacher_path, false, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, teacher_sha, reason) || !DPCCStudyEncoderProfileSHA(profile_sha, reason) || !VerifyFrozenProducerFingerprints(reason)) return(Fail("study_encoder.save.artifact_binding:" + reason, reason)); const string loss = DoubleToString(m_holdout_latent, 17); const string wire = "DPCC_ENCODER_TEACHER_STUDY_V1\nraw_sha256=" + m_raw_sha + "\nencoder_sha256=" + encoder_sha + "\nteacher_frozen_untrained_sha256=" + teacher_sha + "\nprofile_sha256=" + profile_sha + "\nfit_updates=" + (string)m_updates + "\nholdout_examples=" + (string)m_holdout_cursor + "\nlatent_coordinate_mae=" + loss + "\nholdout_server_time=2026.01.01..2026.02.01\nteacher=fresh_frozen_untrained" "\ntimezone_resolution=NOT_CLAIMED\nresearch_acceptance=NOT_CLAIMED\n"; if(!DPCCCheckpointTextSHA(wire, receipt_sha, reason) || !DPCCCheckpointWriteText(receipt_path, false, wire, reason)) return(Fail("study_encoder.save.receipt:" + reason, reason)); SDPCCStudyEncoderReceipt decoded; if(!DPCCReadStudyEncoderReceipt(m_root, receipt_sha, m_raw_sha, (uint)ArraySize(m_fit), (uint)ArraySize(m_holdout), decoded, reason) || decoded.encoder_sha256 != encoder_sha || decoded.teacher_sha256 != teacher_sha || decoded.profile_sha256 != profile_sha || decoded.latent_coordinate_mae != StringToDouble(loss)) return(Fail("study_encoder.save.receipt_readback:" + reason, reason)); Print("DPCC_STUDYENCODER_RECEIPT root=", m_root, " receipt_sha256=", receipt_sha, " encoder_sha256=", encoder_sha, " teacher_sha256=", teacher_sha); reason = ""; return(true); } //+------------------------------------------------------------------+ //| Forward one causal state through the frozen Market CNet | //+------------------------------------------------------------------+ bool ForwardInferenceState(const float &values[], string &reason) { m_z_available = false; if(ArraySize(values) != 45) return(Fail("study_decoder.inference.input", reason)); for(int i = 0; i < 45; i++) if(!MathIsValidNumber(values[i]) || !m_state.Update(i, values[i])) return(Fail("study_decoder.inference.input_buffer", reason)); if(!m_state.BufferWrite() || !m_market.feedForward(GetPointer(m_state), 1, false, (CBufferFloat*)NULL)) return(Fail("study_decoder.inference.market_forward", reason)); return(true); } //+------------------------------------------------------------------+ //| Bind historical Open and the preceding completed ATR(14) | //+------------------------------------------------------------------+ bool ForwardHistoricalState(const int decision, const float &values[], string &reason) { if(decision < 0 || decision >= ArraySize(m_bars)) return(Fail("study_forecast.historical_identity", reason)); if(m_price_joint) { SDPCCHistoricalPriceSnapshot snapshot; CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!decoder || !BuildHistoricalPriceSnapshot(decision, snapshot, reason) || !decoder.BindPriceAnchor(snapshot.anchor, snapshot.scale, snapshot.decision_id)) return(Fail("study_forecast.price_joint.causal_anchor:" + reason, reason)); m_price_observation = snapshot; } return(ForwardInferenceState(values, reason)); } //+------------------------------------------------------------------+ //| Build the four predecessors without admitting live rows offline | //+------------------------------------------------------------------+ bool LivePredecessors(const int live_position, SDPCCStudyForecastBar &prior[], string &reason) { const int pinned = ArraySize(m_bars); if(live_position < 0 || live_position > m_live_count || pinned < 5 || ArrayResize(prior, 4) != 4) return(Fail("study_decoder.live.predecessors", reason)); for(int i = 0; i < 4; i++) { const long index = (long)pinned + live_position - 4 + i; if(index < pinned) prior[i] = m_bars[(int)index]; else prior[i] = m_live_bars[(int)(index - pinned)]; } return(true); } //+------------------------------------------------------------------+ //| Replay causal Market origins from the existing holdout boundary | //+------------------------------------------------------------------+ bool PrepareLiveState(string &reason) { if(m_live_prepared) return(true); if(ArraySize(m_holdout) == 0 || !m_market.Clear()) return(Fail("study_decoder.live.replay_clear", reason)); m_inference_split = -1; m_inference_cursor = -1; for(int decision = m_holdout[0]; decision <= ArraySize(m_bars); decision++) { float state_values[]; if(!DPCCPackStudyForecastInput(m_bars, decision, state_values) || !ForwardHistoricalState(decision, state_values, reason)) return(Fail("study_decoder.live.pinned_replay:" + reason, reason)); } for(int live = 0; live < m_live_count; live++) { SDPCCStudyForecastBar prior[], context[]; if(!LivePredecessors(live, prior, reason) || ArrayResize(context, 5) != 5) return(Fail("study_decoder.live.accepted_replay_context:" + reason, reason)); for(int i = 0; i < 4; i++) context[i] = prior[i]; context[4] = m_live_bars[live]; float state_values[]; if(!DPCCPackStudyForecastInput(context, 5, state_values) || !ForwardInferenceState(state_values, reason)) return(Fail("study_decoder.live.accepted_replay:" + reason, reason)); } m_live_prepared = true; return(true); } bool Fail(const string code, string &reason) { m_failed = true; reason = code; if(m_m05 && m_price_study_phase == defForecastPriceOMPBCalibrate) { CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(forecast && !forecast.SetGradientNormalization(true)) reason += ";study_forecast.ompb.gradient_normalization_restore"; } return(false); } bool CreateGraphs(string &reason, const bool include_teacher = true) { const bool price_head = (m_price_forward_only || m_price_joint); const bool build_teacher = (include_teacher && !price_head); if(!DPCCCreateStudyForecastGraphs(m_market, m_target, reason, build_teacher, price_head)) return(false); COpenCLMy *context = m_market.GetOpenCL(); if(!context || !m_gradient_ops.Bind(context) || !m_state.BufferInit(45, 0) || !m_state.BufferCreate(context) || !m_future.BufferInit(108, 0) || !m_future.BufferCreate(context) || !m_future_latent.BufferInit(9 * 12 * 16, 0) || !m_future_latent.BufferCreate(context) || !m_zero.BufferInit(144, 0) || !m_zero.BufferCreate(context) || !m_negative.BufferInit(144, 0) || !m_negative.BufferCreate(context) || !m_delta.BufferInit(1728, 0) || !m_delta.BufferCreate(context)) return(Fail("study_forecast.buffers", reason)); return(true); } bool BuildFactualFutureLatentTarget(const int decision, CBufferFloat *&target, string &reason) { target = NULL; if(decision < 5 || decision + 12 > ArraySize(m_bars) || !MissingTeacherAllowed() || m_future_latent.Total() != 9 * 12 * 16 || m_future_latent.GetOpenCL() != m_market.GetOpenCL()) return(Fail("study_forecast.factual_target.contract", reason)); float future[]; if(ArrayResize(future, 9 * 12) != 9 * 12) return(Fail("study_forecast.factual_target.allocate", reason)); for(int h = 0; h < 12; h++) { float features[]; if(!DPCCStudyForecastFeature(m_bars[decision + h], features) || ArraySize(features) != 9) return(Fail("study_forecast.factual_target.future_feature", reason)); for(int v = 0; v < 9; v++) { const float value = features[v]; if(!MathIsValidNumber(value)) return(Fail("study_forecast.factual_target.nonfinite_feature", reason)); future[v * 12 + h] = value; } } //--- Normalize each future feature over the horizon before shape adaptation. for(int v = 0; v < 9; v++) { double mean = 0.0; for(int h = 0; h < 12; h++) mean += (double)future[v * 12 + h]; mean /= 12.0; double variance = 0.0; for(int h = 0; h < 12; h++) { const double delta = (double)future[v * 12 + h] - mean; variance += delta * delta; } variance /= 12.0; if(!MathIsValidNumber(mean) || !MathIsValidNumber(variance) || variance < 0.0) return(Fail("study_forecast.factual_target.normalization", reason)); const double stdev = MathSqrt(variance); if(!MathIsValidNumber(stdev)) return(Fail("study_forecast.factual_target.normalization", reason)); const double scale = (stdev > 0.0 ? stdev : 1.0); for(int h = 0; h < 12; h++) { const float value = (float)(((double)future[v * 12 + h] - mean) / scale); if(!MathIsValidNumber((double)value)) return(Fail("study_forecast.factual_target.normalization", reason)); future[v * 12 + h] = value; } } //--- Match the fixed 16-coordinate latent target using the remaining factual path. for(int v = 0; v < 9; v++) for(int h = 0; h < 12; h++) for(int d = 0; d < 16; d++) { const double position = (double)h + ((double)(11 - h) * (double)d) / 15.0; const int left = (int)MathFloor(position); const int right = (int)MathMin(11, left + 1); const double fraction = position - left; const double a = future[v * 12 + left]; const double b = future[v * 12 + right]; const float value = (float)(a + (b - a) * fraction); const int index = (v * 12 + h) * 16 + d; if(!MathIsValidNumber(value) || !m_future_latent.Update(index, value)) return(Fail("study_forecast.factual_target.pack", reason)); } if(!m_future_latent.BufferWrite()) return(Fail("study_forecast.factual_target.upload", reason)); target = GetPointer(m_future_latent); return(target != NULL); } //+------------------------------------------------------------------+ //| Bind retained first-tick quote and the last completed ATR | //+------------------------------------------------------------------+ bool BuildHistoricalPriceSnapshot(const int decision, SDPCCHistoricalPriceSnapshot &snapshot, string &reason) { ZeroMemory(snapshot); //--- The raw reader has verified each row's successor link. if((!m_price_forward_only && !m_price_joint) || !m_ready || m_failed || m_price_timeframe <= PERIOD_CURRENT || PeriodSeconds(m_price_timeframe) <= 0 || m_price_server == "" || (m_raw_identity_bound && (m_price_symbol != _Symbol || !DPCCResolverLowerSha256(m_raw_sha))) || decision < 1 || decision >= ArraySize(m_bars) || m_bars[decision].server_open <= 0 || m_bars[decision - 1].server_open <= 0 || m_bars[decision - 1].server_open >= m_bars[decision].server_open) return(Fail("study_forecast.price_snapshot.identity", reason)); const long target_time_msc = DPCCHistoricalPriceTargetTimeMsc(m_bars[decision]); if(target_time_msc <= 0) return(Fail("TARGET_TIME_UNAVAILABLE", reason)); if(!DPCCPriceTargetWindowAvailable(m_bars, decision, 1, m_price_quote_mode, m_price_h1_ohlc_targets, m_price_timeframe)) return(Fail("TARGET_TIME_UNAVAILABLE", reason)); const double anchor = DPCCHistoricalPriceTargetValue(m_bars[decision], 0); const double scale = (double)m_bars[decision - 1].indicators[2]; if(!MathIsValidNumber(anchor) || anchor <= 0) return(Fail("INVALID_PRICE_ANCHOR", reason)); if(!MathIsValidNumber(scale) || scale <= 0 || m_bars[decision - 1].indicators[2] == EMPTY_VALUE) return(Fail("INVALID_PRICE_SCALE", reason)); double tick_size = 0; if(!SymbolInfoDouble(m_price_symbol, SYMBOL_TRADE_TICK_SIZE, tick_size) || !MathIsValidNumber(tick_size) || tick_size <= 0) return(Fail("study_forecast.price_snapshot.tick_size", reason)); long chart_mode = -1; if(m_raw_identity_bound && m_price_quote_basis_resolved && (!SymbolInfoInteger(m_price_symbol, SYMBOL_CHART_MODE, chart_mode) || chart_mode != (long)m_price_quote_mode)) return(Fail("study_forecast.price_snapshot.quote_basis_changed", reason)); //--- Keep raw-source provenance only when the caller requested bound history. snapshot.decision_id = (ulong)target_time_msc; snapshot.source_server = m_price_server; snapshot.symbol = m_price_symbol; snapshot.raw_sha256 = (m_raw_identity_bound ? m_raw_sha : ""); snapshot.resolved_timeframe = m_price_timeframe; snapshot.quote_basis_resolved = m_price_quote_basis_resolved; snapshot.chart_quote_mode = m_price_quote_mode; snapshot.quote_basis = (m_price_quote_basis_resolved ? (m_price_quote_mode == SYMBOL_CHART_MODE_BID ? "BID" : "LAST") : "UNRESOLVED"); snapshot.quote_basis_source = (m_price_quote_basis_resolved ? (m_raw_identity_bound ? "CALLER_SYMBOL_CHART_MODE" : "RAW_DATASET") : "UNRESOLVED"); snapshot.quote_time_msc = target_time_msc; snapshot.quote_time_server = snapshot.quote_time_msc / 1000; snapshot.quote_time_resolution = (m_price_h1_ohlc_targets ? (m_price_timeframe == PERIOD_H1 ? "SERVER_H1_BAR_OPEN_MSC" : "SERVER_BAR_OPEN_MSC") : "SERVER_FIRST_AVAILABLE_TICK_MSC"); snapshot.target_time_resolved = true; snapshot.anchor_bar_index = decision; snapshot.anchor_bar_open_server = m_bars[decision].server_open; snapshot.completed_bar_index = decision - 1; snapshot.completed_bar_open_server = m_bars[decision - 1].server_open; snapshot.completed_bar_successor_server = m_bars[decision].server_open; snapshot.atr_period = 14; snapshot.anchor = anchor; snapshot.scale = scale; snapshot.tick_size = tick_size; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Forward one anchored generation while preserving recurrent state | //+------------------------------------------------------------------+ bool ForwardHistoricalPriceState(const int decision, const float &values[], SDPCCHistoricalPriceSnapshot &snapshot, string &reason) { if(!BuildHistoricalPriceSnapshot(decision, snapshot, reason)) return(false); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); //--- Clear the consumed price generation without clearing layers 0-5. if(!forecast || !decoder || !decoder.Clear() || !forecast.CancelTrainingFrame() || !forecast.SetPricePhase(defForecastPriceVerify) || !decoder.SetPricePhase(defForecastPriceVerify) || !decoder.BindPriceAnchor(snapshot.anchor, snapshot.scale, snapshot.decision_id)) return(Fail("study_forecast.price_context.release_or_bind", reason)); return(ForwardInferenceState(values, reason)); } bool BuildTargetResponsibilities(const int decision, CNeuronScenarioForecast *forecast, CNeuronBaseOCL *rank, CNeuronBaseOCL *target, string &reason) { if(decision < 5 || decision + 12 > ArraySize(m_bars) || !forecast || !rank || (CheckPointer(target) == POINTER_INVALID && !MissingTeacherAllowed()) || !TeacherFrozenPolicy()) return(Fail("study_forecast.factual_target.freeze_or_shape", reason)); CBufferFloat *future_target = NULL; if(CheckPointer(target) != POINTER_INVALID) { for(int h = 0; h < 12; h++) { float values[]; if(!DPCCStudyForecastFeature(m_bars[decision + h], values)) return(Fail("study_forecast.future_feature", reason)); for(int v = 0; v < 9; v++) if(!m_future.Update(v * 12 + h, values[v])) return(Fail("study_forecast.future_buffer", reason)); } if(!m_future.BufferWrite() || !m_target.feedForward(GetPointer(m_future), 1, false, (CBufferFloat*)NULL)) return(Fail("study_forecast.target_forward", reason)); future_target = target.getOutput(); } else if(!BuildFactualFutureLatentTarget(decision, future_target, reason)) return(false); if(!future_target || future_target.Total() != 9 * 12 * 16 || future_target.GetOpenCL() != m_market.GetOpenCL() || !m_gradient_ops.BuildTargetDelta(GetPointer(m_zero), rank.getOutput(), GetPointer(m_negative), future_target, GetPointer(m_delta), 16, 9) || !TeacherFrozenPolicy()) return(Fail("study_forecast.factual_target_delta_or_target_changed", reason)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(forecast); const bool prepared = (m_price_joint ? (price && price.PrepareLatentSupervision(future_target, GetPointer(m_delta))) : forecast.BuildResponsibilities(future_target, GetPointer(m_delta))); if(!prepared || !forecast.LastTargetAccepted() || !forecast.LastBatchControlValid() || !TeacherFrozenPolicy()) return(Fail("study_forecast.native_loss_or_target_changed", reason)); return(true); } bool BuildSample(const int decision, CNeuronScenarioForecast *&forecast, CNeuronBaseOCL *&rank, CNeuronBaseOCL *&target, string &reason) { float state_values[]; if(!DPCCPackStudyForecastInput(m_bars, decision, state_values) || ArraySize(state_values) != 45) return(Fail("study_forecast.history_feature", reason)); if(!ForwardHistoricalState(decision, state_values, reason)) return(false); rank = m_market.Layer(4); target = m_target.Layer(-1); if(!forecast || !rank || (CheckPointer(target) == POINTER_INVALID && !MissingTeacherAllowed())) return(Fail("study_forecast.native_loss.owner", reason)); return(BuildTargetResponsibilities(decision, forecast, rank, target, reason)); } bool AdvanceFit(double &rho_sum, uint &rho_nonzero, uint &generator_step, uint &router_step, uint &confidence_step, string &reason) { const int decision = m_fit[m_cursor]; if(!m_epoch_started) { if(!m_market.Clear() || !ClearTeacherState()) return(Fail("study_forecast.epoch_initial_clear", reason)); m_epoch_started = true; } CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); if(!ompb || !forecast || !rank || !target || !m_market.TrainMode(true) || !m_market.SetWeightsUpdate(true) || !ompb.SetMode(OMPB_BYPASS) || !ompb.SetSourceAnchor(false) || !forecast.SetCodebookUpdate(true)) return(Fail("study_forecast.train_policy", reason)); ompb.TrainMode(false); if(!BuildSample(decision, forecast, rank, target, reason)) return(false); if(!forecast.CommitBatchDiagnostics()) return(Fail("study_forecast.fit_diagnostics", reason)); CBufferFloat *rho = forecast.GetResponsibilities(); if(!rho || rho.Total() != 21 || !rho.BufferRead()) return(Fail("study_forecast.rho", reason)); rho_sum = 0; rho_nonzero = 0; for(int k = 0; k < 21; k++) { const float value = (*rho)[k]; if(!MathIsValidNumber((double)value) || value < 0 || value > 1) return(Fail("study_forecast.rho_numeric", reason)); rho_sum += (double)value; if(value > 0) rho_nonzero++; } CNeuronConvOCL *heads[3]; heads[0] = forecast.GetGenerator(); heads[1] = forecast.GetRouter(); heads[2] = forecast.GetConfidenceHead(); uint steps_before[3], steps_after[3]; for(int h = 0; h < 3; h++) { if(!heads[h] || !heads[h].GetWeightsConv()) return(Fail("study_forecast.learned_head", reason)); steps_before[h] = heads[h].NativeOptimizerStep(); } if(rho_nonzero == 0 || MathAbs(rho_sum - 1.0) > 1e-5 || !m_market.backPropGradient((CBufferFloat*)NULL, ( CBufferFloat*)NULL, -1, true)) return(Fail("study_forecast.backward", reason)); bool learned = false; for(int h = 0; h < 3; h++) { steps_after[h] = heads[h].NativeOptimizerStep(); if(steps_before[h] < UINT_MAX && steps_after[h] == steps_before[h] + 1) learned = true; } if(!learned) return(Fail("study_forecast.learned_head_no_update", reason)); generator_step = forecast.GetGenerator().NativeOptimizerStep(); router_step = forecast.GetRouter().NativeOptimizerStep(); confidence_step = forecast.GetConfidenceHead().NativeOptimizerStep(); m_updates++; m_cursor++; return(true); } //+------------------------------------------------------------------+ //| Normalize factual prices only after the causal Market forward | //+------------------------------------------------------------------+ bool PreparePriceJointTarget(const int decision, CNeuronScenarioDecoderOCL *decoder, string &reason) { double anchor = 0, scale = 0; ulong decision_id = 0; if(!decoder || decision < 5 || decision + 12 > ArraySize(m_bars) || !decoder.GetPriceAnchor(anchor, scale, decision_id) || !DPCCPriceTargetWindowAvailable(m_bars, decision, 12, m_price_quote_mode, m_price_h1_ohlc_targets, m_price_timeframe) || decision_id != (ulong)DPCCHistoricalPriceTargetTimeMsc(m_bars[decision]) || anchor != DPCCHistoricalPriceTargetValue(m_bars[decision], 0) || scale != (double)m_bars[decision - 1].indicators[2] || !MathIsValidNumber(scale) || scale <= 0 || m_price_target.Total() != 12 * 4) return(Fail("study_forecast.price_joint.target_frame", reason)); //--- One physical scenario conversion validates this forward's fp32/tick compatibility. if(m_price_observation.decision_id != decision_id || m_price_observation.anchor != anchor || m_price_observation.scale != scale || !decoder.BuildNativeScenarios(GetPointer(m_price_native), 12, m_price_observation.tick_size)) return(Fail("study_forecast.price_joint.native_precision", reason)); for(int h = 0; h < 12; h++) { const int row = decision + h; if(m_bars[row].server_open <= m_bars[row - 1].server_open || DPCCHistoricalPriceTargetValue(m_bars[row], 1) < MathMax(DPCCHistoricalPriceTargetValue(m_bars[row], 0), DPCCHistoricalPriceTargetValue(m_bars[row], 3)) || DPCCHistoricalPriceTargetValue(m_bars[row], 2) > MathMin(DPCCHistoricalPriceTargetValue(m_bars[row], 0), DPCCHistoricalPriceTargetValue(m_bars[row], 3))) return(Fail("study_forecast.price_joint.target_geometry", reason)); for(int c = 0; c < 4; c++) { const double price = DPCCHistoricalPriceTargetValue(m_bars[row], c); const double value = (price - anchor) / scale; const float normalized = (float)value; if(!MathIsValidNumber(price) || !MathIsValidNumber(value) || !MathIsValidNumber(normalized) || (h == 0 && c == 0 && normalized != 0) || !m_price_target.Update(h * 4 + c, normalized)) return(Fail("study_forecast.price_joint.target_numeric", reason)); } } if(!m_price_target.BufferWrite()) return(Fail("study_forecast.price_joint.target_upload", reason)); return(true); } //+------------------------------------------------------------------+ //| Freeze one completed joint phase and check its immutable owners | //+------------------------------------------------------------------+ bool FreezePriceJoint(string &reason) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronBaseOCL *ompb_base = ompb; if(!price || !decoder || !ompb || !decoder.Clear() || !price.CancelTrainingFrame() || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !price.SetCodebookUpdate(false) || !price.SetPricePhase(defForecastPriceVerify) || !decoder.SetPricePhase(defForecastPriceVerify) || !FrozenProducer(reason) || ompb_base.TrainMode() || ompb.Mode() != OMPB_BYPASS || ompb.SourceAnchor()) return(Fail("study_forecast.price_joint.freeze", reason)); ulong teacher = 0, ompb_after = 0; if(!TeacherFingerprint(teacher) || teacher != m_teacher_initial_fingerprint || !ompb.AppendParameterFingerprint(ompb_after) || ompb_after != m_price_joint_ompb_fingerprint) return(Fail("study_forecast.price_joint.frozen_boundary", reason)); return(true); } //+------------------------------------------------------------------+ //| Commit native latent and detached price signals in one CNet pass | //+------------------------------------------------------------------+ bool AdvancePriceJointFit(double &loss, double &rho_sum, uint &rho_nonzero, uint &generator_step, uint &router_step, uint &confidence_step, string &reason) { if(!m_epoch_started) { if(!m_market.Clear() || !ClearTeacherState()) return(Fail("study_forecast.price_joint.epoch_clear", reason)); m_epoch_started = true; } CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronBaseOCL *ompb_base = ompb; if(!ompb || !price || !decoder || !decoder.Clear() || !price.CancelTrainingFrame() || !m_market.TrainMode(true) || !m_market.SetWeightsUpdate(true) || !SetTeacherFrozenPolicy() || !ompb.SetMode(OMPB_BYPASS) || !ompb.SetSourceAnchor(false) || !price.SetPricePhase(m_price_study_phase) || !decoder.SetPricePhase(m_price_study_phase) || !price.SetCodebookUpdate(true)) return(Fail("study_forecast.price_joint.train_policy", reason)); ompb.TrainMode(false); if(ompb_base.TrainMode() || !TeacherFrozenPolicy()) return(Fail("study_forecast.price_joint.frozen_owner_policy", reason)); //--- Check optimizer counters without reading full parameter tensors. CNeuronConvOCL *heads[3]; heads[0] = price.GetGenerator(); heads[1] = price.GetRouter(); heads[2] = price.GetConfidenceHead(); uint steps_before[3]; for(int h = 0; h < 3; h++) { if(!heads[h] || heads[h].NativeOptimizerStep() == UINT_MAX) return(Fail("study_forecast.price_joint.head_counter", reason)); steps_before[h] = heads[h].NativeOptimizerStep(); } const uint decoder_before = decoder.UpdateCount(); if(decoder_before == UINT_MAX) return(Fail("study_forecast.price_joint.decoder_counter", reason)); //--- Future labels enter only after the matching anchored Market generation. CNeuronScenarioForecast *forecast = price; CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); const int decision = (m_m05 && m_price_study_phase == defForecastPriceLatentPrepare ? m_latent_fit[m_cursor] : m_fit[m_cursor]); if(!BuildSample(decision, forecast, rank, target, reason) || !price.PriceSupervisionPrepared()) return(Fail("study_forecast.price_fit.latent_supervision:" + reason, reason)); if(m_price_study_phase == defForecastPriceLatentPrepare) { //--- Layer7 records and reverses its tape, then its phase gate completes without a weight write. if(!decoder.getGradient() || !decoder.getGradient().Fill(0.0f)) return(Fail("study_forecast.latent_prepare.zero_price_gradient", reason)); } else if(m_price_study_phase == defForecastPriceJoint) { if(!PreparePriceJointTarget(decision, decoder, reason) || !decoder.PreparePriceLoss(price, GetPointer(m_price_target), m_price_lambda, m_price_huber, m_price_rho_tolerance, true, loss) || !MathIsValidNumber(loss) || loss < 0) return(Fail("study_forecast.price_joint.prepare_loss:" + reason, reason)); } else return(Fail("study_forecast.price_fit.unsupported_phase", reason)); if(!price.CommitBatchDiagnostics()) return(Fail("study_forecast.price_fit.batch_diagnostics", reason)); CBufferFloat *rho = price.GetResponsibilities(); if(!rho || rho.Total() != 21 || !rho.BufferRead()) return(Fail("study_forecast.price_joint.rho", reason)); for(int k = 0; k < 21; k++) { const float value = (*rho)[k]; if(!MathIsValidNumber(value) || value < 0 || value > 1) return(Fail("study_forecast.price_joint.rho_numeric", reason)); rho_sum += (double)value; if(value > 0) rho_nonzero++; } if(rho_nonzero == 0 || MathAbs(rho_sum - 1.0) > (double)m_price_rho_tolerance) return(Fail("study_forecast.price_joint.rho_mass", reason)); //--- Decoder writes dZ; ForecastPrice adds native signals before one Adam/EMA update. if(!m_market.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 7, false)) return(Fail("study_forecast.price_joint.partial_transaction", reason)); for(int h = 0; h < 3; h++) if(heads[h].NativeOptimizerStep() != steps_before[h] + 1) return(Fail("study_forecast.price_joint.head_single_update", reason)); const uint expected_decoder = (m_price_study_phase == defForecastPriceJoint ? decoder_before + 1 : decoder_before); if(decoder.UpdateCount() != expected_decoder || decoder.GenerationPhase() != 0 || price.PriceSupervisionPrepared() || !TeacherFrozenPolicy() || ompb_base.TrainMode() || ompb.Mode() != OMPB_BYPASS || ompb.SourceAnchor() || !decoder.Clear() || !price.CancelTrainingFrame()) return(Fail("study_forecast.price_joint.release_or_freeze", reason)); generator_step = heads[0].NativeOptimizerStep(); router_step = heads[1].NativeOptimizerStep(); confidence_step = heads[2].NativeOptimizerStep(); if(m_price_study_phase == defForecastPriceJoint) { m_price_loss_sum += loss; m_price_joint_updates++; } m_updates++; m_cursor++; return(true); } int ResolveM05TimeBoundary(const datetime boundary) const { for(int i = 0; i < ArraySize(m_bars); i++) if(m_bars[i].server_open >= (long)boundary) return(i); return(ArraySize(m_bars)); } //+------------------------------------------------------------------+ //| Bind donor dates to the captured raw identity | //+------------------------------------------------------------------+ bool ResolveM05OMPBIdentity(string &reason) { reason = ""; if(!m_ompb_configured || m_raw_sha == "" || !DPCCArchitectureSHA256Encoding(m_raw_sha) || ArraySize(m_bars) < 18) return(Fail("study_forecast.ompb.raw_identity", reason)); m_ompb_profile.raw_sha256 = m_raw_sha; m_ompb_profile.reference_first_bar = ResolveM05TimeBoundary(m_ompb_profile.reference_start); m_ompb_profile.reference_split_bar = ResolveM05TimeBoundary(m_ompb_profile.reference_split); m_ompb_profile.reference_end_bar = ResolveM05TimeBoundary(m_ompb_profile.reference_end); m_ompb_profile.calibration_first_bar = ResolveM05TimeBoundary(m_ompb_profile.calibration_start); m_ompb_profile.calibration_split_bar = ResolveM05TimeBoundary(m_ompb_profile.calibration_split); m_ompb_profile.calibration_end_bar = ResolveM05TimeBoundary(m_ompb_profile.calibration_end); if(!DPCCResolverLowerSha256(m_ompb_profile.profile_sha256) || !DPCCResolverLowerSha256(m_ompb_profile.raw_sha256) || m_ompb_profile.reference_first_bar < 5 || m_ompb_profile.reference_split_bar <= m_ompb_profile.reference_first_bar || m_ompb_profile.reference_end_bar <= m_ompb_profile.reference_split_bar || m_ompb_profile.calibration_first_bar < m_ompb_profile.reference_end_bar || m_ompb_profile.calibration_split_bar <= m_ompb_profile.calibration_first_bar || m_ompb_profile.calibration_end_bar <= m_ompb_profile.calibration_split_bar || m_ompb_profile.calibration_end_bar > ArraySize(m_bars)) return(Fail("study_forecast.ompb.time_window_binding", reason)); return(true); } //+------------------------------------------------------------------+ //| Resolve calibration windows using the donor's 80/20 partitions | //+------------------------------------------------------------------+ bool ResolveOMPBFitRows(string &reason) { if(!m_ompb_configured || m_ompb_profile.raw_sha256 != m_raw_sha || m_ompb_profile.calibration_end_bar > ArraySize(m_bars)) return(Fail("study_forecast.ompb.profile_raw_or_window", reason)); ArrayResize(m_ompb_reference, 0); ArrayResize(m_ompb_calibration, 0); ArrayResize(m_ompb_source_evaluation, 0); ArrayResize(m_ompb_target_evaluation, 0); for(int row = 5; row + 12 < ArraySize(m_bars); row++) { if(!DPCCPriceTargetWindowAvailable(m_bars, row, 12, m_price_quote_mode, m_price_h1_ohlc_targets, m_price_timeframe)) continue; const int future = row + 12; if(row >= m_ompb_profile.reference_first_bar && future < m_ompb_profile.reference_split_bar) { const int count = ArraySize(m_ompb_reference); if(ArrayResize(m_ompb_reference, count + 1, 256) != count + 1) return(Fail("study_forecast.ompb.reference_rows", reason)); m_ompb_reference[count] = row; } if(row >= m_ompb_profile.calibration_first_bar && future < m_ompb_profile.calibration_split_bar) { const int count = ArraySize(m_ompb_calibration); if(ArrayResize(m_ompb_calibration, count + 1, 256) != count + 1) return(Fail("study_forecast.ompb.calibration_rows", reason)); m_ompb_calibration[count] = row; } if(row >= m_ompb_profile.reference_split_bar && future < m_ompb_profile.reference_end_bar) { const int count = ArraySize(m_ompb_source_evaluation); if(ArrayResize(m_ompb_source_evaluation, count + 1, 256) != count + 1) return(Fail("study_forecast.ompb.source_evaluation_rows", reason)); m_ompb_source_evaluation[count] = row; } if(row >= m_ompb_profile.calibration_split_bar && future < m_ompb_profile.calibration_end_bar) { const int count = ArraySize(m_ompb_target_evaluation); if(ArrayResize(m_ompb_target_evaluation, count + 1, 256) != count + 1) return(Fail("study_forecast.ompb.target_evaluation_rows", reason)); m_ompb_target_evaluation[count] = row; } } CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CLayerDescription *description = (ompb ? ompb.GetLayerInfo() : NULL); if(!description) return(Fail("study_forecast.ompb.layer_description", reason)); const bool shape = (ArraySize(description.units) == 2 && description.units[0] > 0 && description.units[1] > 0); if(shape) { m_ompb_reference_capacity = description.units[0]; m_ompb_current_capacity = description.units[1]; } delete description; if(!shape || (ulong)ArraySize(m_ompb_reference) < m_ompb_reference_capacity || (ulong)ArraySize(m_ompb_calibration) < m_ompb_current_capacity || ArraySize(m_ompb_source_evaluation) == 0 || ArraySize(m_ompb_target_evaluation) == 0) return(Fail("study_forecast.ompb.insufficient_full_histories", reason)); return(true); } //+------------------------------------------------------------------+ //| Freeze all network writes and permit only the OMPB optimizer | //+------------------------------------------------------------------+ bool SetOMPBStudyPolicy(const ENUM_OMPB_MODE mode, string &reason) { CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !price || !decoder || !decoder.Clear() || !price.CancelTrainingFrame() || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !SetTeacherFrozenPolicy() || !price.SetCodebookUpdate(false) || !price.SetGradientNormalization(mode != OMPB_CALIBRATE) || !price.SetPricePhase(defForecastPriceOMPBCalibrate) || !decoder.SetPricePhase(defForecastPriceOMPBCalibrate) || !ompb.SetSourceAnchor(false) || !ompb.SetMode(mode)) return(Fail("study_forecast.ompb.freeze_policy", reason)); ompb.TrainMode(mode == OMPB_CALIBRATE); return(true); } //+------------------------------------------------------------------+ //| Read immutable parameter owners only at calibration boundaries | //+------------------------------------------------------------------+ bool OMPBFrozenFingerprint(ulong &fingerprint, string &reason) { fingerprint = ulong(1469598103934665603); if(m_market.WeightsUpdateEnabled() || !TeacherFrozenPolicy()) return(Fail("study_forecast.ompb.network_write_enabled", reason)); for(int i = 0; i < 8; i++) { if(i == 5) continue; CNeuronBaseOCL *layer = m_market.Layer(i); if(!layer || layer.TrainMode() || !layer.AppendParameterFingerprint(fingerprint)) return(Fail("study_forecast.ompb.frozen_layer", reason)); } ulong target_fingerprint = 0; if(!TeacherFingerprint(target_fingerprint) || target_fingerprint != m_teacher_initial_fingerprint) return(Fail("study_forecast.ompb.target_changed", reason)); return(true); } //+------------------------------------------------------------------+ //| Enter Stage02 on the retained post-PRICE_JOINT Market graph | //+------------------------------------------------------------------+ bool BeginOMPBCalibrate(string &reason) { if(!m_m05 || !m_full_run || !ResolveOMPBFitRows(reason) || !SetOMPBStudyPolicy(OMPB_REFERENCE, reason)) return(Fail("study_forecast.ompb.begin:" + reason, reason)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !forecast || !decoder || !m_market.Clear() || !ClearTeacherState() || !ompb.ResetReference() || !ompb.SetTau(m_ompb_profile.tau) || !ompb.SetDisagreementMult(m_ompb_profile.lambda_dis) || !ompb.SetKLDMult(m_ompb_profile.lambda_kl) || !ompb.SetAlphaMult(m_ompb_profile.lambda_alpha) || !ompb.SetAlphaPrior(m_ompb_profile.alpha_prior) || !ompb.SetMaxKL(m_ompb_profile.max_kl) || !OMPBFrozenFingerprint(m_ompb_frozen_fingerprint, reason) || !ompb.AppendParameterFingerprint(m_ompb_initial_fingerprint)) return(Fail("study_forecast.ompb.initialize:" + reason, reason)); CNeuronConvOCL *heads[3]; heads[0] = forecast.GetGenerator(); heads[1] = forecast.GetRouter(); heads[2] = forecast.GetConfidenceHead(); for(int h = 0; h < 3; h++) { if(!heads[h] || heads[h].NativeOptimizerStep() == UINT_MAX) return(Fail("study_forecast.ompb.forecast_counter", reason)); m_ompb_forecast_steps[h] = heads[h].NativeOptimizerStep(); } m_ompb_decoder_updates = decoder.UpdateCount(); m_price_study_phase = defForecastPriceOMPBCalibrate; m_cursor = 0; m_ompb_reference_cursor = 0; m_ompb_epoch = 0; m_ompb_epoch_source_updates = 0; m_ompb_reference_complete = false; m_ompb_epoch_started = false; m_ompb_source_pending = false; m_ompb_calibration_completed = false; m_ompb_stage02_quality_pass = false; m_ompb_evaluation_stage = 0; m_ompb_evaluation_cursor = 0; m_ompb_evaluation_posterior = 0; ArrayInitialize(m_ompb_evaluation_loss, 0); ArrayInitialize(m_ompb_evaluation_valid, 0); ArrayInitialize(m_ompb_evaluation_invalid, 0); return(true); } //+------------------------------------------------------------------+ //| Commit native Forecast supervision to only posterior and alpha | //+------------------------------------------------------------------+ bool RunOMPBCalibrationRow(const int decision, const bool source_anchor, string &reason) { if(!SetOMPBStudyPolicy(OMPB_CALIBRATE, reason)) return(false); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(forecast); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); if(!ompb || !price || !decoder || !rank || !ompb.SetSourceAnchor(source_anchor)) return(Fail("study_forecast.ompb.batch_owner", reason)); const uint current_before = ompb.CurrentCount(); const uint reference_before = ompb.ReferenceCount(); if(!source_anchor && !ompb.BeginCurrentTransaction()) return(Fail("study_forecast.ompb.current_begin", reason)); //--- The frozen decoder is a forward output only; native Stage02 starts backward at Forecast. const bool prepared = BuildSample(decision, forecast, rank, target, reason); const bool backward = (prepared && ompb.ForwardAccepted() && price.PriceSupervisionPrepared() && price.CommitBatchDiagnostics() && decoder.Clear() && m_market.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 6, false)); if(!backward) { const bool cleaned = (source_anchor ? ompb.SetSourceAnchor(false) : ompb.RollbackCurrentTransaction()); if(!cleaned) return(Fail("study_forecast.ompb.batch_rollback:" + reason, reason)); return(Fail("study_forecast.ompb.native_batch:" + reason, reason)); } //--- Source anchors retain their flag until all native gradients exist, as in Stage02RunBatch. if(source_anchor && !ompb.SetSourceAnchor(false)) return(Fail("study_forecast.ompb.anchor_release", reason)); if(!ompb.ForwardAccepted() || !ompb.UpdateInputWeights(rank)) { if(!source_anchor && !ompb.RollbackCurrentTransaction()) return(Fail("study_forecast.ompb.update_rollback", reason)); return(Fail("study_forecast.ompb.partial_optimizer_transaction", reason)); } if(!source_anchor && !ompb.CommitCurrentTransaction()) { if(!ompb.RollbackCurrentTransaction()) return(Fail("study_forecast.ompb.commit_rollback", reason)); return(Fail("study_forecast.ompb.current_commit", reason)); } if(ompb.ReferenceCount() != reference_before || (source_anchor && ompb.CurrentCount() != current_before) || price.PriceSupervisionPrepared() || decoder.GenerationPhase() != 0 || !price.CancelTrainingFrame()) return(Fail("study_forecast.ompb.batch_release", reason)); m_updates++; if(source_anchor) m_ompb_source_updates++; else m_ompb_target_updates++; return(true); } //+------------------------------------------------------------------+ //| Finish calibration with frozen owners and an inference posterior | //+------------------------------------------------------------------+ bool FinishOMPBCalibrate(string &reason) { if(!SetOMPBStudyPolicy(OMPB_INFERENCE, reason)) return(false); ulong frozen = 0, posterior = 0; CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !forecast || !decoder || !OMPBFrozenFingerprint(frozen, reason) || frozen != m_ompb_frozen_fingerprint || !ompb.AppendParameterFingerprint(posterior) || posterior == m_ompb_initial_fingerprint || m_ompb_target_updates == 0 || m_ompb_epoch != m_ompb_profile.epochs || decoder.UpdateCount() != m_ompb_decoder_updates || ompb.ReferenceCount() != m_ompb_reference_capacity || ompb.CurrentCount() != m_ompb_current_capacity) return(Fail("study_forecast.ompb.frozen_boundary:" + reason, reason)); CNeuronConvOCL *heads[3]; heads[0] = forecast.GetGenerator(); heads[1] = forecast.GetRouter(); heads[2] = forecast.GetConfidenceHead(); for(int h = 0; h < 3; h++) if(!heads[h] || heads[h].NativeOptimizerStep() != m_ompb_forecast_steps[h]) return(Fail("study_forecast.ompb.forecast_optimizer_changed", reason)); m_ompb_calibration_completed = true; return(true); } //+------------------------------------------------------------------+ //| Advance one held-out Stage02 row and retain donor loss metrics | //+------------------------------------------------------------------+ bool AdvanceOMPBEvaluation(string &reason) { const int lane = m_ompb_evaluation_stage; if(lane < 0 || lane >= 6 || (lane >= 4 && !m_ompb_calibration_completed)) return(Fail("study_forecast.ompb.evaluation_stage", reason)); const bool source = (lane % 2 == 0); const int count = (source ? ArraySize(m_ompb_source_evaluation) : ArraySize(m_ompb_target_evaluation)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(forecast); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !price || !decoder || count <= 0 || m_ompb_evaluation_cursor < 0 || m_ompb_evaluation_cursor >= count) return(Fail("study_forecast.ompb.evaluation_owner_or_cursor", reason)); //--- Rewind chronology without discarding the retained OMPB histories. if(m_ompb_evaluation_cursor == 0) { if(!SetOMPBStudyPolicy(lane < 2 ? OMPB_BYPASS : OMPB_INFERENCE, reason)) return(false); for(int i = 0; i < 5; i++) if(!m_market.Layer(i) || !m_market.Layer(i).Clear()) return(Fail("study_forecast.ompb.evaluation_trunk_clear", reason)); if(!price.Clear() || !decoder.Clear() || !ClearTeacherState() || !price.ResetEpochDiagnostics()) return(Fail("study_forecast.ompb.evaluation_head_clear", reason)); m_ompb_evaluation_posterior = 0; if(!ompb.AppendParameterFingerprint(m_ompb_evaluation_posterior) || (lane < 4 && m_ompb_evaluation_posterior != m_ompb_initial_fingerprint)) return(Fail("study_forecast.ompb.pre_evaluation_posterior_changed", reason)); } const int decision = (source ? m_ompb_source_evaluation[m_ompb_evaluation_cursor] : m_ompb_target_evaluation[m_ompb_evaluation_cursor]); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); if(!BuildSample(decision, forecast, rank, target, reason) || (lane >= 2 && !ompb.ForwardAccepted()) || !price.PriceSupervisionPrepared() || !price.CommitBatchDiagnostics()) { m_ompb_evaluation_invalid[lane]++; return(Fail("study_forecast.ompb.evaluation_row:" + reason, reason)); } m_ompb_evaluation_valid[lane]++; m_ompb_evaluation_cursor++; if(!decoder.Clear() || !price.CancelTrainingFrame()) return(Fail("study_forecast.ompb.evaluation_release", reason)); if(m_ompb_evaluation_cursor < count) return(true); //--- Match ACSRMStage02Baseline / InferenceEvaluation: forecast_loss = lmix / valid. double mix = 0, router = 0, trajectory = 0, confidence = 0, latent = 0, observation = 0; double valid = 0, invalid = 0, entropy = 0, distance = 0, inactive = 0, recovered = 0; ulong frozen = 0, posterior = 0; if(!price.BuildEpochCodebookDiagnostics() || !price.ReadEpochDiagnostics(mix, router, trajectory, confidence, latent, observation, valid, invalid, entropy, distance, inactive, recovered)) return(Fail("study_forecast.ompb.evaluation_metrics_read", reason)); const double attempts = m_ompb_evaluation_valid[lane]; m_ompb_evaluation_valid[lane] = valid; m_ompb_evaluation_invalid[lane] += invalid; if(!MathIsValidNumber(mix) || !MathIsValidNumber(valid) || !MathIsValidNumber(invalid) || valid <= 0 || valid < attempts || invalid < 0) return(Fail("study_forecast.ompb.evaluation_metrics_domain", reason)); m_ompb_evaluation_loss[lane] = mix / valid; if(!MathIsValidNumber(m_ompb_evaluation_loss[lane]) || !OMPBFrozenFingerprint(frozen, reason) || frozen != m_ompb_frozen_fingerprint || !ompb.AppendParameterFingerprint(posterior) || posterior != m_ompb_evaluation_posterior || decoder.UpdateCount() != m_ompb_decoder_updates) return(Fail("study_forecast.ompb.evaluation_frozen_or_loss:" + reason, reason)); PrintFormat("DPCC_M05_STAGE02_EVALUATION lane=%d loss=%.9f valid=%.0f invalid=%.0f", lane, m_ompb_evaluation_loss[lane], valid, m_ompb_evaluation_invalid[lane]); m_ompb_evaluation_cursor = 0; m_ompb_evaluation_stage++; if(m_ompb_evaluation_stage < 6) return(true); //--- Donor acceptance compares frozen BYPASS baselines with calibrated INFERENCE. const double target_improvement = (m_ompb_evaluation_loss[1] - m_ompb_evaluation_loss[5]) / MathMax(MathAbs(m_ompb_evaluation_loss[1]), 1.0e-12); const double source_degradation = (m_ompb_evaluation_loss[4] - m_ompb_evaluation_loss[0]) / MathMax(MathAbs(m_ompb_evaluation_loss[0]), 1.0e-12); m_ompb_stage02_quality_pass = (MathIsValidNumber(target_improvement) && MathIsValidNumber(source_degradation) && m_ompb_evaluation_valid[4] > 0 && m_ompb_evaluation_valid[5] > 0 && target_improvement >= 0.01 && source_degradation <= 0.01); PrintFormat( "DPCC_M05_STAGE02_QUALITY status=%s target_improvement=%.8f source_degradation=%.8f continuation=PERMITTED", (m_ompb_stage02_quality_pass ? "PASS" : "FAIL"), target_improvement, source_degradation); if(!FinishOMPBCalibrate(reason)) return(false); return(BeginPriceAdapt(reason)); } //+------------------------------------------------------------------+ //| Advance one Reference, evaluation, target, or source-anchor row | //+------------------------------------------------------------------+ bool AdvanceOMPBCalibrate(string &reason) { CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !price || !decoder || !m_ompb_configured) return(Fail("study_forecast.ompb.advance_owner", reason)); if(!m_ompb_reference_complete) { if(m_ompb_reference_cursor >= ArraySize(m_ompb_reference) || !SetOMPBStudyPolicy(OMPB_REFERENCE, reason)) return(Fail("study_forecast.ompb.reference_limit:" + reason, reason)); const int decision = m_ompb_reference[m_ompb_reference_cursor]; float values[]; if(!DPCCPackStudyForecastInput(m_bars, decision, values) || !ForwardHistoricalState(decision, values, reason) || !decoder.Clear() || !price.CancelTrainingFrame()) return(Fail("study_forecast.ompb.reference_forward:" + reason, reason)); m_ompb_reference_cursor++; if(ompb.ReferenceCount() == m_ompb_reference_capacity) m_ompb_reference_complete = true; return(true); } if(m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed) return(AdvanceOMPBEvaluation(reason)); if(!m_ompb_epoch_started) { if(!m_market.Clear() || !ClearTeacherState() || !price.ResetEpochDiagnostics()) return(Fail("study_forecast.ompb.epoch_clear", reason)); m_cursor = 0; m_ompb_epoch_source_updates = 0; m_ompb_epoch_started = true; } if(m_ompb_source_pending) { const int source = m_ompb_reference[m_ompb_epoch_source_updates % (uint)ArraySize(m_ompb_reference)]; if(!RunOMPBCalibrationRow(source, true, reason)) return(false); m_ompb_epoch_source_updates++; m_ompb_source_pending = false; } else { if(m_cursor >= ArraySize(m_ompb_calibration) || !RunOMPBCalibrationRow(m_ompb_calibration[m_cursor], false, reason)) return(Fail("study_forecast.ompb.target_row:" + reason, reason)); m_cursor++; m_ompb_source_pending = (m_ompb_profile.source_anchor_period > 0 && (uint)m_cursor % m_ompb_profile.source_anchor_period == 0); } if(m_cursor == ArraySize(m_ompb_calibration) && !m_ompb_source_pending) { m_ompb_epoch++; m_ompb_epoch_started = false; if(m_ompb_epoch == m_ompb_profile.epochs) return(FinishOMPBCalibrate(reason)); } return(true); } //+------------------------------------------------------------------+ //| Keep the calibrated posterior and every upstream parameter fixed | //+------------------------------------------------------------------+ bool SetPriceAdaptPolicy(const bool train_decoder, string &reason) { CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); const uint phase = (train_decoder ? defForecastPriceAdapt : defForecastPriceVerify); if(!ompb || !price || !decoder || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(train_decoder) || !SetTeacherFrozenPolicy() || !price.SetCodebookUpdate(false) || !price.SetPricePhase(phase) || !decoder.SetPricePhase(phase) || !ompb.SetSourceAnchor(false) || !ompb.SetMode(OMPB_INFERENCE)) return(Fail("study_forecast.price_adapt.freeze_policy", reason)); //--- Only Layer7 records a training tape and owns a permitted optimizer write. decoder.TrainMode(train_decoder); for(int i = 0; i < 7; i++) if(!m_market.Layer(i) || m_market.Layer(i).TrainMode()) return(Fail("study_forecast.price_adapt.upstream_train_mode", reason)); if(decoder.TrainMode() != train_decoder || !TeacherFrozenPolicy() || price.CodebookUpdate()) return(Fail("study_forecast.price_adapt.owner_policy", reason)); return(true); } //+------------------------------------------------------------------+ //| Fingerprint only frozen parameters at adaptation boundaries | //+------------------------------------------------------------------+ bool PriceAdaptFrozenFingerprint(ulong &fingerprint, string &reason) { fingerprint = ulong(1469598103934665603); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!ompb || !forecast || ompb.Mode() != OMPB_INFERENCE || ompb.SourceAnchor() || forecast.CodebookUpdate() || !TeacherFrozenPolicy()) return(Fail("study_forecast.price_adapt.frozen_owner", reason)); for(int i = 0; i < 7; i++) { CNeuronBaseOCL *layer = m_market.Layer(i); if(!layer || layer.TrainMode() || !layer.AppendParameterFingerprint(fingerprint)) return(Fail("study_forecast.price_adapt.frozen_parameter", reason)); } ulong target_fingerprint = 0; if(!TeacherFingerprint(target_fingerprint) || target_fingerprint != m_teacher_initial_fingerprint) return(Fail("study_forecast.price_adapt.target_changed", reason)); return(true); } //+------------------------------------------------------------------+ //| Enter decoder adaptation on FIT with the final OMPB posterior | //+------------------------------------------------------------------+ bool BeginPriceAdapt(string &reason, const bool operator_selected = false) { if(!m_m05 || !m_full_run || !m_ompb_configured || (!operator_selected && (!m_ompb_calibration_completed || m_ompb_target_updates == 0)) || !SetPriceAdaptPolicy(true, reason) || !PriceAdaptFrozenFingerprint(m_price_adapt_frozen_fingerprint, reason)) return(Fail("study_forecast.price_adapt.begin:" + reason, reason)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!price || !decoder) return(Fail("study_forecast.price_adapt.graph", reason)); //--- Rewind chronological inference state while retaining OMPB Reference/Current histories. for(int i = 0; i < 5; i++) if(!m_market.Layer(i).Clear()) return(Fail("study_forecast.price_adapt.trunk_runtime_clear", reason)); if(!price.Clear() || !decoder.Clear() || !ClearTeacherState()) return(Fail("study_forecast.price_adapt.head_runtime_clear", reason)); m_price_adapt_decoder_updates = decoder.UpdateCount(); m_price_adapt_updates = 0; m_price_adapt_loss_sum = 0; m_price_adapt_epoch_limit = m_ompb_profile.price_adapt_epochs; if(!DPCCResolveM05PricePhaseBudget(ArraySize(m_fit), m_price_adapt_epoch_limit, m_ompb_profile.price_adapt_max_updates, m_price_adapt_update_limit, reason)) return(Fail("study_forecast.price_adapt.phase_budget:" + reason, reason)); m_price_adapt_epochs_completed = 0; m_price_phase_terminal_status = ""; m_price_study_phase = defForecastPriceAdapt; m_cursor = 0; m_fit_limit = ArraySize(m_fit); m_finished = false; return(true); } bool FreezePriceAdapt(string &reason) { if(!SetPriceAdaptPolicy(false, reason)) return(false); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); ulong frozen_after = 0; if(!decoder || !PriceAdaptFrozenFingerprint(frozen_after, reason) || frozen_after != m_price_adapt_frozen_fingerprint || (ulong)decoder.UpdateCount() != (ulong)m_price_adapt_decoder_updates + m_price_adapt_updates) return(Fail("study_forecast.price_adapt.frozen_boundary:" + reason, reason)); return(true); } //+------------------------------------------------------------------+ //| Finish the four Market phases; PRICE_VERIFY is no longer a gate | //+------------------------------------------------------------------+ bool CompleteM05Training(string &reason) { if(!m_m05 || !m_ready || m_failed || m_price_adapt_epoch_limit == 0 || m_price_adapt_epochs_completed != m_price_adapt_epoch_limit || m_price_adapt_updates == 0 || !SetPriceAdaptPolicy(false, reason)) return(Fail("study_forecast.M05.training_incomplete:" + reason, reason)); m_price_study_phase = defForecastPriceAdapt; m_fit_limit = ArraySize(m_fit); m_cursor = m_fit_limit; m_epoch_started = false; m_price_phase_terminal_status = "PHASE_COMPLETED"; m_finished = true; m_m05_manifest_bound = false; reason = ""; return(true); } bool RestartPriceAdaptEpoch(string &reason) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!price || !decoder || !SetPriceAdaptPolicy(true, reason)) return(Fail("study_forecast.price_adapt.epoch_policy:" + reason, reason)); for(int i = 0; i < 5; i++) if(!m_market.Layer(i) || !m_market.Layer(i).Clear()) return(Fail("study_forecast.price_adapt.epoch_trunk_clear", reason)); if(!price.Clear() || !decoder.Clear() || !ClearTeacherState()) return(Fail("study_forecast.price_adapt.epoch_head_clear", reason)); m_cursor = 0; return(true); } //+------------------------------------------------------------------+ //| Fit only the decoder through the ordinary full Layer7 CNet pass | //+------------------------------------------------------------------+ bool AdvancePriceAdapt(double &loss, double &rho_sum, uint &rho_nonzero, uint &generator_step, uint &router_step, uint &confidence_step, string &reason) { if(!m_m05 || !m_full_run || m_price_study_phase != defForecastPriceAdapt || m_cursor < 0 || m_cursor >= ArraySize(m_fit) || !SetPriceAdaptPolicy(true, reason)) return(Fail("study_forecast.price_adapt.lifecycle:" + reason, reason)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioForecast *forecast = price; CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); if(!price || !decoder || decoder.UpdateCount() == UINT_MAX) return(Fail("study_forecast.price_adapt.owner_or_counter", reason)); CNeuronConvOCL *heads[3]; uint head_steps[3]; heads[0] = price.GetGenerator(); heads[1] = price.GetRouter(); heads[2] = price.GetConfidenceHead(); for(int h = 0; h < 3; h++) { if(!heads[h] || heads[h].NativeOptimizerStep() == UINT_MAX) return(Fail("study_forecast.price_adapt.head_counter", reason)); head_steps[h] = heads[h].NativeOptimizerStep(); } const uint decoder_before = decoder.UpdateCount(); //--- This index comes only from FIT; future values remain post-forward labels. const int decision = m_fit[m_cursor]; if(!BuildSample(decision, forecast, rank, target, reason) || !price.PriceSupervisionPrepared() || !PreparePriceJointTarget(decision, decoder, reason) || !decoder.PreparePriceLoss(price, GetPointer(m_price_target), m_price_lambda, m_price_huber, m_price_rho_tolerance, true, loss) || !MathIsValidNumber(loss) || loss < 0) return(Fail("study_forecast.price_adapt.target_or_loss:" + reason, reason)); CBufferFloat *rho = price.GetResponsibilities(); if(!rho || rho.Total() != 21 || !rho.BufferRead()) return(Fail("study_forecast.price_adapt.rho", reason)); for(int k = 0; k < 21; k++) { const float value = (*rho)[k]; if(!MathIsValidNumber(value) || value < 0 || value > 1) return(Fail("study_forecast.price_adapt.rho_numeric", reason)); rho_sum += (double)value; if(value > 0) rho_nonzero++; } if(rho_nonzero == 0 || MathAbs(rho_sum - 1.0) > (double)m_price_rho_tolerance || !m_market.backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 7, false)) return(Fail("study_forecast.price_adapt.backward_transaction", reason)); for(int h = 0; h < 3; h++) if(heads[h].NativeOptimizerStep() != head_steps[h]) return(Fail("study_forecast.price_adapt.forecast_optimizer_changed", reason)); if(decoder.UpdateCount() != decoder_before + 1 || decoder.GenerationPhase() != 0 || price.PriceSupervisionPrepared() || !decoder.Clear() || !price.CancelTrainingFrame()) return(Fail("study_forecast.price_adapt.update_or_release", reason)); generator_step = head_steps[0]; router_step = head_steps[1]; confidence_step = head_steps[2]; m_price_adapt_loss_sum += loss; m_price_adapt_updates++; m_updates++; m_cursor++; const bool epoch_complete = (m_cursor == m_fit_limit); if(epoch_complete) m_price_adapt_epochs_completed++; const bool update_cap_reached = (m_price_adapt_updates >= m_price_adapt_update_limit); if(update_cap_reached && m_price_adapt_epochs_completed < m_price_adapt_epoch_limit) { if(!FreezePriceAdapt(reason)) return(false); m_price_phase_terminal_status = "STOPPED_COMPLETE_UPDATE"; m_finished = true; return(true); } if(epoch_complete && m_price_adapt_epochs_completed < m_price_adapt_epoch_limit) { if(!RestartPriceAdaptEpoch(reason)) return(false); return(true); } if(epoch_complete) { if(!FreezePriceAdapt(reason)) return(false); return(CompleteM05Training(reason)); } return(true); } bool WarmupM05Row(const int decision, const bool include_target, const ENUM_OMPB_MODE mode, const bool source_anchor, string &reason) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioForecast *forecast = price; CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); float values[]; if(!price || !decoder || !ompb || !DPCCPackStudyForecastInput(m_bars, decision, values) || !ompb.SetMode(mode) || !ompb.SetSourceAnchor(source_anchor)) return(Fail("M05_RESUME.warmup_input", reason)); const bool rollback_current = (include_target && mode == OMPB_CALIBRATE && !source_anchor); if(rollback_current && !ompb.BeginCurrentTransaction()) return(Fail("M05_RESUME.warmup_current_begin", reason)); const bool forwarded = (include_target ? BuildSample(decision, forecast, rank, target, reason) : ForwardHistoricalState(decision, values, reason)); bool restored_source = true, rolled_back = true; if(source_anchor) restored_source = ompb.SetSourceAnchor(false); if(rollback_current) rolled_back = ompb.RollbackCurrentTransaction(); if(!forwarded || !restored_source || !rolled_back || !decoder.Clear() || !price.CancelTrainingFrame()) return(Fail("M05_RESUME.warmup_release", reason)); return(true); } //+-------------------------------------------------------------------+ //| Validate factual labels after causal forward; never backpropagate | //+-------------------------------------------------------------------+ bool AdvancePriceVerify(double &loss, double &rho_sum, uint &rho_nonzero, string &reason) { if(!m_m05 || !m_verify_configured) return(Fail("study_forecast.verify.frozen_profile:" + reason, reason)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!price || !decoder) return(Fail("study_forecast.verify.graph", reason)); if(!m_verify_started) { //--- Reset transient causal state, retaining the calibrated OMPB histories. for(int i = 0; i < 5; i++) if(!m_market.Layer(i) || !m_market.Layer(i).Clear()) return(Fail("study_forecast.verify.trunk_clear", reason)); if(!price.Clear() || !decoder.Clear() || !ClearTeacherState() || !SetPriceAdaptPolicy(false, reason) || !m_market.ParameterFingerprint(m_verify_market_fingerprint) || !TeacherFingerprint(m_verify_target_fingerprint) || m_verify_target_fingerprint != m_teacher_initial_fingerprint) return(Fail("study_forecast.verify.initial_boundary:" + reason, reason)); ArrayInitialize(m_verify_error_sum, 0); m_verify_snapshots = 0; m_verify_covered = 0; m_verify_min_distinct = 21; m_verify_max_price_error = 0; m_verify_coverage = 0; m_verify_status = "RUNNING"; m_cursor = 0; m_verify_started = true; } const int count = ArraySize(m_holdout); if(count == 0) { m_verify_status = "INSUFFICIENT_DATA"; m_price_phase_terminal_status = m_verify_status; m_finished = true; return(true); } if(m_cursor < 0 || m_cursor >= count || !decoder.Clear() || !price.CancelTrainingFrame() || !SetPriceAdaptPolicy(false, reason)) return(Fail("study_forecast.verify.frame:" + reason, reason)); //--- BuildSample forwards only past/current inputs before it admits future latent labels. CNeuronScenarioForecast *forecast = price; CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); const int decision = m_holdout[m_cursor]; if(!BuildSample(decision, forecast, rank, target, reason) || !PreparePriceJointTarget(decision, decoder, reason) || !decoder.PreparePriceLoss(price, GetPointer(m_price_target), m_price_lambda, m_price_huber, m_price_rho_tolerance, false, loss)) return(Fail("study_forecast.verify.label_or_loss:" + reason, reason)); CBufferFloat *rho = price.GetResponsibilities(), *normalized = decoder.GetNormalizedScenarios(); float weights[], paths[], actual[], native[]; if(!rho || !normalized || rho.GetData(weights) != 21 || normalized.GetData(paths) != 21 * 48 || m_price_target.GetData(actual) != 48 || m_price_native.GetData(native) != 21 * 48) return(Fail("study_forecast.verify.metric_buffers", reason)); rho_sum = 0; rho_nonzero = 0; for(int k = 0; k < 21; k++) { if(!MathIsValidNumber(weights[k]) || weights[k] < 0 || weights[k] > 1) return(Fail("study_forecast.verify.rho_numeric", reason)); rho_sum += (double)weights[k]; if(weights[k] > 0) rho_nonzero++; } if(rho_nonzero == 0 || MathAbs(rho_sum - 1) > m_price_rho_tolerance) return(Fail("study_forecast.verify.rho_sum", reason)); double snapshot_error[48]; ArrayInitialize(snapshot_error, 0); for(int c = 0; c < 48; c++) { if(!MathIsValidNumber(actual[c])) return(Fail("study_forecast.verify.target_numeric", reason)); for(int k = 0; k < 21; k++) { const int at = k * 48 + c; if(!MathIsValidNumber(paths[at]) || !MathIsValidNumber(native[at])) return(Fail("study_forecast.verify.scenario_numeric", reason)); snapshot_error[c] += (double)weights[k] * MathAbs((double)paths[at] - (double)actual[c]); } if(!MathIsValidNumber(snapshot_error[c]) || !MathIsValidNumber(m_verify_error_sum[c] + snapshot_error[c])) return(Fail("study_forecast.verify.error_numeric", reason)); } bool covered = true; for(int h = 0; h < 12; h++) { double high = native[h * 4 + 1], low = native[h * 4 + 2]; for(int k = 1; k < 21; k++) { high = MathMax(high, (double)native[k * 48 + h * 4 + 1]); low = MathMin(low, (double)native[k * 48 + h * 4 + 2]); } if(DPCCHistoricalPriceTargetValue(m_bars[decision + h], 1) > high || DPCCHistoricalPriceTargetValue( m_bars[decision + h], 2) < low) covered = false; } int representatives[21]; uint distinct = 0; for(int k = 0; k < 21; k++) { bool duplicate = false; for(uint r = 0; r < distinct && !duplicate; r++) { bool equal = true; for(int c = 0; c < 48 && equal; c++) { const double a = paths[k * 48 + c], b = paths[representatives[r] * 48 + c]; const double tolerance = m_verify_profile.atol + m_verify_profile.rtol * MathMax(1.0, MathMax(MathAbs(a), MathAbs(b))); if(MathAbs(a - b) > tolerance) equal = false; } duplicate = equal; } if(!duplicate) representatives[distinct++] = k; } for(int c = 0; c < 48; c++) m_verify_error_sum[c] += snapshot_error[c]; m_verify_snapshots++; if(covered) m_verify_covered++; m_verify_min_distinct = (uint)MathMin(m_verify_min_distinct, distinct); m_cursor++; if(m_cursor == count) { ulong market_after = 0, target_after = 0; if(!decoder.Clear() || !price.CancelTrainingFrame() || !SetPriceAdaptPolicy(false, reason) || !m_market.ParameterFingerprint(market_after) || !TeacherFingerprint(target_after) || market_after != m_verify_market_fingerprint || target_after != m_verify_target_fingerprint) return(Fail("study_forecast.verify.final_boundary:" + reason, reason)); for(int c = 0; c < 48; c++) m_verify_max_price_error = MathMax(m_verify_max_price_error, m_verify_error_sum[c] / m_verify_snapshots); m_verify_coverage = (double)m_verify_covered / m_verify_snapshots; const bool failed = ( (m_verify_profile.has_max_price_error && m_verify_max_price_error > m_verify_profile.max_price_error) || (m_verify_profile.has_min_scenario_coverage && m_verify_coverage < m_verify_profile.min_scenario_coverage) || (m_verify_profile.has_min_distinct_scenarios && m_verify_min_distinct < m_verify_profile.min_distinct_scenarios)); const bool complete = (m_verify_profile.has_max_price_error && m_verify_profile.has_min_scenario_coverage && m_verify_profile.has_min_distinct_scenarios); m_verify_status = (failed ? "FAIL" : (complete ? "PASS" : "BLOCKED_MISSING_LIMITS")); m_price_phase_terminal_status = m_verify_status; m_finished = true; } return(true); } bool BeginHoldout(string &reason) { CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!ompb || !forecast || !m_market.Clear() || !ClearTeacherState() || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !SetTeacherFrozenPolicy() || !ompb.SetMode(OMPB_BYPASS) || !ompb.SetSourceAnchor(false) || !forecast.SetCodebookUpdate(false) || forecast.CodebookUpdate() || !forecast.ResetEpochDiagnostics() || !m_market.ParameterFingerprint(m_holdout_market_fingerprint) || !TeacherFingerprint(m_holdout_target_fingerprint)) return(Fail("study_forecast.holdout_policy", reason)); if(m_encoder_only) { m_decoder_market_fingerprint = m_holdout_market_fingerprint; m_decoder_target_fingerprint = m_holdout_target_fingerprint; if(m_holdout_target_fingerprint != m_teacher_initial_fingerprint) return(Fail("study_encoder.holdout.teacher_changed", reason)); } m_holdout_started = true; return(true); } bool AdvanceHoldout(string &reason) { if(!m_holdout_started && !BeginHoldout(reason)) return(false); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronBaseOCL *rank = m_market.Layer(4), *target = m_target.Layer(-1); if(!ompb || !forecast || !rank || !target || m_market.WeightsUpdateEnabled() || !TeacherFrozenPolicy() || forecast.CodebookUpdate()) return(Fail("study_forecast.holdout_mutation_policy", reason)); if(!BuildSample(m_holdout[m_holdout_cursor], forecast, rank, target, reason)) return(false); if(!forecast.CommitBatchDiagnostics()) return(Fail("study_forecast.holdout_commit", reason)); m_holdout_cursor++; if(m_holdout_cursor == (int)ArraySize(m_holdout)) { double mix = 0, router = 0, trajectory = 0, confidence = 0, observation = 0, entropy = 0, pairwise = 0, inactive = 0, recovered = 0; ulong market_after = 0, target_after = 0; if(!forecast.BuildEpochCodebookDiagnostics() || !forecast.ReadEpochDiagnostics(mix, router, trajectory, confidence, m_holdout_latent, observation, m_holdout_valid, m_holdout_invalid, entropy, pairwise, inactive, recovered) || m_holdout_valid != (double)ArraySize(m_holdout) || m_holdout_invalid != 0 || !m_market.ParameterFingerprint(market_after) || !TeacherFingerprint(target_after) || market_after != m_holdout_market_fingerprint || target_after != m_holdout_target_fingerprint) return(Fail("study_forecast.holdout_acceptance", reason)); m_holdout_latent /= (m_holdout_valid * 9.0 * 12.0 * 16.0); if(!MathIsValidNumber(m_holdout_latent)) return(Fail("study_forecast.holdout_loss_numeric", reason)); m_finished = true; } return(true); } public: CDPCCStudyForecastCore(void) { m_updates = 0; m_cursor = 0; m_holdout_cursor = 0; m_fit_limit = 0; m_max_updates = 0; m_ready = false; m_finished = false; m_failed = false; m_epoch_started = false; m_full_run = false; m_holdout_started = false; m_holdout_latent = 0; m_holdout_valid = 0; m_holdout_invalid = 0; m_holdout_market_fingerprint = 0; m_holdout_target_fingerprint = 0; m_decoder_market_fingerprint = 0; m_decoder_target_fingerprint = 0; m_restore_only = false; m_encoder_only = true; m_teacher_initial_fingerprint = 0; m_z_available = false; m_inference_split = -1; m_inference_cursor = -1; m_live_count = 0; m_raw_successor = 0; m_live_next_open = 0; m_live_prepared = false; m_price_forward_only = false; m_raw_identity_bound = false; m_ready_encoder_restored = false; m_ready_encoder_live = false; m_ready_encoder_cursor = -1; m_ready_encoder_generation = 0; m_price_joint = false; m_price_lambda = 0; m_price_huber = 0; m_price_rho_tolerance = 0; m_price_loss_sum = 0; m_price_joint_ompb_fingerprint = 0; m_price_latent_decoder_fingerprint = 0; m_latent_epochs_completed = 0; m_price_joint_updates = 0; m_price_study_phase = defForecastPriceJoint; ZeroMemory(m_ompb_profile); m_ompb_profile.profile_sha256 = ""; m_ompb_profile.raw_sha256 = ""; m_ompb_configured = false; m_ompb_calibration_completed = false; m_ompb_stage02_quality_pass = false; m_ompb_evaluation_stage = 0; m_ompb_evaluation_cursor = 0; m_ompb_evaluation_posterior = 0; ArrayInitialize(m_ompb_evaluation_loss, 0); ArrayInitialize(m_ompb_evaluation_valid, 0); ArrayInitialize(m_ompb_evaluation_invalid, 0); m_ompb_reference_complete = false; m_ompb_epoch_started = false; m_ompb_source_pending = false; m_ompb_reference_cursor = 0; m_ompb_epoch = 0; m_ompb_reference_capacity = 0; m_ompb_current_capacity = 0; m_ompb_target_updates = 0; m_ompb_source_updates = 0; m_ompb_frozen_fingerprint = 0; m_ompb_initial_fingerprint = 0; m_ompb_decoder_updates = 0; m_ompb_epoch_source_updates = 0; ArrayInitialize(m_ompb_forecast_steps, 0); m_m05 = false; m_m05_manifest_bound = false; m_price_adapt_frozen_fingerprint = 0; m_price_adapt_updates = 0; m_price_adapt_decoder_updates = 0; m_price_adapt_loss_sum = 0; ZeroMemory(m_verify_profile); m_verify_configured = false; m_verify_started = false; ArrayInitialize(m_verify_error_sum, 0); m_verify_max_price_error = 0; m_verify_coverage = 0; m_verify_snapshots = 0; m_verify_covered = 0; m_verify_min_distinct = 0; m_verify_market_fingerprint = 0; m_verify_target_fingerprint = 0; m_verify_status = "NOT_RUN"; m_price_joint_epoch_limit = 0; m_price_joint_epochs_completed = 0; m_price_joint_update_limit = 0; m_price_adapt_epoch_limit = 0; m_price_adapt_epochs_completed = 0; m_price_adapt_update_limit = 0; m_price_phase_terminal_status = ""; m_price_quote_basis_resolved = false; m_price_h1_ohlc_targets = false; m_price_server = ""; m_price_symbol = ""; m_price_timeframe = PERIOD_CURRENT; m_price_quote_mode = (ENUM_SYMBOL_CHART_MODE) - 1; } //+------------------------------------------------------------------+ //| Initialize a fresh frozen historical price graph without saving | //+------------------------------------------------------------------+ bool InitializePriceForwardLoaded(const string server, const string symbol, const ENUM_TIMEFRAMES timeframe, const string indicator_params, string &reason, const bool training_splits, const bool require_h1_ohlc_targets, const bool verify_chart_identity) { const string expected_params = (m_ompb_configured ? DPCCStudyForecastIndicatorParams(m_ompb_profile.rsi_price, m_ompb_profile.cci_price, m_ompb_profile.macd_price) : DPCCStudyForecastIndicatorParams(RSIPrice, CCIPrice, MACDPrice)); if(m_ready || m_failed || timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || server == "" || ArraySize(m_bars) == 0 || (verify_chart_identity && indicator_params != expected_params)) return(Fail("study_forecast.price_initialize.arguments_or_indicator_profile", reason)); const bool h1_ohlc_raw = (m_bars[0].raw_version == 3); if((m_bars[0].raw_version != 2 && !h1_ohlc_raw) || (require_h1_ohlc_targets && !h1_ohlc_raw)) return(Fail("TARGET_TIME_UNAVAILABLE.raw_price_target_required", reason)); if(h1_ohlc_raw) PrintFormat("DPCC_M05_RAW_INPUT timeframe=%s rows=%d/%d", EnumToString(timeframe), ArraySize(m_bars), DPCC_RAW_V3_MAX_ROWS); long chart_mode = -1; if(verify_chart_identity) { if(!SymbolInfoInteger(symbol, SYMBOL_CHART_MODE, chart_mode) || (chart_mode != SYMBOL_CHART_MODE_BID && chart_mode != SYMBOL_CHART_MODE_LAST) || m_bars[0].chart_quote_mode != (ENUM_SYMBOL_CHART_MODE)chart_mode) return(Fail("study_forecast.price_initialize.quote_basis", reason)); m_price_quote_mode = (ENUM_SYMBOL_CHART_MODE)chart_mode; } else m_price_quote_mode = m_bars[0].chart_quote_mode; if(m_price_quote_mode != SYMBOL_CHART_MODE_BID && m_price_quote_mode != SYMBOL_CHART_MODE_LAST) return(Fail("study_forecast.price_initialize.quote_basis", reason)); if(training_splits) { if(!DPCCBuildStudyForecastSplits(m_bars, m_fit, m_holdout, reason)) return(Fail("study_forecast.price_initialize.quote_basis_or_split:" + reason, reason)); } else if(ArrayResize(m_fit, 0) < 0 || ArrayResize(m_holdout, 0) < 0) return(Fail("study_forecast.price_initialize.split_clear", reason)); m_price_h1_ohlc_targets = h1_ohlc_raw; m_price_quote_basis_resolved = true; if(training_splits && !DPCCFilterStudyForecastPriceTargets(m_bars, m_price_quote_mode, m_price_h1_ohlc_targets, m_fit, m_holdout, reason, timeframe)) return(Fail("study_forecast.price_initialize.price_split:" + reason, reason)); if(training_splits && (ArraySize(m_fit) == 0 || ArraySize(m_holdout) == 0)) return(Fail("TARGET_TIME_UNAVAILABLE", reason)); m_price_server = server; m_price_symbol = symbol; m_price_timeframe = timeframe; m_raw_identity_bound = verify_chart_identity; m_price_forward_only = true; m_encoder_only = false; //--- Freeze all neural parameters; native output remains caller-owned. if(!CreateGraphs(reason, training_splits)) return(Fail("study_forecast.price_initialize.graph_or_buffers:" + reason, reason)); if(training_splits && (!m_market.Clear() || !ClearTeacherState() || !m_price_native.BufferInit(21 * 12 * 4, 0) || !m_price_native.BufferCreate( m_market.GetOpenCL()) || !m_market.ParameterFingerprint(m_decoder_market_fingerprint) || !TeacherFingerprint(m_decoder_target_fingerprint))) return(Fail("study_forecast.price_initialize.graph_or_buffers:" + reason, reason)); m_ready = true; if(training_splits && !FrozenProducer(reason)) return(false); reason = ""; return(true); } bool InitializePriceForwardFromBars(const SDPCCStudyForecastBar &bars[], const string server, const ENUM_TIMEFRAMES timeframe, const string indicator_params, const long successor_open, string &reason, const bool require_h1_ohlc_targets = false) { if(m_ready || m_failed || ArraySize(bars) == 0 || ArrayCopy(m_bars, bars) != ArraySize(bars)) return(Fail("study_forecast.price_initialize.preloaded_bars", reason)); m_raw_sha = ""; m_raw_successor = successor_open; return(InitializePriceForwardLoaded(server, _Symbol, timeframe, indicator_params, reason, false, require_h1_ohlc_targets, false)); } bool InitializePriceForward(const string raw_path, const string raw_sha, const ENUM_TIMEFRAMES timeframe, string &reason, const bool common_files = false, const bool training_splits = true, const bool require_h1_ohlc_targets = false, const bool verify_raw_sha = true, const bool verify_source_identity = true) { string server = "", symbol = "", period = "", indicator_params = "", actual_sha = "", canonical_raw = ""; const string expected_params = (m_ompb_configured ? DPCCStudyForecastIndicatorParams(m_ompb_profile.rsi_price, m_ompb_profile.cci_price, m_ompb_profile.macd_price) : DPCCStudyForecastIndicatorParams(RSIPrice, CCIPrice, MACDPrice)); if(m_ready || m_failed || timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || !DPCCCheckpointCanonicalPath(raw_path, canonical_raw, reason) || !DPCCReadStudyForecastRawEx(raw_path, raw_sha, m_bars, actual_sha, m_raw_successor, reason, server, symbol, period, indicator_params, common_files, (verify_source_identity ? expected_params : ""), verify_raw_sha, verify_raw_sha, verify_source_identity) || server == "" || (verify_source_identity && (symbol != _Symbol || period != EnumToString(timeframe)))) return(Fail("study_forecast.price_initialize.identity_or_split:" + reason, reason)); m_raw_sha = actual_sha; return(InitializePriceForwardLoaded(server, (verify_source_identity ? symbol : _Symbol), timeframe, indicator_params, reason, training_splits, require_h1_ohlc_targets, verify_source_identity)); } bool RestoreReadyEncoderGraph(const string market_path, const bool common_files, const SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training) { CDPCCAPRoleReader reader; if(!reader.Open(market_path, common_files, 1, expected.source_abi_sha256, expected.max_native_bytes, 64, reason)) return(Fail("ready_encoder.restore.open:" + reason, reason)); const bool loaded = reader.LoadReadyMarketInto(expected, m_market, m_market.GetOpenCL(), reason, allow_pending_training); reader.Close(); if(!loaded || !m_price_native.BufferInit(21 * 12 * 4, 0) || !m_price_native.BufferCreate(m_market.GetOpenCL())) return(Fail("ready_encoder.restore.native:" + reason, reason)); if(!m_market.SetWeightsUpdate(false)) return(Fail("ready_encoder.restore.freeze", reason)); for(int i = 0; i < 8; i++) { CNeuronBaseOCL *layer = m_market.Layer(i); if(!layer) return(Fail("ready_encoder.restore.layer", reason)); layer.TrainMode(false); } CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); if(!ompb || !forecast || !forecast.SetCodebookUpdate(false) || !forecast.CancelTrainingFrame() || !forecast.SetGradientNormalization(true) || !forecast.SetPricePhase(defForecastPriceVerify) || !ompb.SetSourceAnchor(false) || !ompb.SetMode(OMPB_INFERENCE)) return(Fail("ready_encoder.restore.policy", reason)); //--- Rebuild causal runtime by warmup; OMPB Clear retains learned parameters and raw Reference. for(int i = 0; i < 6; i++) if(!m_market.Layer(i).Clear()) return(Fail("ready_encoder.restore.trunk_runtime", reason)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!decoder || !forecast.Clear() || !decoder.Clear()) return(Fail("ready_encoder.restore.forecast_runtime", reason)); m_restore_only = true; m_ready_encoder_restored = true; m_ready_encoder_live = false; m_ready_encoder_cursor = -1; m_ready_encoder_generation = 0; m_live_count = 0; m_live_next_open = m_raw_successor; m_live_prepared = false; ArrayResize(m_live_bars, 0); return(FrozenProducer(reason)); } //+------------------------------------------------------------------+ //| Restore an accepted Market for frozen encoder-only decisions | //+------------------------------------------------------------------+ bool RestoreReadyEncoder(const string market_path, const bool common_files, const string raw_path, const string raw_sha, const ENUM_TIMEFRAMES timeframe, const SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training = false, const bool allow_unbound_history = false) { reason = ""; string archive_sha = expected.source_archive_sha256; StringToLower(archive_sha); if(m_ready || m_failed || m_ready_encoder_restored || timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || (!allow_unbound_history && timeframe != PERIOD_H1) || expected.role != 1 || expected.section_id != 100 || expected.encoding != 2 || expected.max_native_bytes == 0 || expected.max_native_bytes > (ulong)LONG_MAX || !DPCCResolverLowerSha256(expected.source_abi_sha256) || !DPCCResolverLowerSha256(expected.profile_sha256) || !DPCCResolverLowerSha256(expected.data_sha256) || !DPCCResolverLowerSha256(archive_sha) || (!allow_unbound_history && !DPCCResolverLowerSha256(raw_sha)) || expected.market_dependency_sha256 != "" || expected.task_id == "" || expected.stage == "") return(Fail("ready_encoder.restore.contract", reason)); if(!InitializePriceForward(raw_path, raw_sha, timeframe, reason, common_files, false, false, !allow_unbound_history, !allow_unbound_history)) return(false); return(RestoreReadyEncoderGraph(market_path, common_files, expected, reason, allow_pending_training)); } bool RestoreReadyEncoderFromBars(const string market_path, const bool common_files, const SDPCCStudyForecastBar &bars[], const string source_server, const string indicator_params, const long successor_open, const ENUM_TIMEFRAMES timeframe, const SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training = false) { reason = ""; string archive_sha = expected.source_archive_sha256; StringToLower(archive_sha); if(m_ready || m_failed || m_ready_encoder_restored || timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || expected.role != 1 || expected.section_id != 100 || expected.encoding != 2 || expected.max_native_bytes == 0 || expected.max_native_bytes > (ulong)LONG_MAX || !DPCCResolverLowerSha256(expected.source_abi_sha256) || !DPCCResolverLowerSha256(expected.profile_sha256) || !DPCCResolverLowerSha256(expected.data_sha256) || !DPCCResolverLowerSha256(archive_sha) || expected.market_dependency_sha256 != "" || expected.task_id == "" || expected.stage == "") return(Fail("ready_encoder.restore.contract", reason)); if(!InitializePriceForwardFromBars(bars, source_server, timeframe, indicator_params, successor_open, reason)) return(false); return(RestoreReadyEncoderGraph(market_path, common_files, expected, reason, allow_pending_training)); } //+------------------------------------------------------------------+ //| Borrow the owning context without exposing the complete CNet | //+------------------------------------------------------------------+ COpenCLMy *ReadyEncoderContext(void) { if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only) return(NULL); return(m_market.GetOpenCL()); } //+------------------------------------------------------------------+ //| Borrow the accepted Market network for Action6 policy binding | //+------------------------------------------------------------------+ CNet *ReadyMarketNetwork(void) { if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only) return(NULL); return(GetPointer(m_market)); } //+------------------------------------------------------------------+ //| Borrow native Decoder prices only during the current generation | //+------------------------------------------------------------------+ CBufferFloat *ReadyMarketNativeOHLC(void) { if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || !m_ready_encoder_live || m_ready_encoder_generation == 0 || m_price_native.GetOpenCL() != m_market.GetOpenCL() || m_price_native.Total() != 21 * 12 * 4) return(NULL); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); if(!forecast || forecast.Generation() != m_ready_encoder_generation) return(NULL); return(GetPointer(m_price_native)); } //+------------------------------------------------------------------+ //| Borrow only the live encoder producer for an Action6 transaction | //+------------------------------------------------------------------+ CNeuronScenarioForecastPriceOCL *ReadyEncoderForecast(void) { if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || !m_ready_encoder_live) return(NULL); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); if(!forecast || forecast.Generation() != m_ready_encoder_generation) return(NULL); return(forecast); } //+------------------------------------------------------------------+ //| Copy one chronological encoder generation to the caller's GPU | //+------------------------------------------------------------------+ bool InferReadyEncoderZ(const long server_open, CBufferFloat *destination, ulong &generation, string &reason) { generation = 0; reason = ""; if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || m_ready_encoder_live) { reason = "ready_encoder.infer.lifecycle_or_pending_generation"; return(false); } int decision = -1; if(!DPCCStudyForecastUniqueServerOpen(m_bars, server_open, decision, reason) || decision < DPCC_ENCODER_INPUT_WINDOW || (m_ready_encoder_cursor >= 0 && decision != m_ready_encoder_cursor + 1)) { reason = "ready_encoder.infer.chronology:" + reason; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); COpenCLMy *context = m_market.GetOpenCL(); const ulong count = forecast ? (ulong)forecast.Scenarios() * forecast.Variables() * forecast.Horizon() * forecast.Dimension() : 0; if(!forecast || CheckPointer(destination) == POINTER_INVALID || !NNQValid(context, destination, count) || destination.Total() != (int)count || context.GetBufferSize(destination.GetIndex()) != count * sizeof(float)) { reason = "ready_encoder.infer.destination"; return(false); } for(int i = 0; i < 7; i++) if(!m_market.Layer(i) || m_market.Layer(i).OwnsBufferOrDeviceSlot(destination)) { reason = "ready_encoder.infer.destination_alias"; return(false); } CBufferFloat *owned[] = {GetPointer(m_state), GetPointer(m_future), GetPointer(m_zero), GetPointer(m_negative), GetPointer(m_delta), GetPointer(m_price_native), GetPointer(m_price_target) }; for(int i = 0; i < ArraySize(owned); i++) if(destination == owned[i] || (owned[i].GetIndex() >= 0 && destination.GetIndex() == owned[i].GetIndex())) { reason = "ready_encoder.infer.destination_owner"; return(false); } if(!FrozenProducer(reason)) return(false); //--- Caller-selected warmup bars use this same chronological prefix and release path. float values[]; if(!DPCCPackStudyForecastInput(m_bars, decision, values) || ArraySize(values) != 45) return(Fail("ready_encoder.infer.input", reason)); for(int i = 0; i < 45; i++) if(!MathIsValidNumber(values[i]) || !m_state.Update(i, values[i])) return(Fail("ready_encoder.infer.input_values", reason)); if(!m_state.BufferWrite()) return(Fail("ready_encoder.infer.input_upload", reason)); if(!m_market.ForwardEncoderPrefix(GetPointer(m_state), reason)) return(Fail("ready_encoder.infer.prefix:" + reason, reason)); if(!m_gradient_ops.CopyFinite(forecast.GetZ(), destination, (uint)count)) return(Fail("ready_encoder.infer.z_copy", reason)); if(forecast.Generation() == 0) return(Fail("ready_encoder.infer.generation", reason)); m_ready_encoder_cursor = decision; m_ready_encoder_generation = forecast.Generation(); m_ready_encoder_live = true; generation = m_ready_encoder_generation; return(true); } //+------------------------------------------------------------------+ //| Run the accepted Market and Decoder for one historical decision | //+------------------------------------------------------------------+ bool InferReadyMarketScenario(const long server_open, CBufferFloat *z_destination, ulong &generation, string &reason) { generation = 0; reason = ""; if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || m_ready_encoder_live) { reason = "ready_market.infer.lifecycle_or_pending_generation"; return(false); } int decision = -1; if(!DPCCStudyForecastUniqueServerOpen(m_bars, server_open, decision, reason) || decision < DPCC_ENCODER_INPUT_WINDOW || (m_ready_encoder_cursor >= 0 && decision != m_ready_encoder_cursor + 1)) { reason = "ready_market.infer.chronology:" + reason; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); COpenCLMy *context = m_market.GetOpenCL(); const ulong z_count = forecast ? (ulong)forecast.Scenarios() * forecast.Variables() * forecast.Horizon() * forecast.Dimension() : 0; const ulong price_count = 21 * 12 * 4; if(!forecast || !decoder || CheckPointer(context) == POINTER_INVALID || CheckPointer(z_destination) == POINTER_INVALID || !NNQValid(context, z_destination, z_count) || z_destination.Total() != (int)z_count || context.GetBufferSize(z_destination.GetIndex()) != z_count * sizeof( float) || z_count != 21 * 9 * 12 * 16 || !NNQValid(context, GetPointer(m_price_native), price_count) || m_price_native.Total() != (int)price_count || context.GetBufferSize(m_price_native.GetIndex()) != price_count * sizeof(float)) { reason = "ready_market.infer.output_contract"; return(false); } for(int i = 0; i < 8; i++) if(!m_market.Layer(i) || m_market.Layer(i).OwnsBufferOrDeviceSlot(z_destination) || m_market.Layer(i).OwnsBufferOrDeviceSlot(GetPointer(m_price_native))) { reason = "ready_market.infer.output_alias"; return(false); } if(z_destination == GetPointer(m_price_native) || z_destination.GetIndex() == m_price_native.GetIndex()) { reason = "ready_market.infer.output_alias"; return(false); } CBufferFloat *owned[] = {GetPointer(m_state), GetPointer(m_future), GetPointer(m_future_latent), GetPointer(m_zero), GetPointer(m_negative), GetPointer(m_delta), GetPointer(m_price_target) }; for(int i = 0; i < ArraySize(owned); i++) if(z_destination == owned[i] || z_destination.GetIndex() == owned[i].GetIndex()) { reason = "ready_market.infer.output_owner"; return(false); } SDPCCHistoricalPriceSnapshot snapshot; float values[]; if(!BuildHistoricalPriceSnapshot(decision, snapshot, reason) || !DPCCPackStudyForecastInput(m_bars, decision, values) || ArraySize(values) != 45) return(Fail("ready_market.infer.input_or_snapshot:" + reason, reason)); //--- Reuse the same causal anchor and native Decoder contract as M05 training. if(!decoder.Clear() || !forecast.CancelTrainingFrame() || !forecast.SetPricePhase(defForecastPriceVerify) || !decoder.SetPricePhase(defForecastPriceVerify) || !decoder.BindPriceAnchor(snapshot.anchor, snapshot.scale, snapshot.decision_id)) return(Fail("ready_market.infer.release_or_anchor:" + reason, reason)); if(m_ready_encoder_cursor < 0) { //--- Short source prefixes use every available prior bar. //--- A prefix shorter than 508 bars is warmed with the data available. const int first_decision = (int)MathMax(DPCC_ENCODER_INPUT_WINDOW, decision - 1 - DPCC_ENCODER_WARMUP_HISTORY + DPCC_ENCODER_INPUT_WINDOW); for(int warm_decision = first_decision; warm_decision < decision; warm_decision++) { float warm_values[]; if(!DPCCPackStudyForecastInput(m_bars, warm_decision, warm_values) || ArraySize(warm_values) != 45) return(Fail("ready_market.infer.warmup_input", reason)); for(int i = 0; i < 45; i++) if(!MathIsValidNumber(warm_values[i]) || !m_state.Update(i, warm_values[i])) return(Fail("ready_market.infer.warmup_values", reason)); if(!m_state.BufferWrite() || !m_market.ForwardEncoderPrefix(GetPointer(m_state), reason) || !forecast.CancelTrainingFrame()) return(Fail("ready_market.infer.warmup_forward:" + reason, reason)); } m_ready_encoder_cursor = decision - 1; } for(int i = 0; i < 45; i++) if(!MathIsValidNumber(values[i]) || !m_state.Update(i, values[i])) return(Fail("ready_market.infer.input_values", reason)); if(!m_state.BufferWrite() || !m_market.feedForward(GetPointer(m_state), 1, false, (CBufferFloat*)NULL) || !decoder.BuildNativeScenarios(GetPointer(m_price_native), 12, snapshot.tick_size) || !m_gradient_ops.CopyFinite(forecast.GetZ(), z_destination, (uint)z_count) || forecast.Generation() == 0) return(Fail("ready_market.infer.forward_or_publish:" + reason, reason)); m_ready_encoder_cursor = decision; m_ready_encoder_generation = forecast.Generation(); m_ready_encoder_live = true; generation = m_ready_encoder_generation; return(true); } //+------------------------------------------------------------------+ //| Rebase the restored Market before live evaluation. | //+------------------------------------------------------------------+ bool RebaseReadyMarketLiveHistory(const SDPCCClosedHistoryRow &history[], const long first_live_open, string &reason) { reason = ""; const int count = ArraySize(history); //--- Validate the restored model and its requested live boundary. if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || !m_price_forward_only || m_encoder_only || m_ready_encoder_live || m_ready_encoder_generation != 0 || m_live_count != 0 || m_live_next_open != m_raw_successor || count != DPCC_ENCODER_WARMUP_HISTORY || m_raw_successor <= 0 || first_live_open <= 0) { reason = "ready_market.live.rebase.lifecycle_or_window"; return(false); } SDPCCStudyForecastBar candidate[]; //--- Copy the validated chronological warm-up bars. if(ArrayResize(candidate, count) != count) { reason = "ready_market.live.rebase.capacity"; return(false); } for(int i = 0; i < count; i++) { const SDPCCClosedHistoryRow row = history[i]; float features[]; if(row.ordinal != 0 || row.bar.server_open <= 0 || row.successor_open_server <= row.bar.server_open || row.spread < 0 || row.bar.spread != row.spread || (i > 0 && (history[i - 1].bar.server_open >= row.bar.server_open || history[i - 1].successor_open_server != row.bar.server_open)) || row.bar.server_open >= first_live_open || !DPCCStudyForecastFeature(row.bar, features)) { reason = StringFormat("ready_market.live.rebase.history.%d", i); return(false); } candidate[i] = row.bar; } //--- Publish the rebased history at the first live boundary. if(history[count - 1].successor_open_server != first_live_open || ArrayResize(m_bars, count) != count || ArrayCopy(m_bars, candidate) != count) { reason = "ready_market.live.rebase.publish"; return(false); } ArrayResize(m_live_bars, 0); m_live_count = 0; m_live_next_open = first_live_open; m_live_prepared = false; m_ready_encoder_cursor = -1; m_ready_encoder_generation = 0; m_ready_encoder_live = false; return(true); } bool InferReadyMarketLive(const SDPCCClosedHistoryRow &captured, const double successor_first_quote, const long successor_quote_time_msc, const double symbol_tick_size, CBufferFloat *z_destination, ulong &generation, string &reason) { generation = 0; reason = ""; if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || !m_price_forward_only || m_encoder_only || m_ready_encoder_live || !m_price_quote_basis_resolved || m_price_timeframe != PERIOD_H1 || m_price_symbol == "" || m_price_symbol != _Symbol || m_live_next_open <= 0) { reason = "ready_market.live.lifecycle_or_binding"; return(false); } //--- The source profile and current terminal symbol must describe the same quote basis/grid. long chart_mode = -1; double current_tick_size = 0; if((m_price_quote_mode != SYMBOL_CHART_MODE_BID && m_price_quote_mode != SYMBOL_CHART_MODE_LAST) || !SymbolInfoInteger(m_price_symbol, SYMBOL_CHART_MODE, chart_mode) || chart_mode != (long)m_price_quote_mode || !SymbolInfoDouble(m_price_symbol, SYMBOL_TRADE_TICK_SIZE, current_tick_size) || !MathIsValidNumber(current_tick_size) || current_tick_size <= 0 || !MathIsValidNumber(symbol_tick_size) || symbol_tick_size <= 0 || MathAbs(current_tick_size - symbol_tick_size) > MathMax(1e-12, current_tick_size * 1e-9)) { reason = "ready_market.live.quote_basis_or_tick_size"; return(false); } const long successor_open = (long)iTime(m_price_symbol, PERIOD_H1, 0); const long period_seconds = (long)PeriodSeconds(PERIOD_H1); if(successor_open <= 0 || successor_open != captured.successor_open_server || successor_open > LONG_MAX / 1000 - period_seconds || successor_quote_time_msc <= 0 || successor_quote_time_msc < successor_open * 1000 || successor_quote_time_msc >= (successor_open + period_seconds) * 1000 || !MathIsValidNumber(successor_first_quote) || successor_first_quote <= 0) { reason = "ready_market.live.successor_quote_link"; return(false); } const double quote_ticks = successor_first_quote / symbol_tick_size; if(!MathIsValidNumber(quote_ticks) || quote_ticks < 0 || quote_ticks > (double)LONG_MAX || MathAbs(quote_ticks - MathRound(quote_ticks)) > MathMax(1e-7, MathAbs(quote_ticks) * 1e-12)) { reason = "ready_market.live.quote_price_grid"; return(false); } if(m_live_count == 0 && captured.bar.server_open != m_live_next_open) { int source_decision = -1; if(!DPCCStudyForecastUniqueServerOpen(m_bars, captured.bar.server_open, source_decision, reason)) { reason = "ready_market.live.replay_start:" + reason; return(false); } if(source_decision < DPCC_ENCODER_INPUT_WINDOW) { reason = "ready_market.live.replay_prefix_short"; return(false); } const int source_count = ArraySize(m_bars); const long source_successor = (source_decision + 1 < source_count ? m_bars[source_decision + 1].server_open : m_raw_successor); if(source_successor != captured.successor_open_server) { reason = "ready_market.live.replay_successor"; return(false); } if(ArrayResize(m_bars, source_decision) != source_decision) { reason = "ready_market.live.replay_prefix_capacity"; return(false); } m_live_next_open = captured.bar.server_open; } const long ordinal = (long)ArraySize(m_bars) + m_live_count; SDPCCClosedHistoryRow row; SDPCCStudyForecastBar prior[]; float state_values[], price_norm[]; if(ordinal >= INT_MAX || !DPCCRebaseStudyLiveRow(captured, m_live_next_open, ordinal, row, reason) || row.successor_open_server != successor_open || !MathIsValidNumber((double)row.bar.indicators[2]) || row.bar.indicators[2] == EMPTY_VALUE || row.bar.indicators[2] <= 0 || !LivePredecessors(m_live_count, prior, reason) || !DPCCPrepareStudyLiveInput(prior, row, m_live_next_open, ordinal, state_values, price_norm, reason) || ArraySize(state_values) != 45 || ArraySize(price_norm) != 16) { if(reason == "") reason = "ready_market.live.row_or_atr"; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); COpenCLMy *context = m_market.GetOpenCL(); const ulong z_count = forecast ? (ulong)forecast.Scenarios() * forecast.Variables() * forecast.Horizon() * forecast.Dimension() : 0; const ulong price_count = 21 * 12 * 4; if(!forecast || !decoder || CheckPointer(context) == POINTER_INVALID || CheckPointer(z_destination) == POINTER_INVALID || !NNQValid(context, z_destination, z_count) || z_destination.Total() != (int)z_count || context.GetBufferSize(z_destination.GetIndex()) != z_count * sizeof( float) || z_count != 21 * 9 * 12 * 16 || !NNQValid(context, GetPointer(m_price_native), price_count) || m_price_native.Total() != (int)price_count || context.GetBufferSize(m_price_native.GetIndex()) != price_count * sizeof(float)) { reason = "ready_market.live.output_contract"; return(false); } for(int i = 0; i < 8; i++) if(!m_market.Layer(i) || m_market.Layer(i).OwnsBufferOrDeviceSlot(z_destination) || m_market.Layer(i).OwnsBufferOrDeviceSlot(GetPointer(m_price_native))) { reason = "ready_market.live.output_alias"; return(false); } if(z_destination == GetPointer(m_price_native) || z_destination.GetIndex() == m_price_native.GetIndex()) { reason = "ready_market.live.output_alias"; return(false); } CBufferFloat *owned[] = {GetPointer(m_state), GetPointer(m_future), GetPointer(m_future_latent), GetPointer(m_zero), GetPointer(m_negative), GetPointer(m_delta), GetPointer(m_price_target) }; for(int i = 0; i < ArraySize(owned); i++) if(z_destination == owned[i] || z_destination.GetIndex() == owned[i].GetIndex()) { reason = "ready_market.live.output_owner"; return(false); } if(ArrayResize(m_live_bars, m_live_count + 1) != m_live_count + 1) { reason = "ready_market.live.row_capacity"; return(false); } //--- Rebuild bounded recurrent state from the pinned source before its first live successor. if(m_live_count == 0) { const int history_count = ArraySize(m_bars); if(history_count < DPCC_ENCODER_INPUT_WINDOW) { reason = "ready_market.live.pinned_warmup_history"; return(false); } for(int i = 0; i < 8; i++) if(!m_market.Layer(i) || !m_market.Layer(i).Clear()) return(Fail("ready_market.live.warmup_clear", reason)); m_ready_encoder_cursor = -1; m_ready_encoder_generation = 0; const int first_decision = (int)MathMax(DPCC_ENCODER_INPUT_WINDOW, history_count - DPCC_ENCODER_WARMUP_HISTORY + DPCC_ENCODER_INPUT_WINDOW); for(int decision = first_decision; decision <= history_count; decision++) { float warm_values[]; if(!DPCCPackStudyForecastInput(m_bars, decision, warm_values) || ArraySize(warm_values) != 45) return(Fail("ready_market.live.warmup_input", reason)); for(int i = 0; i < 45; i++) if(!MathIsValidNumber(warm_values[i]) || !m_state.Update(i, warm_values[i])) return(Fail("ready_market.live.warmup_values", reason)); if(!m_state.BufferWrite() || !m_market.ForwardEncoderPrefix(GetPointer(m_state), reason) || !forecast.CancelTrainingFrame()) return(Fail("ready_market.live.warmup_forward:" + reason, reason)); } m_ready_encoder_cursor = history_count; } //--- Only the successor's first quote and the just-closed bar ATR anchor the native Decoder. const ulong decision_id = (ulong)successor_quote_time_msc; if(!decoder.Clear() || !forecast.CancelTrainingFrame() || !forecast.SetPricePhase(defForecastPriceVerify) || !decoder.SetPricePhase(defForecastPriceVerify) || !decoder.BindPriceAnchor(successor_first_quote, (double)row.bar.indicators[2], decision_id)) return(Fail("ready_market.live.release_or_anchor", reason)); for(int i = 0; i < 45; i++) if(!MathIsValidNumber(state_values[i]) || !m_state.Update(i, state_values[i])) return(Fail("ready_market.live.input_values", reason)); if(!m_state.BufferWrite() || !m_market.feedForward(GetPointer(m_state), 1, false, (CBufferFloat*)NULL) || !decoder.BuildNativeScenarios(GetPointer(m_price_native), 12, symbol_tick_size) || !m_gradient_ops.CopyFinite(forecast.GetZ(), z_destination, (uint)z_count) || forecast.Generation() == 0) return(Fail("ready_market.live.forward_or_publish:" + reason, reason)); m_live_bars[m_live_count] = row.bar; m_live_count++; m_live_next_open = row.successor_open_server; m_ready_encoder_generation = forecast.Generation(); m_ready_encoder_live = true; generation = m_ready_encoder_generation; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Release a consumed encoder frame without clearing its history | //+------------------------------------------------------------------+ bool ReleaseReadyEncoderZ(const ulong generation, string &reason) { reason = ""; if(!m_ready_encoder_restored || !m_ready || m_failed || !m_ready_encoder_live || generation == 0 || generation != m_ready_encoder_generation) { reason = "ready_encoder.release.identity_or_boundary"; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); if(!forecast || forecast.Generation() != generation) return(Fail("ready_encoder.release.generation_or_policy:" + reason, reason)); if(!forecast.CancelTrainingFrame()) return(Fail("ready_encoder.release.forecast", reason)); m_ready_encoder_live = false; m_ready_encoder_generation = 0; return(true); } //+------------------------------------------------------------------+ //| Start a new causal history pass without reinitializing the model | //+------------------------------------------------------------------+ bool ResetReadyEncoderHistoryPass(string &reason) { reason = ""; if(!m_ready_encoder_restored || !m_ready || m_failed || !m_restore_only || m_ready_encoder_live || m_ready_encoder_generation != 0) { reason = "ready_encoder.history_pass.lifecycle_or_live_generation"; return(false); } CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); if(CheckPointer(forecast) == POINTER_INVALID || CheckPointer(ompb) == POINTER_INVALID || forecast.GetOpenCL() != m_market.GetOpenCL() || ompb.GetOpenCL() != m_market.GetOpenCL() || forecast.TrainMode() || forecast.CodebookUpdate() || ompb.Mode() != OMPB_INFERENCE || ompb.SourceAnchor()) { reason = "ready_encoder.history_pass.market_contract"; return(false); } //--- CNet::Clear clears layer execution buffers. RankTCM clears its //--- recurrent memory; OMPB::Clear explicitly retains durable Reference. if(!forecast.CancelTrainingFrame() || !m_market.Clear()) { reason = "ready_encoder.history_pass.market_clear"; return(false); } m_ready_encoder_cursor = -1; m_ready_encoder_generation = 0; m_ready_encoder_live = false; return(true); } //+------------------------------------------------------------------+ //| Freeze the selected loss profile before any joint-fit update | //+------------------------------------------------------------------+ bool InitializePriceJoint(const string raw_path, const string raw_sha, const uint max_updates, const bool require_h1_ohlc_targets, const float lambda_price, const float huber_delta, const float rho_tolerance, string &reason, const bool common_files = false, const ENUM_TIMEFRAMES timeframe = PERIOD_H1) { if(!MathIsValidNumber(lambda_price) || lambda_price <= 0 || !MathIsValidNumber(huber_delta) || huber_delta <= 0 || !MathIsValidNumber(rho_tolerance) || rho_tolerance <= 0 || rho_tolerance >= 1) return(Fail("study_forecast.price_joint.loss_profile", reason)); if(!InitializePriceForward(raw_path, raw_sha, timeframe, reason, common_files, true, require_h1_ohlc_targets)) return(false); if(!m_price_quote_basis_resolved || max_updates > (uint)ArraySize(m_fit)) return(Fail("study_forecast.price_joint.quote_basis_or_fit_limit", reason)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); if(!ompb || !m_price_target.BufferInit(12 * 4, 0) || !m_price_target.BufferCreate(m_market.GetOpenCL()) || !ompb.AppendParameterFingerprint(m_price_joint_ompb_fingerprint)) return(Fail("study_forecast.price_joint.target_buffer_or_ompb_boundary", reason)); m_price_lambda = lambda_price; m_price_huber = huber_delta; m_price_rho_tolerance = rho_tolerance; m_teacher_initial_fingerprint = m_decoder_target_fingerprint; m_max_updates = max_updates; m_full_run = (max_updates == 0); m_fit_limit = (m_full_run ? ArraySize(m_fit) : (int)max_updates); m_price_forward_only = false; m_price_joint = true; return(true); } //+------------------------------------------------------------------+ //| Bind resolved OMPB settings without choosing any profile values | //+------------------------------------------------------------------+ bool ConfigureM05OMPB(const SDPCCM05OMPBProfile &profile, string &reason) { reason = ""; if(m_ready || m_failed || m_ompb_configured || profile.reference_start <= 0 || profile.reference_end <= profile.reference_start || profile.reference_end > profile.calibration_start || profile.calibration_start <= 0 || profile.calibration_end <= profile.calibration_start || profile.epochs == 0 || profile.latent_epochs == 0 || profile.latent_start <= 0 || profile.latent_end <= profile.latent_start || !DPCCResolverLowerSha256(profile.market_preset_sha256) || profile.market_preset_path == "" || profile.calibration_preset_path == "" || !DPCCResolverLowerSha256(profile.assigned_profile_sha256) || !DPCCResolverLowerSha256(profile.market_profile_sha256) || !DPCCResolverLowerSha256(profile.calibration_preset_sha256) || !DPCCStudyForecastAppliedPriceValid((long)profile.rsi_price) || !DPCCStudyForecastAppliedPriceValid((long)profile.cci_price) || !DPCCStudyForecastAppliedPriceValid((long)profile.macd_price) || !MathIsValidNumber(profile.tau) || profile.tau <= 0 || profile.tau > 1 || !MathIsValidNumber(profile.lambda_dis) || profile.lambda_dis < 0 || !MathIsValidNumber(profile.lambda_kl) || profile.lambda_kl < 0 || !MathIsValidNumber(profile.lambda_alpha) || profile.lambda_alpha < 0 || !MathIsValidNumber(profile.alpha_prior) || !MathIsValidNumber(profile.max_kl) || profile.max_kl < 0) return(Fail("study_forecast.ompb.required_profile", reason)); const long reference_span = (long)profile.reference_end - (long)profile.reference_start; const long calibration_span = (long)profile.calibration_end - (long)profile.calibration_start; SDPCCM05OMPBProfile staged = profile; staged.reference_split = (datetime)((long)profile.reference_start + reference_span * 4 / 5); staged.calibration_split = (datetime)((long)profile.calibration_start + calibration_span * 4 / 5); if(staged.reference_split <= profile.reference_start || staged.reference_split >= profile.reference_end || staged.calibration_split <= profile.calibration_start || staged.calibration_split >= profile.calibration_end) return(Fail("study_forecast.ompb.profile_partition", reason)); const string profile_wire = StringFormat( "ACSRM_02CALIBRATION_ADAPTER_V2\n%I64d\n%I64d\n%I64d\n%I64d\n%u\n%u\n" "%.9g\n%.9g\n%.9g\n%.9g\n%.9g\n%.9g\n%u\n%u\n%u\n", (long)profile.reference_start, (long)profile.reference_end, (long)profile.calibration_start, (long)profile.calibration_end, profile.epochs, profile.source_anchor_period, (double)profile.tau, (double)profile.lambda_dis, (double)profile.lambda_kl, ( double)profile.lambda_alpha, (double)profile.alpha_prior, (double)profile.max_kl, (uint)profile.rsi_price, (uint)profile.cci_price, (uint)profile.macd_price); uchar profile_bytes[]; const int profile_count = StringToCharArray(profile_wire, profile_bytes, 0, WHOLE_ARRAY, CP_UTF8); if(profile_count <= 1 || ArrayResize(profile_bytes, profile_count - 1) != profile_count - 1 || !DPCCSHA256Lower(profile_bytes, staged.profile_sha256, reason)) return(Fail("study_forecast.ompb.profile_hash", reason)); staged.raw_sha256 = ""; staged.reference_first_bar = -1; staged.reference_split_bar = -1; staged.reference_end_bar = -1; staged.calibration_first_bar = -1; staged.calibration_split_bar = -1; staged.calibration_end_bar = -1; m_ompb_profile = staged; m_ompb_configured = true; return(true); } //+------------------------------------------------------------------+ //| Begin latent preparation on the same eight-layer price graph | //+------------------------------------------------------------------+ bool ConfigureM05Verify(const SDPCCM05VerifyProfile &profile, string &reason) { if(m_ready || m_failed || m_verify_configured || !MathIsValidNumber(profile.atol) || profile.atol <= 0 || !MathIsValidNumber(profile.rtol) || profile.rtol <= 0 || (profile.has_max_price_error && (!MathIsValidNumber(profile.max_price_error) || profile.max_price_error <= 0)) || (profile.has_min_scenario_coverage && (!MathIsValidNumber(profile.min_scenario_coverage) || profile.min_scenario_coverage <= 0 || profile.min_scenario_coverage > 1)) || (profile.has_min_distinct_scenarios && (profile.min_distinct_scenarios < 1 || profile.min_distinct_scenarios > 21))) return(Fail("study_forecast.verify.profile_domain", reason)); m_verify_profile = profile; m_verify_configured = true; reason = ""; return(true); } bool OMPBCalibrationCompleted(void) const { return(m_ompb_calibration_completed); } bool OMPBStage02QualityPassed(void) const { return(m_ompb_stage02_quality_pass); } bool OMPBStage02QualityEvaluated(void) const { return(m_ompb_evaluation_stage >= 6); } uint OMPBTrainingEpochsCompleted(void) const { return(m_ompb_epoch); } uint OMPBTrainingEpochLimit(void) const { return(m_ompb_profile.epochs); } bool OMPBTrainingBudgetComplete(void) const { return(m_ompb_epoch >= m_ompb_profile.epochs); } uint OMPBStage02EvaluationLanes(void) const { return((uint)(m_ompb_evaluation_stage > 0 ? m_ompb_evaluation_stage : 0)); } bool OMPBStage02Evaluation(const int lane, double &loss, double &valid, double &invalid) const { if(lane < 0 || lane >= 6) return(false); loss = m_ompb_evaluation_loss[lane]; valid = m_ompb_evaluation_valid[lane]; invalid = m_ompb_evaluation_invalid[lane]; return(lane < m_ompb_evaluation_stage); } //+------------------------------------------------------------------+ //| Copy already-computed OMPB counters for the StudyForecast chart | //+------------------------------------------------------------------+ bool M05ChartStatus(SDPCCM05ChartMetrics &status) { ZeroMemory(status); if(!m_m05 || !m_ready || m_failed || !m_ompb_configured || m_price_study_phase != defForecastPriceOMPBCalibrate) return(false); SDPCCM05CompletedBoundary boundary; string reason = ""; CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); if(!ompb || !M05CompletedBoundary(boundary, reason)) return(false); status.subphase = boundary.subphase; status.epoch = m_ompb_epoch; status.epoch_limit = m_ompb_profile.epochs; status.disagreement = ompb.LastDisagreement(); status.kl = ompb.LastKL(); status.alpha_prior = ompb.LastAlphaPrior(); status.invalid_fallbacks = ompb.InvalidFallbacks(); status.kl_rejects = ompb.KLRejects(); status.diagnostics_ready = (m_ompb_target_updates > 0 || m_ompb_source_updates > 0); status.evaluation_stage = m_ompb_evaluation_stage; return(true); } string PriceVerifyStatus(void) const { return(m_verify_status); } uint PriceVerifySnapshots(void) const { return(m_verify_snapshots); } double PriceVerifyMaxError(void) const { return(m_verify_max_price_error); } double PriceVerifyCoverage(void) const { return(m_verify_coverage); } uint PriceVerifyMinDistinct(void) const { return(m_verify_min_distinct); } //+------------------------------------------------------------------+ //| Copy the actual phase and OMPB subphase without claiming D02 | //+------------------------------------------------------------------+ bool M05CompletedBoundary(SDPCCM05CompletedBoundary &state, string &reason) const { ZeroMemory(state); reason = ""; if(!m_m05 || !m_ready || m_failed || !m_ompb_configured || !DPCCResolverLowerSha256(m_ompb_profile.assigned_profile_sha256) || !DPCCResolverLowerSha256(m_ompb_profile.market_profile_sha256) || !DPCCResolverLowerSha256(m_ompb_profile.calibration_preset_sha256)) { reason = "study_forecast.M05.completed_boundary_unavailable"; return(false); } state.stage = PriceStudyStage(); state.subphase = state.stage; if(m_price_study_phase == defForecastPriceOMPBCalibrate) state.subphase = (!m_ompb_reference_complete ? "REFERENCE" : (m_ompb_source_pending ? "SOURCE_ANCHOR" : "CALIBRATION")); if(m_price_study_phase == defForecastPriceOMPBCalibrate && m_ompb_reference_complete && (m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed)) state.subphase = "EVALUATION_" + IntegerToString(m_ompb_evaluation_stage); state.profile_sha256 = m_ompb_profile.assigned_profile_sha256; state.market_profile_sha256 = m_ompb_profile.market_profile_sha256; state.source_preset_sha256 = m_ompb_profile.calibration_preset_sha256; state.raw_sha256 = m_raw_sha; state.noise_seed = m_ompb_profile.noise_seed; state.rng_state[0] = rnd_x; state.rng_state[1] = rnd_y; state.rng_state[2] = rnd_z; state.rng_state[3] = rnd_w; state.rng_state[4] = rnd_t; state.completed_updates = m_updates; state.price_joint_updates = m_price_joint_updates; state.price_adapt_updates = m_price_adapt_updates; state.ompb_target_updates = m_ompb_target_updates; state.ompb_source_updates = m_ompb_source_updates; state.latent_epochs_completed = m_latent_epochs_completed; state.price_joint_epochs_completed = m_price_joint_epochs_completed; state.price_adapt_epochs_completed = m_price_adapt_epochs_completed; state.ompb_epoch = m_ompb_epoch; state.ompb_epoch_source_updates = m_ompb_epoch_source_updates; state.next_record = PriceStudyPosition(); state.ompb_reference_cursor = m_ompb_reference_cursor; state.epoch_started = m_epoch_started; state.ompb_reference_complete = m_ompb_reference_complete; state.ompb_epoch_started = m_ompb_epoch_started; state.ompb_source_pending = m_ompb_source_pending; return(true); } //+------------------------------------------------------------------+ //| Admit a complete Market for the downstream research stages | //+------------------------------------------------------------------+ bool M05TrainingCompleted(void) const { return(m_m05 && m_ready && !m_failed && m_finished && m_price_study_phase == defForecastPriceAdapt && m_price_adapt_epoch_limit > 0 && m_price_adapt_epochs_completed == m_price_adapt_epoch_limit && m_price_adapt_updates > 0); } bool MarketConsumerReady(string &reason) const { reason = ""; if(!M05TrainingCompleted()) reason = "M05_TRAINING_NOT_COMPLETED;"; //--- Native CNet retains parameters and Adam state; recurrent history is rebuilt by warmup. if(!m_ompb_calibration_completed) reason += "D01_OMPB_CALIBRATION_NOT_COMPLETED;"; if(!m_m05_manifest_bound) reason += "D01_MANIFEST_NOT_BOUND;"; //--- Technical acceptance stays NOT_RUN; it does not block internal research stages. return(reason == ""); } bool MarketConsumerReady(void) const { string reason = ""; return(MarketConsumerReady(reason)); } //+------------------------------------------------------------------+ //| Write one completed M05 boundary to the Common current/bak pair | //+------------------------------------------------------------------+ bool SaveM05Boundary(const SDPCCExecutionControl &control, const string cause, string &path, string &reason) { path = ""; reason = ""; m_m05_manifest_bound = false; SDPCCM05CompletedBoundary boundary; if(!M05CompletedBoundary(boundary, reason) || control.stage != DPCC_STAGE_STUDYFORECAST || control.profile_sha256 != boundary.profile_sha256 || control.market_profile_sha256 != boundary.market_profile_sha256 || control.task_id == "" || !MissingTeacherAllowed() || HasTeacherNetwork() || (cause != "PHASE_COMPLETED" && cause != "CONTROLLED_STOP" && cause != "VERIFY_COMPLETED" && cause != "EPOCH_COMPLETED" && cause != "USER_ACCEPTED_LATENT_PREPARE" && cause != "USER_ACCEPTED_PARTIAL_OMPB" && cause != "USER_SELECTED_PHASE")) { if(reason == "") reason = "M05_SAVE.completed_boundary_or_profile"; return(false); } CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); if(!price || !decoder || !ompb || decoder.GenerationPhase() != 0 || price.PriceSupervisionPrepared()) { reason = "M05_SAVE.incomplete_frame"; return(false); } //--- Resolve persistence from the same assigned profile used by the producer. string profile_text = control.profile_text, observed_sha = ""; if(profile_text == "" || !DPCCCheckpointTextSHA(profile_text, observed_sha, reason) || observed_sha != boundary.profile_sha256) { reason = "M05_SAVE.profile_changed:" + reason; return(false); } CDPCCJsonDocument profile; if(!profile.Parse(profile_text, reason)) return(false); string market_profile_sha = ""; if(!DPCCExecutionMarketProfileSHA(profile, profile.Root(), market_profile_sha, reason) || market_profile_sha != boundary.market_profile_sha256) { reason = "M05_SAVE.market_profile_changed:" + reason; return(false); } const int persistence = profile.FindMember(profile.Root(), "persistence"); string directory = "", canonical = ""; long maximum_signed = 0; if(!DPCCResourceReadString(profile, persistence, "task_directory", directory, reason, "profile.persistence.task_directory") || !DPCCResourceReadLong(profile, persistence, "max_file_bytes", maximum_signed, reason, "profile.persistence.max_file_bytes") || maximum_signed <= 0 || !DPCCCheckpointCanonicalPath(directory, canonical, reason) || directory != canonical) { if(reason == "") reason = "M05_SAVE.persistence_required"; return(false); } const ulong maximum = (ulong)maximum_signed; const string current = directory + "/Market.current.dpc"; const string backup = directory + "/Market.bak.dpc"; const string lock_path = directory + "/Market.writer.lock"; //--- Bind the actual graphs, native ABI and immutable producer inputs. string graph = "", source_abi = ""; if(!DPCCAPM05Graph(m_market, 8, graph, reason) || !DPCCCheckpointTextSHA(DPCC_AP_CONTRACT + "\n" + DPCC_M05_MARKET_REVISION + "\n" + "NATIVE_CNET_LAYERS;OMPB_HISTORY_TAIL;SECTION102_OWNER_REFERENCE_V1\n" + "AUXILIARY_TARGET=" + DPCC_M05_TARGET_POLICY + "\n" + "PRICE_TARGET_POLICY=" + DPCC_M05_PRICE_TARGET_POLICY + "\n" + graph, source_abi, reason)) return(false); const string archive_sha = "2ed3f91b038394980f73d566a4c0c8c2fb53f651898d2d101b914811d28de3c8"; const string state_owner = "section100.layer5"; const string ompb_state = "{\"state_owner\":\"" + state_owner + "\"}"; const bool consumer_ready = (M05TrainingCompleted() && m_ompb_calibration_completed); //--- Translate the actual phase cursor to its terminal/server bar boundary. int decision = -1, replay_decision = -1; if(m_price_study_phase == defForecastPriceOMPBCalibrate) { if(!m_ompb_reference_complete) { if(ArraySize(m_ompb_reference) > 0) replay_decision = m_ompb_reference[0]; if(m_ompb_reference_cursor >= 0 && m_ompb_reference_cursor < ArraySize(m_ompb_reference)) decision = m_ompb_reference[m_ompb_reference_cursor]; } else if(m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed) { const bool source = (m_ompb_evaluation_stage % 2 == 0); const int count = (source ? ArraySize(m_ompb_source_evaluation) : ArraySize(m_ompb_target_evaluation)); if(count > 0) replay_decision = (source ? m_ompb_source_evaluation[0] : m_ompb_target_evaluation[0]); if(m_ompb_evaluation_cursor >= 0 && m_ompb_evaluation_cursor < count) decision = (source ? m_ompb_source_evaluation[m_ompb_evaluation_cursor] : m_ompb_target_evaluation[m_ompb_evaluation_cursor]); } else { if(ArraySize(m_ompb_calibration) > 0) replay_decision = m_ompb_calibration[0]; if(m_cursor >= 0 && m_cursor < ArraySize(m_ompb_calibration)) decision = m_ompb_calibration[m_cursor]; } } else { const int count = (m_price_study_phase == defForecastPriceVerify ? ArraySize(m_holdout) : (m_price_study_phase == defForecastPriceLatentPrepare ? ArraySize(m_latent_fit) : ArraySize( m_fit))); if(count > 0) replay_decision = (m_price_study_phase == defForecastPriceVerify ? m_holdout[0] : (m_price_study_phase == defForecastPriceLatentPrepare ? m_latent_fit[0] : m_fit[0])); if(m_cursor >= 0 && m_cursor < count) decision = (m_price_study_phase == defForecastPriceVerify ? m_holdout[m_cursor] : (m_price_study_phase == defForecastPriceLatentPrepare ? m_latent_fit[m_cursor] : m_fit[m_cursor])); } const long replay_bar = (replay_decision >= 0 && replay_decision < ArraySize(m_bars) ? m_bars[replay_decision].server_open : 0); const long next_bar = (decision >= 0 && decision < ArraySize(m_bars) ? m_bars[decision].server_open : 0); const uint epoch = (m_price_study_phase == defForecastPriceLatentPrepare ? m_latent_epochs_completed : (m_price_study_phase == defForecastPriceJoint ? m_price_joint_epochs_completed : (m_price_study_phase == defForecastPriceOMPBCalibrate ? m_ompb_epoch : m_price_adapt_epochs_completed))); string rng = "["; for(int i = 0; i < 5; i++) rng += (i == 0 ? "" : ",") + (string)boundary.rng_state[i]; rng += "]"; string evaluation = "["; for(int i = 0; i < 6; i++) evaluation += (i == 0 ? "" : ",") + "{\"loss\":" + StringFormat("%.17g", m_ompb_evaluation_loss[i]) + ",\"valid\":" + DoubleToString(m_ompb_evaluation_valid[i], 0) + ",\"invalid\":" + DoubleToString(m_ompb_evaluation_invalid[i], 0) + "}"; evaluation += "]"; string verify_errors = "["; for(int i = 0; i < 48; i++) verify_errors += (i == 0 ? "" : ",") + StringFormat("%.17g", m_verify_error_sum[i]); verify_errors += "]"; const string resume = "{\"stage\":" + DPCCAPJSONString(boundary.stage) + ",\"subphase\":" + DPCCAPJSONString(boundary.subphase) + ",\"epoch\":" + (string)epoch + ",\"next_record\":" + (string)boundary.next_record + ",\"cursor\":" + (string)m_cursor + ",\"fit_limit\":" + (string)m_fit_limit + ",\"holdout_cursor\":" + (string)m_holdout_cursor + ",\"record_order\":\"CHRONOLOGICAL_TERMINAL_BARS\",\"completed_transitions\":0,\"policy_updates\":0" + ",\"target_updates\":0,\"target_order_flag\":false,\"rng_state\":" + rng + ",\"noise_seed\":" + (string)boundary.noise_seed + ",\"profile_sha256\":" + DPCCAPJSONString(boundary.profile_sha256) + ",\"market_profile_sha256\":" + DPCCAPJSONString(boundary.market_profile_sha256) + ",\"source_preset_sha256\":" + DPCCAPJSONString(boundary.source_preset_sha256) + ",\"raw_sha256\":" + DPCCAPJSONString(boundary.raw_sha256) + ",\"journal_path\":null,\"journal_sha256\":null,\"time_basis\":\"TERMINAL_SERVER\"" + ",\"replay_from_bar\":" + (string)replay_bar + ",\"next_bar\":" + (string)next_bar + ",\"pending\":null,\"runtime_restore\":\"WARMUP_FROM_LATEST_WEIGHTS\"" + ",\"cause\":" + DPCCAPJSONString(cause) + ",\"finished\":" + (m_finished ? "true" : "false") + ",\"terminal_status\":" + DPCCAPJSONString(m_price_phase_terminal_status) + ",\"completed_updates\":" + (string)m_updates + ",\"price_joint_updates\":" + (string)m_price_joint_updates + ",\"price_adapt_updates\":" + (string)m_price_adapt_updates + ",\"ompb_target_updates\":" + (string)m_ompb_target_updates + ",\"ompb_source_updates\":" + (string)m_ompb_source_updates + ",\"latent_epochs_completed\":" + (string)m_latent_epochs_completed + ",\"price_joint_epochs_completed\":" + (string)m_price_joint_epochs_completed + ",\"price_adapt_epochs_completed\":" + (string)m_price_adapt_epochs_completed + ",\"epoch_started\":" + (m_epoch_started ? "true" : "false") + ",\"ompb_epoch_started\":" + (m_ompb_epoch_started ? "true" : "false") + ",\"ompb_epoch\":" + (string)m_ompb_epoch + ",\"ompb_epoch_source_updates\":" + (string)m_ompb_epoch_source_updates + ",\"ompb_reference_cursor\":" + (string)m_ompb_reference_cursor + ",\"ompb_reference_capacity\":" + (string)m_ompb_reference_capacity + ",\"ompb_current_capacity\":" + (string)m_ompb_current_capacity + ",\"ompb_reference_count\":" + (string)ompb.ReferenceCount() + ",\"ompb_current_count\":" + (string)ompb.CurrentCount() + ",\"ompb_reference_complete\":" + (m_ompb_reference_complete ? "true" : "false") + ",\"ompb_source_pending\":" + (m_ompb_source_pending ? "true" : "false") + ",\"ompb_evaluation_stage\":" + (string)m_ompb_evaluation_stage + ",\"ompb_evaluation_cursor\":" + (string)m_ompb_evaluation_cursor + ",\"ompb_evaluation\":" + evaluation + ",\"calibration_completed\":" + (m_ompb_calibration_completed ? "true" : "false") + ",\"stage02_quality_pass\":" + (m_ompb_stage02_quality_pass ? "true" : "false") + ",\"price_joint_loss_sum\":" + StringFormat("%.17g", m_price_loss_sum) + ",\"price_adapt_loss_sum\":" + StringFormat("%.17g", m_price_adapt_loss_sum) + ",\"teacher_initial_fingerprint\":" + (string)m_teacher_initial_fingerprint + ",\"price_joint_ompb_fingerprint\":" + (string)m_price_joint_ompb_fingerprint + ",\"latent_decoder_fingerprint\":" + (string)m_price_latent_decoder_fingerprint + ",\"ompb_frozen_fingerprint\":" + (string)m_ompb_frozen_fingerprint + ",\"ompb_initial_fingerprint\":" + (string)m_ompb_initial_fingerprint + ",\"ompb_evaluation_posterior\":" + (string)m_ompb_evaluation_posterior + ",\"price_adapt_frozen_fingerprint\":" + (string)m_price_adapt_frozen_fingerprint + ",\"verify_market_fingerprint\":" + (string)m_verify_market_fingerprint + ",\"verify_target_fingerprint\":" + (string)m_verify_target_fingerprint + ",\"ompb_decoder_step_baseline\":" + (string)m_ompb_decoder_updates + ",\"price_adapt_decoder_step_baseline\":" + (string)m_price_adapt_decoder_updates + ",\"ompb_forecast_step_baselines\":[" + (string)m_ompb_forecast_steps[0] + "," + (string)m_ompb_forecast_steps[1] + "," + (string)m_ompb_forecast_steps[2] + "]" + ",\"verify_started\":" + (m_verify_started ? "true" : "false") + ",\"verify_status\":" + DPCCAPJSONString(m_verify_status) + ",\"verify_snapshots\":" + (string)m_verify_snapshots + ",\"verify_covered\":" + (string)m_verify_covered + ",\"verify_min_distinct\":" + (string)m_verify_min_distinct + ",\"verify_max_price_error\":" + StringFormat("%.17g", m_verify_max_price_error) + ",\"verify_coverage\":" + StringFormat("%.17g", m_verify_coverage) + ",\"verify_error_sum\":" + verify_errors + "}"; const string manifest = "{\"contract_id\":\"" + DPCC_AP_CONTRACT + "\",\"revision\":\"" + DPCC_M05_MARKET_REVISION + "\",\"role\":\"Market\",\"task_id\":" + DPCCAPJSONString(control.task_id) + ",\"stage\":" + DPCCAPJSONString(boundary.stage) + ",\"graph\":" + graph + ",\"auxiliary_target_policy\":\"" + DPCC_M05_TARGET_POLICY + "\"" + ",\"source_archive_sha256\":\"" + archive_sha + "\",\"source_abi_sha256\":\"" + source_abi + "\",\"market_dependency_sha256\":null,\"data_sha256\":\"" + boundary.raw_sha256 + "\",\"time_basis\":\"TERMINAL_SERVER\",\"profile\":{\"task_id\":" + DPCCAPJSONString(control.task_id) + ",\"sha256\":\"" + boundary.profile_sha256 + "\",\"market_sha256\":\"" + boundary.market_profile_sha256 + "\",\"contents\":" + DPCCAPJSONString(profile_text) + "},\"source_presets\":{\"market_sha256\":\"" + m_ompb_profile.market_preset_sha256 + "\",\"calibration_sha256\":\"" + boundary.source_preset_sha256 + "\"},\"ompb_state_owner\":\"" + state_owner + "\",\"runtime_restore\":\"WARMUP_FROM_LATEST_WEIGHTS\",\"ready\":" + (consumer_ready ? "true" : "false") + ",\"calibration_completed\":" + (m_ompb_calibration_completed ? "true" : "false") + ",\"stage02_quality_pass\":" + (m_ompb_stage02_quality_pass ? "true" : "false") + ",\"verify_status\":" + DPCCAPJSONString(m_verify_status) + ",\"acceptance\":[{\"case_id\":\"A01-A15\",\"status\":\"NOT_RUN\",\"evidence_sha256\":null}]}"; CDPCCJsonDocument metadata; if(!metadata.Parse(manifest, reason) || !metadata.Parse(resume, reason)) return(false); //--- Reject oversized metadata before moving the previous generation. uchar metadata_bytes[]; const int manifest_bytes = StringToCharArray(manifest, metadata_bytes, 0, WHOLE_ARRAY, CP_UTF8) - 1; const int resume_bytes = StringToCharArray(resume, metadata_bytes, 0, WHOLE_ARRAY, CP_UTF8) - 1; const int ompb_bytes = StringToCharArray(ompb_state, metadata_bytes, 0, WHOLE_ARRAY, CP_UTF8) - 1; if(manifest_bytes <= 0 || manifest_bytes > 1048576 || resume_bytes <= 0 || resume_bytes > 1048576 || ompb_bytes <= 0 || ompb_bytes > 1048576 || (ulong)manifest_bytes + (ulong)resume_bytes + (ulong)ompb_bytes + DPCC_AP_HEADER_BYTES + 4 * DPCC_AP_SECTION_BYTES + DPCC_AP_FOOTER_BYTES >= maximum) { reason = "M05_SAVE.metadata_size"; return(false); } //--- One exclusive writer blocks all cooperative current/bak readers. string folders = directory + "/"; int slash = StringFind(folders, "/"); while(slash >= 0) { FolderCreate(StringSubstr(folders, 0, slash), FILE_COMMON); slash = StringFind(folders, "/", slash + 1); } const int lock = FileOpen(lock_path, FILE_READ | FILE_WRITE | FILE_BIN | FILE_COMMON); if(lock == INVALID_HANDLE) { reason = "M05_SAVE.writer_lock"; return(false); } bool saved = false; CDPCCAPRoleWriter writer; do { if(FileIsExist(current, FILE_COMMON)) { string current_reason = ""; if(DPCCAPValidateMarketFile(current, maximum, current_reason)) { if(!FileMove(current, FILE_COMMON, backup, FILE_COMMON | FILE_REWRITE)) { reason = "M05_SAVE.current_to_bak"; break; } } else { //--- An invalid current never replaces an authenticated recovery generation. if(!FileIsExist(backup, FILE_COMMON) || !DPCCAPValidateMarketFile(backup, maximum, reason)) { reason = "M05_SAVE.current_invalid_without_valid_bak:" + current_reason; break; } if(!FileDelete(current, FILE_COMMON)) { reason = "M05_SAVE.invalid_current_remove"; break; } } } if(!writer.Open(current, true, 1, source_abi, 4, maximum, reason) || !DPCCAPWriteJSON(writer, 1, manifest, reason) || !writer.WriteMarketNativeSection(100, m_market, reason) || !DPCCAPWriteJSON(writer, 102, ompb_state, reason) || !DPCCAPWriteJSON(writer, 200, resume, reason) || !writer.Finish(reason)) break; saved = true; path = current; m_m05_manifest_bound = true; } while(false); writer.Close(); FileClose(lock); return(saved); } bool InitializeM05(const string raw_path, const string raw_sha, const float lambda_price, const float huber_delta, const float rho_tolerance, string &reason, const bool common_files = false, const ENUM_TIMEFRAMES timeframe = PERIOD_H1) { if(!m_ompb_configured || !DPCCArchitectureSHA256Encoding(raw_sha)) return(Fail("study_forecast.M05.ompb_profile_required", reason)); if(m_ompb_profile.price_joint_epochs == 0 || m_ompb_profile.price_joint_early_stop_patience != 0 || m_ompb_profile.price_adapt_epochs == 0 || m_ompb_profile.price_adapt_early_stop_patience != 0) return(Fail("study_forecast.M05.profile_phase_budget", reason)); m_price_joint_epoch_limit = m_ompb_profile.price_joint_epochs; m_price_joint_epochs_completed = 0; m_price_phase_terminal_status = ""; //--- M05 phase budgets are profile-owned; zero here preserves the full eligible data index. if(!InitializePriceJoint(raw_path, raw_sha, 0, true, lambda_price, huber_delta, rho_tolerance, reason, common_files, timeframe)) return(false); if(!DPCCResolveM05PricePhaseBudget(ArraySize(m_fit), m_price_joint_epoch_limit, m_ompb_profile.price_joint_max_updates, m_price_joint_update_limit, reason)) return(Fail("study_forecast.M05.price_joint_budget:" + reason, reason)); m_price_adapt_epoch_limit = m_ompb_profile.price_adapt_epochs; if(!DPCCResolveM05PricePhaseBudget(ArraySize(m_fit), m_price_adapt_epoch_limit, m_ompb_profile.price_adapt_max_updates, m_price_adapt_update_limit, reason)) return(Fail("study_forecast.M05.price_adapt_budget:" + reason, reason)); if(HasTeacherNetwork()) return(Fail("study_forecast.M05.unified_market_has_teacher", reason)); if(m_raw_sha != raw_sha || !ResolveM05OMPBIdentity(reason)) return(Fail("study_forecast.M05.ompb_source_binding:" + reason, reason)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!decoder || !decoder.AppendParameterFingerprint(m_price_latent_decoder_fingerprint)) return(Fail("study_forecast.M05.latent_decoder_boundary", reason)); //--- Keep donor bounds for identity, but train every Market phase on the price-eligible 2025 fit rows. const int latent_first = ResolveM05TimeBoundary(m_ompb_profile.latent_start); if(latent_first < 5 || latent_first >= ArraySize(m_bars) || m_raw_successor < (long)m_ompb_profile.latent_end) return(Fail("study_forecast.M05.latent_window_context_or_end_unavailable", reason)); const int latent_count = ArraySize(m_fit); if(latent_count <= 0 || ArrayResize(m_latent_fit, latent_count) != latent_count || ArrayCopy(m_latent_fit, m_fit, 0, 0, latent_count) != latent_count) return(Fail("study_forecast.M05.latent_fit_copy", reason)); m_latent_epochs_completed = 0; m_fit_limit = ArraySize(m_latent_fit); m_price_study_phase = defForecastPriceLatentPrepare; m_m05 = true; if(!ResolveOMPBFitRows(reason)) return(false); return(true); } //+------------------------------------------------------------------+ //| Restart one phase on the current native weights and Adam state | //+------------------------------------------------------------------+ bool RestartM05Phase(const string selected_phase, string &reason, const bool rebind_history = false) { reason = ""; if(!m_m05 || !m_ready || m_failed || !m_price_joint || m_price_forward_only || m_restore_only || !m_ompb_configured || ArraySize(m_fit) == 0 || (selected_phase != "LATENT_PREPARE" && selected_phase != "PRICE_JOINT" && selected_phase != "OMPB_CALIBRATE" && selected_phase != "PRICE_ADAPT")) return(Fail("M05_SELECT.initialized_phase_required", reason)); //--- Clear execution tapes and chronological state without resetting native optimizers. CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !price || !decoder || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !SetTeacherFrozenPolicy() || !price.CancelTrainingFrame() || !m_market.Clear() || !ClearTeacherState() || !price.SetCodebookUpdate(false) || !price.SetGradientNormalization(true) || !price.ResetEpochDiagnostics() || !ompb.SetSourceAnchor(false) || !ompb.SetMode(OMPB_BYPASS)) return(Fail("M05_SELECT.runtime_reset", reason)); if(rebind_history && !ompb.ResetReference()) return(Fail("M05_SELECT.source_history_reset", reason)); if(!ompb.SetTau(m_ompb_profile.tau) || !ompb.SetDisagreementMult(m_ompb_profile.lambda_dis) || !ompb.SetKLDMult(m_ompb_profile.lambda_kl) || !ompb.SetAlphaMult(m_ompb_profile.lambda_alpha) || !ompb.SetAlphaPrior(m_ompb_profile.alpha_prior) || !ompb.SetMaxKL(m_ompb_profile.max_kl)) return(Fail("M05_SELECT.current_ompb_profile", reason)); //--- Only the cumulative update count survives as provenance; phase progress starts at row zero. m_cursor = 0; m_holdout_cursor = 0; m_fit_limit = ArraySize(m_fit); m_full_run = true; m_finished = false; m_epoch_started = false; m_holdout_started = false; m_price_phase_terminal_status = ""; m_m05_manifest_bound = false; m_latent_epochs_completed = 0; m_price_joint_epochs_completed = 0; m_price_joint_updates = 0; m_price_loss_sum = 0; m_price_adapt_epochs_completed = 0; m_price_adapt_updates = 0; m_price_adapt_loss_sum = 0; m_ompb_epoch = 0; m_ompb_epoch_source_updates = 0; m_ompb_target_updates = 0; m_ompb_source_updates = 0; m_ompb_reference_cursor = 0; m_ompb_reference_complete = (ompb.ReferenceCount() == m_ompb_reference_capacity); m_ompb_epoch_started = false; m_ompb_source_pending = false; m_ompb_calibration_completed = false; m_ompb_stage02_quality_pass = false; m_ompb_evaluation_stage = 0; m_ompb_evaluation_cursor = 0; m_ompb_evaluation_posterior = 0; ArrayInitialize(m_ompb_evaluation_loss, 0); ArrayInitialize(m_ompb_evaluation_valid, 0); ArrayInitialize(m_ompb_evaluation_invalid, 0); m_verify_started = false; m_verify_status = "NOT_RUN"; m_verify_snapshots = 0; m_verify_covered = 0; m_verify_min_distinct = 0; m_verify_max_price_error = 0; m_verify_coverage = 0; m_verify_market_fingerprint = 0; m_verify_target_fingerprint = 0; ArrayInitialize(m_verify_error_sum, 0); m_price_joint_ompb_fingerprint = 0; m_price_latent_decoder_fingerprint = 0; m_ompb_initial_fingerprint = 0; if(!TeacherFingerprint(m_teacher_initial_fingerprint) || !ompb.AppendParameterFingerprint(m_price_joint_ompb_fingerprint) || !decoder.AppendParameterFingerprint(m_price_latent_decoder_fingerprint)) return(Fail("M05_SELECT.parameter_baselines", reason)); //--- Set the selected phase directly; prior completion and quality remain advisory. if(selected_phase == "OMPB_CALIBRATE") return(BeginOMPBCalibrate(reason)); if(selected_phase == "PRICE_ADAPT") return(BeginPriceAdapt(reason, true)); m_price_study_phase = (selected_phase == "LATENT_PREPARE" ? defForecastPriceLatentPrepare : defForecastPriceJoint); if(m_price_study_phase == defForecastPriceLatentPrepare) m_fit_limit = ArraySize(m_latent_fit); if(m_fit_limit <= 0 || !price.SetPricePhase(m_price_study_phase) || !decoder.SetPricePhase(m_price_study_phase)) return(Fail("M05_SELECT.price_phase", reason)); return(true); } //+------------------------------------------------------------------+ //| AUTO resumes progress; an explicit phase loads the model only | //+------------------------------------------------------------------+ bool RestoreM05Boundary(const SDPCCExecutionControl &control, bool &restored, string &reason, const string selected_phase = "AUTO") { restored = false; reason = ""; const bool model_only = (selected_phase != "AUTO"); if(model_only && selected_phase != "LATENT_PREPARE" && selected_phase != "PRICE_JOINT" && selected_phase != "OMPB_CALIBRATE" && selected_phase != "PRICE_ADAPT") return(Fail("M05_SELECT.phase", reason)); if(!m_m05 || !m_ready || control.stage != DPCC_STAGE_STUDYFORECAST || control.task_id == "" || control.profile_sha256 != m_ompb_profile.assigned_profile_sha256 || control.market_profile_sha256 != m_ompb_profile.market_profile_sha256 || m_raw_sha != m_ompb_profile.raw_sha256) return(Fail("M05_RESUME.initialized_identity", reason)); string profile_text = control.profile_text, profile_sha = "", directory = "", canonical = ""; if(profile_text == "" || !DPCCCheckpointTextSHA(profile_text, profile_sha, reason) || profile_sha != control.profile_sha256) return(Fail("M05_RESUME.profile_input:" + reason, reason)); CDPCCJsonDocument profile_document; if(!profile_document.Parse(profile_text, reason)) return(Fail("M05_RESUME.profile_parse:" + reason, reason)); const int profile_root = profile_document.Root(); string profile_task = ""; long max_file_signed = 0; const int persistence = profile_document.FindMember(profile_root, "persistence"); if(!DPCCResourceReadString(profile_document, profile_root, "task_id", profile_task, reason, "profile.task_id") || profile_task != control.task_id || !DPCCResourceReadString(profile_document, persistence, "task_directory", directory, reason, "profile.persistence.task_directory") || !DPCCResourceReadLong(profile_document, persistence, "max_file_bytes", max_file_signed, reason, "profile.persistence.max_file_bytes") || max_file_signed <= 0 || !DPCCCheckpointCanonicalPath(directory, canonical, reason) || canonical != directory) return(Fail("M05_RESUME.persistence_or_task:" + reason, reason)); const ulong maximum = (ulong)max_file_signed; string market_graph = "", source_abi = ""; if(!DPCCAPM05Graph(m_market, 8, market_graph, reason) || !DPCCCheckpointTextSHA(DPCC_AP_CONTRACT + "\n" + DPCC_M05_MARKET_REVISION + "\n" + "NATIVE_CNET_LAYERS;OMPB_HISTORY_TAIL;SECTION102_OWNER_REFERENCE_V1\n" + "AUXILIARY_TARGET=" + DPCC_M05_TARGET_POLICY + "\n" + "PRICE_TARGET_POLICY=" + DPCC_M05_PRICE_TARGET_POLICY + "\n" + market_graph, source_abi, reason)) return(Fail("M05_RESUME.graph_identity:" + reason, reason)); const string archive_sha = "2ed3f91b038394980f73d566a4c0c8c2fb53f651898d2d101b914811d28de3c8"; string candidate_paths[2]; candidate_paths[0] = directory + "/Market.current.dpc"; candidate_paths[1] = directory + "/Market.bak.dpc"; if(!FileIsExist(candidate_paths[0], FILE_COMMON) && !FileIsExist(candidate_paths[1], FILE_COMMON)) { if(model_only) return(RestartM05Phase(selected_phase, reason)); reason = ""; return(true); } SDPCCM05ResumeRecord saved; ulong resolved_joint_cap = 0, resolved_adapt_cap = 0; bool rebind_history = false; bool loaded = false; string last_reason = "M05_RESUME.no_valid_generation"; for(int candidate = 0; candidate < 2 && !loaded; candidate++) { CDPCCAPRoleReader reader; string open_reason = ""; if(!reader.Open(candidate_paths[candidate], true, 1, source_abi, maximum, 64, open_reason)) { last_reason = open_reason; continue; } string manifest_text = "", locator_text = "", resume_text = "", read_reason = "", source_task = ""; bool metadata_ok = (reader.SectionCount() == 4 && reader.ReadJSONSection(1, 1048576, manifest_text, read_reason) && reader.ReadJSONSection(102, 1048576, locator_text, read_reason) && reader.ReadJSONSection(200, 1048576, resume_text, read_reason)); if(metadata_ok) metadata_ok = reader.VerifySectionIntegrity(100, 2, read_reason); string stage = ""; SDPCCM05ResumeBounds bounds; ZeroMemory(bounds); if(metadata_ok) { CDPCCJsonDocument resume_header; metadata_ok = (resume_header.Parse(resume_text, read_reason) && DPCCResourceReadString(resume_header, resume_header.Root(), "stage", stage, read_reason, "resume.stage")); } if(metadata_ok) { bounds.fit_limit = (stage == "LATENT_PREPARE" ? (ulong)ArraySize(m_latent_fit) : (ulong)ArraySize(m_fit)); bounds.holdout_count = ArraySize(m_holdout); bounds.reference_count = ArraySize(m_ompb_reference); bounds.reference_capacity = m_ompb_reference_capacity; bounds.current_capacity = m_ompb_current_capacity; bounds.source_evaluation_count = ArraySize(m_ompb_source_evaluation); bounds.target_evaluation_count = ArraySize(m_ompb_target_evaluation); bounds.calibration_count = ArraySize(m_ompb_calibration); if(model_only) metadata_ok = DPCCParseM05ModelBoundary(resume_text, saved, read_reason) && saved.stage == stage && saved.ompb_reference_capacity == bounds.reference_capacity && saved.ompb_current_capacity == bounds.current_capacity; else metadata_ok = DPCCParseM05ResumeRecord(resume_text, bounds, control.market_profile_sha256, m_ompb_profile.calibration_preset_sha256, m_raw_sha, saved, read_reason) && saved.stage == stage; } if(metadata_ok && !DPCCResolveM05PricePhaseBudget(ArraySize(m_fit), m_ompb_profile.price_joint_epochs, m_ompb_profile.price_joint_max_updates, resolved_joint_cap, read_reason)) { read_reason = "M05_RESUME.price_joint_budget:" + read_reason; metadata_ok = false; } if(metadata_ok && !DPCCResolveM05PricePhaseBudget(ArraySize(m_fit), m_ompb_profile.price_adapt_epochs, m_ompb_profile.price_adapt_max_updates, resolved_adapt_cap, read_reason)) { read_reason = "M05_RESUME.price_adapt_budget:" + read_reason; metadata_ok = false; } if(metadata_ok && !model_only && (saved.price_joint_updates > resolved_joint_cap || saved.price_adapt_updates > resolved_adapt_cap || saved.price_joint_epochs_completed > m_ompb_profile.price_joint_epochs || saved.price_adapt_epochs_completed > m_ompb_profile.price_adapt_epochs)) { read_reason = "M05_RESUME.profile_phase_budget_exceeded"; metadata_ok = false; } CDPCCJsonDocument manifest, locator; if(metadata_ok) metadata_ok = manifest.Parse(manifest_text, read_reason) && locator.Parse(locator_text, read_reason); if(metadata_ok) { const int root = manifest.Root(), profile = manifest.FindMember(root, "profile"), presets = manifest.FindMember(root, "source_presets"); string contract = "", revision = "", role = "", task = "", manifest_stage = "", abi = "", data = "", archive = "", profile_task = "", manifest_profile_sha = "", manifest_market_profile_sha = "", time_basis = "", restore = "", manifest_owner = "", locator_owner = "", auxiliary_target_policy = "", market_preset_sha = "", calibration_preset_sha = ""; bool ready = true; const int locator_root = locator.Root(); metadata_ok = (DPCCM05KnownMembers(manifest, root, "|contract_id|revision|role|task_id|stage|graph|auxiliary_target_policy|source_archive_sha256|source_abi_sha256|market_dependency_sha256|data_sha256|time_basis|profile|source_presets|ompb_state_owner|runtime_restore|ready|calibration_completed|stage02_quality_pass|verify_status|acceptance|", "manifest", read_reason) && DPCCResourceReadString(manifest, root, "contract_id", contract, read_reason, "manifest.contract") && DPCCResourceReadString(manifest, root, "revision", revision, read_reason, "manifest.revision") && DPCCResourceReadString(manifest, root, "role", role, read_reason, "manifest.role") && DPCCResourceReadString(manifest, root, "task_id", task, read_reason, "manifest.task") && DPCCResourceReadString(manifest, root, "stage", manifest_stage, read_reason, "manifest.stage") && DPCCResourceReadString(manifest, root, "auxiliary_target_policy", auxiliary_target_policy, read_reason, "manifest.auxiliary_target_policy") && DPCCResourceReadString(manifest, root, "source_archive_sha256", archive, read_reason, "manifest.archive") && DPCCResourceReadString(manifest, root, "source_abi_sha256", abi, read_reason, "manifest.abi") && DPCCResourceReadString(manifest, root, "data_sha256", data, read_reason, "manifest.data") && DPCCResourceReadString(manifest, root, "time_basis", time_basis, read_reason, "manifest.time_basis") && DPCCResourceReadString(manifest, profile, "task_id", profile_task, read_reason, "manifest.profile.task") && DPCCResourceReadString(manifest, profile, "sha256", manifest_profile_sha, read_reason, "manifest.profile.sha") && DPCCResourceReadString(manifest, root, "ompb_state_owner", manifest_owner, read_reason, "manifest.ompb_owner") && DPCCResourceReadString(manifest, root, "runtime_restore", restore, read_reason, "manifest.runtime_restore") && DPCCResourceReadBool(manifest, root, "ready", ready, read_reason, "manifest.ready") && DPCCResourceReadString(manifest, presets, "market_sha256", market_preset_sha, read_reason, "manifest.market_preset") && DPCCResourceReadString(manifest, presets, "calibration_sha256", calibration_preset_sha, read_reason, "manifest.calibration_preset")); if(metadata_ok) metadata_ok = DPCCM05KnownMembers(manifest, profile, "|task_id|sha256|market_sha256|contents|", "manifest.profile", read_reason) && DPCCResourceReadString(manifest, profile, "market_sha256", manifest_market_profile_sha, read_reason, "manifest.profile.market_sha"); StringToLower(archive); const int dependency = manifest.FindMember(root, "market_dependency_sha256"); const int profile_contents = manifest.FindMember(profile, "contents"); const int graph_node = manifest.FindMember(root, "graph"); const int verify_accept = manifest.FindMember(root, "acceptance"); CDPCCJsonDocument embedded_profile; string embedded_profile_sha = "", embedded_market_profile_sha = ""; uint source_adapt_epochs = m_ompb_profile.price_adapt_epochs; if(metadata_ok) { metadata_ok = (manifest.Type(profile_contents) == DPCC_JSON_STRING && DPCCCheckpointTextSHA(manifest.Text(profile_contents), embedded_profile_sha, read_reason) && embedded_profile_sha == manifest_profile_sha && embedded_profile.Parse(manifest.Text(profile_contents), read_reason) && DPCCExecutionMarketProfileSHA(embedded_profile, embedded_profile.Root(), embedded_market_profile_sha, read_reason)); if(!metadata_ok && read_reason == "") read_reason = "M05_RESUME.embedded_profile_identity"; } if(metadata_ok && model_only) { ulong source_adapt_max_updates = 0; uint source_adapt_patience = 0; metadata_ok = DPCCReadM05PricePhaseBudget(embedded_profile, embedded_profile.FindMember(embedded_profile.Root(), "market_phases"), "price_adapt", source_adapt_epochs, source_adapt_max_updates, source_adapt_patience, read_reason); } bool manifest_calibration = false, manifest_quality = false; string manifest_verify_status = ""; metadata_ok = (metadata_ok && DPCCResourceReadBool(manifest, root, "calibration_completed", manifest_calibration, read_reason, "manifest.calibration_completed") && DPCCResourceReadBool(manifest, root, "stage02_quality_pass", manifest_quality, read_reason, "manifest.stage02_quality_pass") && DPCCResourceReadString(manifest, root, "verify_status", manifest_verify_status, read_reason, "manifest.verify_status")); CNeuronOMPBOCL *initialized_ompb = dynamic_cast(m_market.Layer(5)); CLayerDescription *initialized_ompb_description = (initialized_ompb ? initialized_ompb.GetLayerInfo() : NULL); const bool initialized_capacity_ok = (initialized_ompb_description && ArraySize(initialized_ompb_description.units) == 2 && (ulong)initialized_ompb_description.units[0] == bounds.reference_capacity && (ulong)initialized_ompb_description.units[1] == bounds.current_capacity); if(initialized_ompb_description) delete initialized_ompb_description; CDPCCJsonDocument expected_market; if(metadata_ok) metadata_ok = (expected_market.Parse(market_graph, read_reason) && contract == DPCC_AP_CONTRACT && revision == DPCC_M05_MARKET_REVISION && role == "Market" && auxiliary_target_policy == DPCC_M05_TARGET_POLICY && task != "" && profile_task == task && (model_only || task == control.task_id) && manifest_stage == saved.stage && archive == archive_sha && abi == source_abi && data == saved.raw_sha256 && (model_only || data == m_raw_sha) && time_basis == "TERMINAL_SERVER" && manifest_owner == "section100.layer5" && restore == "WARMUP_FROM_LATEST_WEIGHTS" && (!ready || (saved.finished && (saved.stage == "PRICE_ADAPT" || saved.stage == "VERIFY") && saved.price_adapt_epochs_completed == source_adapt_epochs && saved.price_adapt_updates > 0 && manifest_calibration)) && manifest_profile_sha == saved.profile_sha256 && (model_only || manifest_market_profile_sha == control.market_profile_sha256) && saved.market_profile_sha256 == manifest_market_profile_sha && DPCCResolverLowerSha256(market_preset_sha) && (model_only || market_preset_sha == m_ompb_profile.market_preset_sha256) && calibration_preset_sha == saved.source_preset_sha256 && (model_only || calibration_preset_sha == m_ompb_profile.calibration_preset_sha256) && manifest.Type(dependency) == DPCC_JSON_NULL && manifest.Type(profile_contents) == DPCC_JSON_STRING && DPCCResolverLowerSha256(manifest_profile_sha) && DPCCResolverLowerSha256(manifest_market_profile_sha) && embedded_market_profile_sha == manifest_market_profile_sha && DPCCM05JsonEqual(manifest, graph_node, expected_market, expected_market.Root()) && locator.Type(locator_root) == DPCC_JSON_OBJECT && locator.ChildCount(locator_root) == 1 && DPCCM05KnownMembers(locator, locator_root, "|state_owner|", "locator", read_reason) && DPCCResourceReadString(locator, locator_root, "state_owner", locator_owner, read_reason, "locator.owner") && locator_owner == "section100.layer5" && manifest_owner == "section100.layer5" && manifest.Type(verify_accept) == DPCC_JSON_ARRAY && manifest.ChildCount(verify_accept) == 1 && initialized_capacity_ok && saved.ompb_reference_count <= bounds.reference_capacity && saved.ompb_current_count <= bounds.current_capacity && saved.calibration_completed == manifest_calibration && saved.stage02_quality_pass == manifest_quality && saved.verify_status == manifest_verify_status && (model_only || saved.noise_seed == m_ompb_profile.noise_seed)); if(metadata_ok) { source_task = task; rebind_history = (data != m_raw_sha || manifest_market_profile_sha != control.market_profile_sha256 || market_preset_sha != m_ompb_profile.market_preset_sha256 || calibration_preset_sha != m_ompb_profile.calibration_preset_sha256); if(model_only) { string embedded_task = ""; metadata_ok = (DPCCResourceReadString(embedded_profile, embedded_profile.Root(), "task_id", embedded_task, read_reason, "manifest.embedded_profile.task_id") && embedded_task == task); } } if(metadata_ok) { const int acceptance = manifest.ChildAt(verify_accept, 0); string case_id = "", status = ""; metadata_ok = (DPCCM05KnownMembers(manifest, acceptance, "|case_id|status|evidence_sha256|", "manifest.acceptance", read_reason) && DPCCResourceReadString(manifest, acceptance, "case_id", case_id, read_reason, "acceptance.case_id") && DPCCResourceReadString(manifest, acceptance, "status", status, read_reason, "acceptance.status") && case_id == "A01-A15" && status == "NOT_RUN" && manifest.Type(manifest.FindMember(acceptance, "evidence_sha256")) == DPCC_JSON_NULL); } if(metadata_ok) metadata_ok = DPCCResourceReadString(locator, locator_root, "state_owner", locator_owner, read_reason, "locator.owner") && locator_owner == manifest_owner; if(!metadata_ok && read_reason == "") read_reason = "M05_RESUME.manifest_identity"; } if(!metadata_ok) { last_reason = read_reason; reader.Close(); continue; } //--- Metadata is now bound to this generation. A native Load failure is terminal: //--- CNet::Load is destructive, so do not try another file on these objects. SDPCCAPNativeLoadContract native; native.role = 1; native.encoding = 2; native.max_native_bytes = maximum; native.source_abi_sha256 = source_abi; native.profile_sha256 = saved.profile_sha256; native.market_profile_sha256 = saved.market_profile_sha256; native.data_sha256 = saved.raw_sha256; native.market_dependency_sha256 = ""; native.source_archive_sha256 = archive_sha; native.task_id = source_task; native.stage = saved.stage; native.section_id = 100; if(!reader.LoadNativeInto(native, m_market, m_market.GetOpenCL(), read_reason)) { reader.Close(); return(Fail("M05_RESUME.native_market:" + read_reason, reason)); } reader.Close(); loaded = true; } if(!loaded) return(Fail("M05_RESUME.no_valid_generation:" + last_reason, reason)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecastPriceOCL *price = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !price || !decoder || ompb.ReferenceCount() != saved.ompb_reference_count || ompb.CurrentCount() != saved.ompb_current_count || ompb.ReferenceCount() > m_ompb_reference_capacity || ompb.CurrentCount() > m_ompb_current_capacity) return(Fail("M05_RESUME.loaded_ompb_counts", reason)); //--- Explicit selection discards source progress before applying any cursor or replaying recurrence. if(model_only) { m_updates = saved.completed_updates; if(!RestartM05Phase(selected_phase, reason, rebind_history)) return(false); restored = true; return(true); } //--- Apply the validated record only after both networks and OMPB counts agree. m_price_study_phase = (saved.stage == "LATENT_PREPARE" ? defForecastPriceLatentPrepare : (saved.stage == "PRICE_JOINT" ? defForecastPriceJoint : (saved.stage == "OMPB_CALIBRATE" ? defForecastPriceOMPBCalibrate : (saved.stage == "PRICE_ADAPT" ? defForecastPriceAdapt : defForecastPriceVerify)))); m_price_joint_epoch_limit = m_ompb_profile.price_joint_epochs; m_price_joint_update_limit = resolved_joint_cap; m_price_adapt_epoch_limit = m_ompb_profile.price_adapt_epochs; m_price_adapt_update_limit = resolved_adapt_cap; m_cursor = (int)saved.cursor; m_fit_limit = (int)saved.fit_limit; m_holdout_cursor = (int)saved.holdout_cursor; m_finished = saved.finished; m_price_phase_terminal_status = saved.terminal_status; m_updates = saved.completed_updates; m_price_joint_updates = saved.price_joint_updates; m_price_adapt_updates = saved.price_adapt_updates; m_ompb_target_updates = saved.ompb_target_updates; m_ompb_source_updates = saved.ompb_source_updates; m_latent_epochs_completed = (uint)saved.latent_epochs_completed; m_price_joint_epochs_completed = (uint)saved.price_joint_epochs_completed; m_price_adapt_epochs_completed = (uint)saved.price_adapt_epochs_completed; m_epoch_started = saved.epoch_started; m_ompb_epoch_started = saved.ompb_epoch_started; m_ompb_epoch = (uint)saved.ompb_epoch; m_ompb_epoch_source_updates = (uint)saved.ompb_epoch_source_updates; m_ompb_reference_cursor = (int)saved.ompb_reference_cursor; m_ompb_reference_complete = saved.ompb_reference_complete; m_ompb_source_pending = saved.ompb_source_pending; m_ompb_evaluation_stage = (int)saved.ompb_evaluation_stage; m_ompb_evaluation_cursor = (int)saved.ompb_evaluation_cursor; for(int i = 0; i < 6; i++) { m_ompb_evaluation_loss[i] = saved.ompb_evaluation_loss[i]; m_ompb_evaluation_valid[i] = (double)saved.ompb_evaluation_valid[i]; m_ompb_evaluation_invalid[i] = (double)saved.ompb_evaluation_invalid[i]; } m_ompb_calibration_completed = saved.calibration_completed; m_ompb_stage02_quality_pass = saved.stage02_quality_pass; m_price_loss_sum = saved.price_joint_loss_sum; m_price_adapt_loss_sum = saved.price_adapt_loss_sum; m_teacher_initial_fingerprint = saved.teacher_initial_fingerprint; m_price_joint_ompb_fingerprint = saved.price_joint_ompb_fingerprint; m_price_latent_decoder_fingerprint = saved.latent_decoder_fingerprint; m_ompb_frozen_fingerprint = saved.ompb_frozen_fingerprint; m_ompb_initial_fingerprint = saved.ompb_initial_fingerprint; m_ompb_evaluation_posterior = saved.ompb_evaluation_posterior; m_price_adapt_frozen_fingerprint = saved.price_adapt_frozen_fingerprint; m_verify_market_fingerprint = saved.verify_market_fingerprint; m_verify_target_fingerprint = saved.verify_target_fingerprint; m_ompb_decoder_updates = (uint)saved.ompb_decoder_step_baseline; m_price_adapt_decoder_updates = (uint)saved.price_adapt_decoder_step_baseline; for(int i = 0; i < 3; i++) m_ompb_forecast_steps[i] = (uint)saved.ompb_forecast_step_baselines[i]; m_verify_started = saved.verify_started; m_verify_status = saved.verify_status; m_verify_snapshots = (uint)saved.verify_snapshots; m_verify_covered = (uint)saved.verify_covered; m_verify_min_distinct = (uint)saved.verify_min_distinct; m_verify_max_price_error = saved.verify_max_price_error; m_verify_coverage = saved.verify_coverage; for(int i = 0; i < 48; i++) m_verify_error_sum[i] = saved.verify_error_sum[i]; rnd_x = (uint)saved.rng_state[0]; rnd_y = (uint)saved.rng_state[1]; rnd_z = (uint)saved.rng_state[2]; rnd_w = (uint)saved.rng_state[3]; rnd_t = (uint)saved.rng_state[4]; m_full_run = true; m_m05_manifest_bound = false; //--- Legacy VERIFY snapshots already completed PRICE_ADAPT; no holdout pass is resumed. if(saved.stage == "VERIFY" || (saved.stage == "PRICE_ADAPT" && saved.finished && saved.price_adapt_epochs_completed == m_price_adapt_epoch_limit)) { if(!CompleteM05Training(reason)) return(false); restored = true; return(true); } const uint reference_before = ompb.ReferenceCount(), current_before = ompb.CurrentCount(); const uint layer_phase = (m_price_study_phase == defForecastPriceLatentPrepare ? defForecastPriceLatentPrepare : (m_price_study_phase == defForecastPriceJoint ? defForecastPriceJoint : (m_price_study_phase == defForecastPriceOMPBCalibrate ? defForecastPriceOMPBCalibrate : (m_price_study_phase == defForecastPriceAdapt ? defForecastPriceAdapt : defForecastPriceVerify)))); ENUM_OMPB_MODE warmup_mode = OMPB_BYPASS; if(m_price_study_phase == defForecastPriceAdapt || m_price_study_phase == defForecastPriceVerify) warmup_mode = OMPB_INFERENCE; else if(m_price_study_phase == defForecastPriceOMPBCalibrate) { if(saved.subphase == "EVALUATION_2" || saved.subphase == "EVALUATION_3" || saved.subphase == "EVALUATION_4" || saved.subphase == "EVALUATION_5" || saved.subphase == "CALIBRATION" || saved.subphase == "SOURCE_ANCHOR") warmup_mode = OMPB_INFERENCE; if(saved.subphase == "CALIBRATION" || saved.subphase == "SOURCE_ANCHOR") warmup_mode = OMPB_CALIBRATE; } if(!m_market.Layer(5) || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !SetTeacherFrozenPolicy() || !price.SetCodebookUpdate(false) || !price.SetGradientNormalization(warmup_mode != OMPB_CALIBRATE) || !ompb.SetMode(warmup_mode) || !ompb.SetSourceAnchor(false) || !price.SetPricePhase(layer_phase) || !decoder.SetPricePhase(layer_phase)) return(Fail("M05_RESUME.warmup_phase", reason)); for(int i = 0; i < 5; i++) if(!m_market.Layer(i) || !m_market.Layer(i).Clear()) return(Fail("M05_RESUME.warmup_trunk_clear", reason)); if(!price.Clear() || !decoder.Clear() || !ClearTeacherState()) return(Fail("M05_RESUME.warmup_head_clear", reason)); if(!price.ResetEpochDiagnostics()) return(Fail("M05_RESUME.warmup_diagnostics_reset", reason)); int replay_decision = -1, next_decision = -1, replay_count = 0; if(m_price_study_phase == defForecastPriceLatentPrepare) { replay_decision = (ArraySize(m_latent_fit) > 0 ? m_latent_fit[0] : -1); next_decision = (m_cursor < ArraySize(m_latent_fit) ? m_latent_fit[m_cursor] : -1); replay_count = (saved.epoch_started ? m_cursor : 0); if(!saved.epoch_started) replay_decision = next_decision; } else if(m_price_study_phase == defForecastPriceJoint) { replay_decision = (ArraySize(m_fit) > 0 ? m_fit[0] : -1); next_decision = (m_cursor < ArraySize(m_fit) ? m_fit[m_cursor] : -1); replay_count = (saved.epoch_started ? m_cursor : 0); if(!saved.epoch_started) replay_decision = next_decision; } else if(m_price_study_phase == defForecastPriceAdapt) { replay_decision = (ArraySize(m_fit) > 0 ? m_fit[0] : -1); next_decision = (m_cursor < ArraySize(m_fit) ? m_fit[m_cursor] : -1); replay_count = m_cursor; } else if(m_price_study_phase == defForecastPriceVerify) { replay_decision = (ArraySize(m_holdout) > 0 ? m_holdout[0] : -1); next_decision = (m_cursor < ArraySize(m_holdout) ? m_holdout[m_cursor] : -1); replay_count = (saved.verify_started ? m_cursor : 0); if(!saved.verify_started) replay_decision = next_decision; } else if(saved.subphase == "REFERENCE") { replay_decision = (ArraySize(m_ompb_reference) > 0 ? m_ompb_reference[0] : -1); next_decision = (m_ompb_reference_cursor < ArraySize(m_ompb_reference) ? m_ompb_reference[m_ompb_reference_cursor] : -1); replay_count = m_ompb_reference_cursor; } else if(StringFind(saved.subphase, "EVALUATION_") == 0) { const bool source = (m_ompb_evaluation_stage % 2 == 0); const int row_count = (source ? ArraySize(m_ompb_source_evaluation) : ArraySize( m_ompb_target_evaluation)); replay_decision = (row_count > 0 ? (source ? m_ompb_source_evaluation[0] : m_ompb_target_evaluation[0]) : -1); next_decision = (m_ompb_evaluation_cursor < row_count ? (source ? m_ompb_source_evaluation[m_ompb_evaluation_cursor] : m_ompb_target_evaluation[m_ompb_evaluation_cursor]) : -1); replay_count = m_ompb_evaluation_cursor; } else { replay_decision = (ArraySize(m_ompb_calibration) > 0 ? m_ompb_calibration[0] : -1); next_decision = (m_cursor < ArraySize(m_ompb_calibration) ? m_ompb_calibration[m_cursor] : -1); replay_count = (saved.ompb_epoch_started ? m_cursor : 0); if(!saved.ompb_epoch_started) replay_decision = next_decision; } const long expected_replay = (replay_decision >= 0 && replay_decision < ArraySize(m_bars) ? m_bars[replay_decision].server_open : 0); const long expected_next = (next_decision >= 0 && next_decision < ArraySize(m_bars) ? m_bars[next_decision].server_open : 0); if(replay_count < 0 || (ulong)replay_count > saved.next_record || expected_replay != ( long)saved.replay_from_bar || expected_next != (long)saved.next_bar) return(Fail("M05_RESUME.server_time_replay_identity", reason)); if(m_price_study_phase == defForecastPriceOMPBCalibrate && saved.subphase != "REFERENCE" && StringFind(saved.subphase, "EVALUATION_") != 0 && saved.ompb_epoch_started) { uint replayed_sources = 0; const uint period = m_ompb_profile.source_anchor_period; for(int row = 0; row < replay_count; row++) { if(!WarmupM05Row(m_ompb_calibration[row], true, OMPB_CALIBRATE, false, reason)) return(false); const bool due = (period > 0 && (uint)(row + 1) % period == 0 && ((row + 1) < replay_count || !m_ompb_source_pending)); if(due) { if(ArraySize(m_ompb_reference) == 0) return(Fail("M05_RESUME.warmup_source_reference_empty", reason)); const uint source_index = replayed_sources % (uint)ArraySize(m_ompb_reference); if(!WarmupM05Row(m_ompb_reference[source_index], true, OMPB_CALIBRATE, true, reason)) return(Fail("M05_RESUME.warmup_source_anchor:" + reason, reason)); replayed_sources++; } } if(replayed_sources != m_ompb_epoch_source_updates) return(Fail("M05_RESUME.warmup_source_anchor_count", reason)); } else if(m_price_study_phase == defForecastPriceOMPBCalibrate && saved.subphase == "REFERENCE") { for(int row = 0; row < replay_count; row++) if(!WarmupM05Row(m_ompb_reference[row], false, OMPB_BYPASS, false, reason)) return(false); } else if(m_price_study_phase == defForecastPriceOMPBCalibrate && StringFind(saved.subphase, "EVALUATION_") == 0) { const bool source = (m_ompb_evaluation_stage % 2 == 0); for(int row = 0; row < replay_count; row++) { const int decision = (source ? m_ompb_source_evaluation[row] : m_ompb_target_evaluation[row]); if(!WarmupM05Row(decision, true, warmup_mode, false, reason)) return(false); } } else if(replay_count > 0) { if(m_price_study_phase == defForecastPriceLatentPrepare) { for(int row = 0; row < replay_count; row++) if(!WarmupM05Row(m_latent_fit[row], true, warmup_mode, false, reason)) return(false); } else if(m_price_study_phase == defForecastPriceVerify) { for(int row = 0; row < replay_count; row++) if(!WarmupM05Row(m_holdout[row], true, warmup_mode, false, reason)) return(false); } else { for(int row = 0; row < replay_count; row++) if(!WarmupM05Row(m_fit[row], true, warmup_mode, false, reason)) return(false); } } if(!price.ResetEpochDiagnostics()) return(Fail("M05_RESUME.warmup_diagnostics_finish", reason)); if(ompb.ReferenceCount() != reference_before || ompb.CurrentCount() != current_before) return(Fail("M05_RESUME.warmup_mutated_ompb_history", reason)); ENUM_OMPB_MODE restore_mode = OMPB_BYPASS; if(m_price_study_phase == defForecastPriceOMPBCalibrate) { if(!m_ompb_reference_complete) restore_mode = OMPB_REFERENCE; else if(m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed) restore_mode = (m_ompb_evaluation_stage < 2 ? OMPB_BYPASS : OMPB_INFERENCE); else restore_mode = OMPB_CALIBRATE; } if(!m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false) || !SetTeacherFrozenPolicy() || !price.SetCodebookUpdate(false) || !price.SetGradientNormalization(restore_mode != OMPB_CALIBRATE) || !price.SetPricePhase(layer_phase) || !decoder.SetPricePhase(layer_phase) || !ompb.SetMode(restore_mode) || !ompb.SetSourceAnchor(false)) return(Fail("M05_RESUME.restore_phase_policy", reason)); ompb.TrainMode(restore_mode == OMPB_CALIBRATE); if(ompb.ReferenceCount() != saved.ompb_reference_count || ompb.CurrentCount() != saved.ompb_current_count) return(Fail("M05_RESUME.final_ompb_counts", reason)); restored = true; reason = ""; return(true); } string PriceStudyStage(void) const { if(m_price_study_phase == defForecastPriceOMPBCalibrate) return("OMPB_CALIBRATE"); if(m_price_study_phase == defForecastPriceAdapt) return("PRICE_ADAPT"); if(m_price_study_phase == defForecastPriceVerify) return("VERIFY"); return(m_price_study_phase == defForecastPriceLatentPrepare ? "LATENT_PREPARE" : "PRICE_JOINT"); } int PriceStudyPosition(void) const { if(m_price_study_phase == defForecastPriceOMPBCalibrate) { if(!m_ompb_reference_complete) return(m_ompb_reference_cursor); if(m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed) return(m_ompb_evaluation_cursor); } return(m_cursor); } uint PriceStudyEpochsCompleted(void) const { if(m_price_study_phase == defForecastPriceLatentPrepare) return(m_latent_epochs_completed); if(m_price_study_phase == defForecastPriceJoint) return(m_price_joint_epochs_completed); if(m_price_study_phase == defForecastPriceOMPBCalibrate) return(m_ompb_epoch); if(m_price_study_phase == defForecastPriceAdapt) return(m_price_adapt_epochs_completed); return(0); } int PriceStudyTotal(void) const { if(m_price_study_phase == defForecastPriceOMPBCalibrate) { if(!m_ompb_reference_complete) return((int)m_ompb_reference_capacity); if(m_ompb_evaluation_stage < 4 || m_ompb_calibration_completed) return(m_ompb_evaluation_stage % 2 == 0 ? ArraySize(m_ompb_source_evaluation) : ArraySize(m_ompb_target_evaluation)); return(ArraySize(m_ompb_calibration)); } return(m_price_study_phase == defForecastPriceVerify ? ArraySize(m_holdout) : m_fit_limit); } string PriceStudyNextStage(void) const { if(m_price_study_phase == defForecastPriceLatentPrepare) return("PRICE_JOINT"); if(m_price_study_phase == defForecastPriceAdapt || m_price_study_phase == defForecastPriceVerify) return("COMPLETE"); return(m_price_study_phase == defForecastPriceOMPBCalibrate ? "PRICE_ADAPT" : "OMPB_CALIBRATE"); } //+------------------------------------------------------------------+ //| Accept the operator-selected latent boundary without new updates | //+------------------------------------------------------------------+ bool AcceptLatentPrepareComplete(string &reason) { reason = ""; if(!m_m05 || !m_ready || m_failed || m_finished || !m_price_joint || m_price_forward_only || m_restore_only || m_price_study_phase != defForecastPriceLatentPrepare || m_latent_epochs_completed == 0 || m_cursor != 0 || m_epoch_started || ArraySize(m_fit) == 0) { reason = "M05_ACCEPT_LATENT.completed_epoch_boundary_required"; return(false); } m_price_study_phase = defForecastPriceJoint; m_fit_limit = ArraySize(m_fit); m_cursor = 0; m_epoch_started = false; m_price_joint_epochs_completed = 0; m_price_phase_terminal_status = ""; m_m05_manifest_bound = false; return(true); } //+------------------------------------------------------------------+ //| Explicit PRICE_ADAPT selection accepts the current OMPB weights | //+------------------------------------------------------------------+ bool AcceptPartialOMPBForPriceAdapt(string &reason) { reason = ""; if(!m_m05 || !m_ready || m_failed || m_finished || !m_price_joint || m_price_forward_only || m_restore_only || m_price_study_phase != defForecastPriceOMPBCalibrate || !m_ompb_configured || !m_ompb_reference_complete || m_ompb_epoch == 0 || m_ompb_target_updates == 0 || ArraySize(m_fit) == 0) { reason = "M05_ACCEPT_PARTIAL_OMPB.restored_trained_state_required"; return(false); } CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); if(!ompb || !forecast || !decoder || ompb.ReferenceCount() != m_ompb_reference_capacity || ompb.CurrentCount() != m_ompb_current_capacity || !SetOMPBStudyPolicy(OMPB_INFERENCE, reason)) return(Fail("study_forecast.ompb.operator_acceptance_state:" + reason, reason)); ulong frozen = 0, posterior = 0; if(!OMPBFrozenFingerprint(frozen, reason) || frozen != m_ompb_frozen_fingerprint || !ompb.AppendParameterFingerprint(posterior) || posterior == m_ompb_initial_fingerprint || decoder.UpdateCount() != m_ompb_decoder_updates) return(Fail("study_forecast.ompb.operator_acceptance_frozen_boundary:" + reason, reason)); CNeuronConvOCL *heads[3]; heads[0] = forecast.GetGenerator(); heads[1] = forecast.GetRouter(); heads[2] = forecast.GetConfidenceHead(); for(int h = 0; h < 3; h++) if(!heads[h] || heads[h].NativeOptimizerStep() != m_ompb_forecast_steps[h]) return(Fail("study_forecast.ompb.operator_acceptance_forecast_optimizer", reason)); //--- This is operator acceptance for experimentation, not a quality PASS or full-budget completion. m_ompb_calibration_completed = true; m_ompb_epoch_started = false; m_ompb_source_pending = false; m_m05_manifest_bound = false; return(BeginPriceAdapt(reason)); } string PricePhaseTerminalStatus(void) const { return(m_price_phase_terminal_status); } //+-------------------------------------------------------------------+ //| Advance a whole PRICE_JOINT transaction, then freeze at phase end | //+-------------------------------------------------------------------+ bool AdvancePriceJoint(double &loss, double &rho_sum, uint &rho_nonzero, uint &generator_step, uint &router_step, uint &confidence_step, string &reason) { loss = 0; rho_sum = 0; rho_nonzero = 0; generator_step = 0; router_step = 0; confidence_step = 0; reason = ""; if(!m_ready || m_failed || m_finished || !m_price_joint || m_price_forward_only || m_restore_only) return(Fail("study_forecast.price_joint.lifecycle", reason)); m_m05_manifest_bound = false; if(m_price_study_phase == defForecastPriceOMPBCalibrate) return(AdvanceOMPBCalibrate(reason)); if(m_price_study_phase == defForecastPriceAdapt) return(AdvancePriceAdapt(loss, rho_sum, rho_nonzero, generator_step, router_step, confidence_step, reason)); if(m_price_study_phase == defForecastPriceVerify) return(CompleteM05Training(reason)); if(m_cursor >= m_fit_limit) return(Fail("study_forecast.price_joint.fit_limit", reason)); if(!AdvancePriceJointFit(loss, rho_sum, rho_nonzero, generator_step, router_step, confidence_step, reason)) return(false); if(m_price_study_phase == defForecastPriceJoint && m_m05) { const bool epoch_complete = (m_cursor == m_fit_limit); if(epoch_complete) m_price_joint_epochs_completed++; const bool update_cap_reached = (m_price_joint_updates >= m_price_joint_update_limit); if(update_cap_reached && m_price_joint_epochs_completed < m_price_joint_epoch_limit) { if(!FreezePriceJoint(reason)) return(false); m_price_phase_terminal_status = "STOPPED_COMPLETE_UPDATE"; m_finished = true; return(true); } if(epoch_complete && m_price_joint_epochs_completed < m_price_joint_epoch_limit) { m_cursor = 0; m_epoch_started = false; return(true); } } if(m_cursor == m_fit_limit) { if(!FreezePriceJoint(reason)) return(false); if(m_price_study_phase == defForecastPriceLatentPrepare) { CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); ulong decoder_after = 0; if(!decoder || !decoder.AppendParameterFingerprint(decoder_after) || decoder_after != m_price_latent_decoder_fingerprint) return(Fail("study_forecast.M05.latent_decoder_changed", reason)); m_latent_epochs_completed++; if(m_latent_epochs_completed < m_ompb_profile.latent_epochs) { //--- The next fit step clears recurrent tapes, retaining weights and Adam state. m_cursor = 0; m_epoch_started = false; return(true); } //--- Restart chronological runtime tapes for fit; every learned parameter stays in this graph. m_price_study_phase = defForecastPriceJoint; m_fit_limit = ArraySize(m_fit); m_cursor = 0; m_epoch_started = false; m_price_joint_epochs_completed = 0; m_price_phase_terminal_status = ""; } else if(m_m05 && m_full_run) return(BeginOMPBCalibrate(reason)); else m_finished = true; } return(true); } double PriceJointMeanLoss(void) const { return(m_price_joint_updates > 0 ? m_price_loss_sum / (double)m_price_joint_updates : 0); } //+------------------------------------------------------------------+ //| Publish one historical normalized and native OHLC generation | //+------------------------------------------------------------------+ bool AdvancePriceForward(float &normalized[], float &native[], SDPCCHistoricalPriceSnapshot &snapshot, string &reason) { ArrayResize(normalized, 0); ArrayResize(native, 0); ZeroMemory(snapshot); if(m_ready_encoder_restored || !m_price_forward_only || !m_ready || m_failed || m_cursor >= ArraySize(m_fit)) return(Fail("study_forecast.price_forward.lifecycle", reason)); const int decision = m_fit[m_cursor]; float state[]; SDPCCHistoricalPriceSnapshot observed; if(!DPCCPackStudyForecastInput(m_bars, decision, state) || !ForwardHistoricalPriceState(decision, state, observed, reason)) return(Fail("study_forecast.price_forward.causal_input:" + reason, reason)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(m_market.Layer(7)); CNeuronScenarioForecastPriceOCL *forecast = dynamic_cast(m_market.Layer(6)); ulong decision_id = 0; double anchor = 0, scale = 0; float norm_values[], native_values[]; if(!decoder || !forecast || !decoder.GetPriceAnchor(anchor, scale, decision_id) || decision_id != observed.decision_id || anchor != observed.anchor || scale != observed.scale || decoder.getOutput().GetData(norm_values) != 21 * 12 * 4 || !decoder.BuildNativeScenarios(GetPointer(m_price_native), 12, observed.tick_size) || m_price_native.GetData(native_values) != 21 * 12 * 4) return(Fail("study_forecast.price_forward.outputs_or_native_precision", reason)); //--- Admit the complete generation before publishing any scenario. for(int k = 0; k < 21; k++) for(int h = 0; h < 12; h++) { const int base = (k * 12 + h) * 4; for(int c = 0; c < 4; c++) if(!MathIsValidNumber(norm_values[base + c]) || !MathIsValidNumber(native_values[base + c]) || native_values[base + c] <= 0) return(Fail("study_forecast.price_forward.nonfinite_or_nonpositive", reason)); if(norm_values[base + 1] < MathMax(norm_values[base], norm_values[base + 3]) || norm_values[base + 2] > MathMin(norm_values[base], norm_values[base + 3]) || native_values[base + 1] < MathMax(native_values[base], native_values[base + 3]) || native_values[base + 2] > MathMin(native_values[base], native_values[base + 3]) || (h == 0 && (norm_values[base] != 0 || MathAbs((double)native_values[base] - anchor) > 0.5 * observed.tick_size))) return(Fail("study_forecast.price_forward.OHLC_or_first_open", reason)); } if(!decoder.Clear() || !forecast.CancelTrainingFrame() || ArrayCopy(normalized, norm_values) != 21 * 12 * 4 || ArrayCopy(native, native_values) != 21 * 12 * 4) { ArrayResize(normalized, 0); ArrayResize(native, 0); return(Fail("study_forecast.price_forward.release_or_publish:" + reason, reason)); } if(m_cursor + 1 == ArraySize(m_fit) && !VerifyFrozenProducerFingerprints(reason)) { ArrayResize(normalized, 0); ArrayResize(native, 0); return(false); } snapshot = observed; m_cursor++; reason = ""; return(true); } bool Initialize(const string raw_path, const string raw_sha, const string root, const uint max_updates, string &reason) { reason = ""; if(m_ready || m_failed || !DPCCReadStudyForecastRaw(raw_path, raw_sha, m_bars, m_raw_sha, reason) || !DPCCBuildStudyForecastSplits(m_bars, m_fit, m_holdout, reason)) return(Fail("study_forecast.raw_or_split:" + reason, reason)); if(max_updates > 0 && max_updates > (uint)ArraySize(m_fit)) return(Fail("study_forecast.max_updates_exceeds_fit", reason)); m_max_updates = max_updates; m_full_run = (max_updates == 0); m_fit_limit = (m_full_run ? ArraySize(m_fit) : (int)max_updates); if(!DPCCCheckpointCanonicalPath(root, m_root, reason) || FileIsExist(m_root + "/Market.initial.nnw") || FileIsExist(m_root + "/Target.initial.nnw") || FileIsExist( m_root + "/Encoder.initial.nnw") || FileIsExist(m_root + "/Teacher.initial.nnw") || FileIsExist(m_root + "/EncoderTeacher.study.txt") || FileIsExist(m_root + "/ForecastDecoder.study.txt") || !CreateGraphs(reason) || !m_target.ParameterFingerprint( m_teacher_initial_fingerprint)) return(Fail("study_forecast.initialize:" + reason, reason)); FolderCreate(m_root); const datetime now = TimeCurrent(); if(!m_market.Save(m_root + "/Encoder.initial.nnw", 0, 0, 0, now, false) || !m_target.Save(m_root + "/Teacher.initial.nnw", 0, 0, 0, now, false)) return(Fail("study_forecast.initial_save", reason)); m_ready = true; return(true); } bool Advance(double &rho_sum, uint &rho_nonzero, uint &generator_step, uint &router_step, uint &confidence_step, bool &holdout_step, string &reason) { rho_sum = 0; rho_nonzero = 0; generator_step = 0; router_step = 0; confidence_step = 0; holdout_step = false; reason = ""; if(!m_ready || m_failed || m_finished || m_restore_only || m_price_forward_only || m_price_joint) return(Fail("study_forecast.advance.lifecycle_or_price_mode", reason)); if(m_cursor < m_fit_limit) return(AdvanceFit(rho_sum, rho_nonzero, generator_step, router_step, confidence_step, reason)); if(!m_full_run) return(Fail("study_forecast.partial_holdout_refused", reason)); holdout_step = true; return(AdvanceHoldout(reason)); } bool Finished(void) const { return(m_finished && !m_failed); } bool PartialFitComplete(void) const { return(!m_full_run && m_cursor >= m_fit_limit && !m_failed); } ulong Updates(void) const { return(m_updates); } int FitCount(void) const { return(ArraySize(m_fit)); } int HoldoutCount(void) const { return(ArraySize(m_holdout)); } int HoldoutCompleted(void) const { return(m_holdout_cursor); } double HoldoutLatentError(void) const { return(m_holdout_latent); } double HoldoutValidExamples(void) const { return(m_holdout_valid); } double HoldoutInvalidExamples(void) const { return(m_holdout_invalid); } string RawSHA(void) const { return(m_raw_sha); } string OMPBProfileSHA256(void) const { return(m_ompb_profile.profile_sha256); } //+------------------------------------------------------------------+ //| CopyPinnedHistory returns the authenticated history and split | //+------------------------------------------------------------------+ bool CopyPinnedHistory(SDPCCStudyForecastBar &bars[], int &fit[], int &holdout[], string &raw_sha, string &reason) const { reason = ""; raw_sha = ""; bool outputs_cleared = true; if(ArrayResize(bars, 0) != 0) outputs_cleared = false; if(ArrayResize(fit, 0) != 0) outputs_cleared = false; if(ArrayResize(holdout, 0) != 0) outputs_cleared = false; if(!outputs_cleared) { reason = "study_forecast.copy_pinned_history.clear_outputs"; return(false); } //--- Validate the authenticated source and its split indices. const int bar_count = ArraySize(m_bars); const int fit_count = ArraySize(m_fit); const int holdout_count = ArraySize(m_holdout); if(!m_ready || m_failed || !m_restore_only || bar_count <= 0 || fit_count <= 0 || holdout_count <= 0 || !DPCCArchitectureSHA256Encoding(m_raw_sha)) { reason = "study_forecast.copy_pinned_history.lifecycle_or_shape"; return(false); } for(int i = 0; i < fit_count; i++) { if(m_fit[i] < 0 || m_fit[i] >= bar_count || (i > 0 && m_fit[i] <= m_fit[i - 1])) { reason = "study_forecast.copy_pinned_history.fit_indices"; return(false); } } for(int i = 0; i < holdout_count; i++) { if(m_holdout[i] < 0 || m_holdout[i] >= bar_count || (i > 0 && m_holdout[i] <= m_holdout[i - 1])) { reason = "study_forecast.copy_pinned_history.holdout_indices"; return(false); } } //--- Stage each array before publishing any output. SDPCCStudyForecastBar staged_bars[]; int staged_fit[], staged_holdout[]; if(ArrayResize(staged_bars, bar_count) != bar_count) { reason = "study_forecast.copy_pinned_history.stage_bars_resize"; return(false); } if(ArrayResize(staged_fit, fit_count) != fit_count) { reason = "study_forecast.copy_pinned_history.stage_fit_resize"; return(false); } if(ArrayResize(staged_holdout, holdout_count) != holdout_count) { reason = "study_forecast.copy_pinned_history.stage_holdout_resize"; return(false); } if(ArrayCopy(staged_bars, m_bars) != bar_count) { reason = "study_forecast.copy_pinned_history.stage_bars_copy"; return(false); } if(ArrayCopy(staged_fit, m_fit) != fit_count) { reason = "study_forecast.copy_pinned_history.stage_fit_copy"; return(false); } if(ArrayCopy(staged_holdout, m_holdout) != holdout_count) { reason = "study_forecast.copy_pinned_history.stage_holdout_copy"; return(false); } //--- Publish the staged arrays; clear every output if any copy fails. if(ArrayResize(bars, bar_count) != bar_count) { reason = "study_forecast.copy_pinned_history.publish_bars_resize"; return(false); } if(ArrayResize(fit, fit_count) != fit_count) { if(ArrayResize(bars, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_bars"; if(ArrayResize(fit, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_fit"; if(ArrayResize(holdout, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_holdout"; if(reason == "") reason = "study_forecast.copy_pinned_history.publish_fit_resize"; return(false); } if(ArrayResize(holdout, holdout_count) != holdout_count) { if(ArrayResize(bars, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_bars"; if(ArrayResize(fit, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_fit"; if(ArrayResize(holdout, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_holdout"; if(reason == "") reason = "study_forecast.copy_pinned_history.publish_holdout_resize"; return(false); } if(ArrayCopy(bars, staged_bars) != bar_count) { if(ArrayResize(bars, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_bars"; if(ArrayResize(fit, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_fit"; if(ArrayResize(holdout, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_holdout"; if(reason == "") reason = "study_forecast.copy_pinned_history.publish_bars_copy"; return(false); } if(ArrayCopy(fit, staged_fit) != fit_count) { if(ArrayResize(bars, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_bars"; if(ArrayResize(fit, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_fit"; if(ArrayResize(holdout, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_holdout"; if(reason == "") reason = "study_forecast.copy_pinned_history.publish_fit_copy"; return(false); } if(ArrayCopy(holdout, staged_holdout) != holdout_count) { if(ArrayResize(bars, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_bars"; if(ArrayResize(fit, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_fit"; if(ArrayResize(holdout, 0) != 0) reason = "study_forecast.copy_pinned_history.clear_holdout"; if(reason == "") reason = "study_forecast.copy_pinned_history.publish_holdout_copy"; return(false); } //--- Finalize the output identity after all arrays have been copied. raw_sha = m_raw_sha; return(true); } bool RawIndexForServerOpen(const long server_open, const bool require_fit, int &index, string &reason) const { if(!m_ready || m_failed || !DPCCStudyForecastUniqueServerOpen(m_bars, server_open, index, reason)) return(false); const int fit_position = ArrayBsearch(m_fit, index); if(require_fit && (fit_position < 0 || fit_position >= ArraySize(m_fit) || m_fit[fit_position] != index)) {index = -1; reason = "study_forecast.time_join.not_fit"; return(false);} reason = ""; return(true); } bool MatchesNativeOHLC(const int decision, const float &target_native_ohlc[], const long &server_opens[], string &reason) const { return(m_ready && !m_failed && DPCCStudyForecastMatchNativeHorizon(m_bars, decision, target_native_ohlc, server_opens, reason)); } //+------------------------------------------------------------------+ //| Restore authenticated native models without training permission | //+------------------------------------------------------------------+ bool RestoreEncoderCompleted(const string raw_path, const string expected_raw_sha, const string root, const string trusted_receipt_sha, string &reason, const bool common_files = false) { if(m_ready || m_failed || !m_encoder_only || !DPCCArchitectureSHA256Encoding(expected_raw_sha)) return(Fail("study_encoder.restore.lifecycle_or_raw_pin", reason)); if(!DPCCCheckpointCanonicalPath(root, m_root, reason) || m_root != root || !DPCCReadStudyForecastRaw(raw_path, expected_raw_sha, m_bars, m_raw_sha, m_raw_successor, reason, common_files) || !DPCCBuildStudyForecastSplits(m_bars, m_fit, m_holdout, reason)) return(Fail("study_encoder.restore.raw_or_split:" + reason, reason)); SDPCCStudyEncoderReceipt receipt; if(!DPCCReadStudyEncoderReceipt(m_root, trusted_receipt_sha, m_raw_sha, (uint)ArraySize(m_fit), (uint)ArraySize(m_holdout), receipt, reason, common_files)) return(Fail("study_encoder.restore.receipt:" + reason, reason)); const string encoder_path = m_root + "/Encoder.trained.nnw"; string actual = ""; if(!DPCCCheckpointFileSHA(encoder_path, common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, actual, reason) || actual != receipt.encoder_sha256) return(Fail("study_encoder.restore.artifact_sha", reason)); Print("DPCC_STUDY_GRAPH_INIT_BEGIN mode=restore_encoder"); if(!CreateGraphs(reason, false)) return(false); Print("DPCC_STUDY_GRAPH_INIT_COMPLETE mode=restore_encoder"); float error = 0, undefined = 0, forecast_error = 0; datetime time = 0; if(!m_market.Load(encoder_path, error, undefined, forecast_error, time, common_files) || !m_market.TrainMode(false) || !m_market.SetWeightsUpdate(false)) return(Fail("study_encoder.restore.native_load_or_freeze", reason)); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); CNeuronOMPBOCL *ompb = dynamic_cast(m_market.Layer(5)); const int types[] = {defNeuronBaseOCL, defNeuronBatchNormOCL, defNeuronDropoutOCL, defNeuronCogDriverData, defNeuronCogDriverRankTCM, defNeuronOMPBOCL, defNeuronScenarioForecast }; for(int i = 0; i < 7; i++) if(!m_market.Layer(i) || m_market.Layer(i).Type() != types[i]) return(Fail("study_encoder.restore.encoder_graph", reason)); if(!forecast || !ompb || m_market.Layer(7) != NULL || m_target.Layer(0) != NULL || m_market.Layer(0).Neurons() != 45 || forecast.Scenarios() != 21 || forecast.Variables() != 9 || forecast.GetTopK() != 5 || forecast.Horizon() != 12 || forecast.Dimension() != 16 || forecast.RecoveryAge() != 256 || !ompb.SetMode(OMPB_BYPASS) || !ompb.SetSourceAnchor(false) || !forecast.SetCodebookUpdate(false) || !m_market.ParameterFingerprint(m_decoder_market_fingerprint)) return(Fail("study_encoder.restore.geometry_or_frozen_policy", reason)); if(!DPCCCheckpointFileSHA(encoder_path, common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, actual, reason) || actual != receipt.encoder_sha256) return(Fail("study_encoder.restore.changed_artifact", reason)); if(!DPCCReadStudyEncoderReceipt(m_root, trusted_receipt_sha, m_raw_sha, (uint)ArraySize(m_fit), (uint)ArraySize(m_holdout), receipt, reason, common_files)) return(Fail("study_encoder.restore.changed_receipt:" + reason, reason)); m_decoder_target_fingerprint = 0; m_teacher_initial_fingerprint = 0; m_updates = receipt.fit_updates; m_restore_only = true; m_live_next_open = m_raw_successor; m_ready = true; reason = ""; return(true); } //+------------------------------------------------------------------+ //| AdvanceInference advances only admitted causal Encoder origins | //+------------------------------------------------------------------+ bool AdvanceInference(const int decision, string &reason) { if(!m_encoder_only || !m_ready || m_failed || !m_restore_only) return(Fail("study_encoder.inference.lifecycle", reason)); int selected = -1, position = -1; for(int split = 0; split < 2 && selected < 0; split++) { const int count = (split == 0 ? ArraySize(m_fit) : ArraySize(m_holdout)); if(count == 0) continue; const int i = (split == 0 ? ArrayBsearch(m_fit, decision) : ArrayBsearch(m_holdout, decision)); if(i >= 0 && i < count && (split == 0 ? m_fit[i] : m_holdout[i]) == decision) { selected = split; position = i; } } if(selected < 0) return(Fail("study_encoder.inference.unadmitted_decision", reason)); m_live_prepared = false; //--- RankTCM retains causal history. Forward requests continue one chronological pass. //--- A split change or repeated/backward request replays from that split's first eligible origin. if(selected != m_inference_split || position <= m_inference_cursor) { if(!m_market.Clear()) return(Fail("study_encoder.inference.split_clear", reason)); m_inference_split = selected; m_inference_cursor = -1; } CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!forecast || forecast.CodebookUpdate()) return(Fail("study_encoder.inference.forecast_policy", reason)); for(int i = m_inference_cursor + 1; i <= position; i++) { const int row = (selected == 0 ? m_fit[i] : m_holdout[i]); float state_values[]; if(!DPCCPackStudyForecastInput(m_bars, row, state_values) || ArraySize(state_values) != 45) return(Fail("study_encoder.inference.input", reason)); if(!ForwardHistoricalState(row, state_values, reason)) return(false); m_inference_cursor = i; } reason = ""; return(true); } bool InferZ(const int decision, float &z[], string &reason) { ArrayResize(z, 0); if(!AdvanceInference(decision, reason)) return(false); return(PublishZ(z, reason)); } bool InferZToDevice(const int decision, CBufferFloat *destination, string &reason) { m_z_available = false; if(!destination || destination.GetOpenCL() != m_market.GetOpenCL() || destination.GetIndex() < 0 || destination.Total() != 36288) return(Fail("study_encoder.inference.device_destination", reason)); if(!AdvanceInference(decision, reason)) return(false); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!forecast || forecast.CodebookUpdate() || m_market.WeightsUpdateEnabled()) return(Fail("study_encoder.inference.device_frozen", reason)); if(!m_gradient_ops.CopyFinite(forecast.GetZ(), destination, 36288)) return(Fail("study_encoder.inference.device_finite_copy", reason)); m_z_available = true; reason = ""; return(true); } //+-------------------------------------------------------------------+ //| InferTesterZ advances frozen Encoder on actual closed Tester bars | //+-------------------------------------------------------------------+ bool InferTesterZ(const SDPCCStudyForecastBar &closed[], const long decision_open, float &z[], string &reason) { ArrayResize(z, 0); if(!DPCCTesterOnly(reason)) return(false); const bool cold = (m_inference_split != 2); const int expected_rows = (cold ? DPCC_ENCODER_WARMUP_HISTORY : DPCC_ENCODER_INPUT_WINDOW); if(!m_encoder_only || !m_ready || m_failed || !m_restore_only || ArraySize(closed) != expected_rows || decision_open <= 0 || decision_open > INT_MAX || (m_inference_split != -1 && m_inference_split != 2) || (m_inference_split == 2 && decision_open <= m_inference_cursor)) return(Fail("study_encoder.tester.lifecycle_or_chronology_or_warmup_rows", reason)); for(int i = 0; i < expected_rows; i++) if(closed[i].server_open <= 0 || closed[i].server_open >= decision_open || (i > 0 && closed[i].server_open <= closed[i - 1].server_open)) return(Fail("study_encoder.tester.closed_input_order", reason)); if(cold) { if(!m_market.Clear()) return(Fail("study_encoder.tester.initial_clear", reason)); //--- CogDriverData consumes the frozen input window. Replay enough overlapping //--- states to fill the frozen recurrent stack; only the final state is published. SDPCCStudyForecastBar window[]; if(ArrayResize(window, DPCC_ENCODER_INPUT_WINDOW) != DPCC_ENCODER_INPUT_WINDOW) return(Fail("study_encoder.tester.warmup_window_allocate", reason)); for(int end = DPCC_ENCODER_INPUT_WINDOW - 1; end < expected_rows; end++) { for(int j = 0; j < DPCC_ENCODER_INPUT_WINDOW; j++) window[j] = closed[end - (DPCC_ENCODER_INPUT_WINDOW - 1) + j]; float state[]; if(!DPCCPackStudyForecastInput(window, DPCC_ENCODER_INPUT_WINDOW, state) || !ForwardInferenceState(state, reason)) return(Fail("study_encoder.tester.causal_warmup:" + (string)(end - DPCC_ENCODER_INPUT_WINDOW + 2) + ":" + reason, reason)); } } else { float state[]; if(!DPCCPackStudyForecastInput(closed, DPCC_ENCODER_INPUT_WINDOW, state) || !ForwardInferenceState(state, reason)) return(Fail("study_encoder.tester.incremental_state:" + reason, reason)); } if(!PublishZ(z, reason)) return(false); m_inference_split = 2; m_inference_cursor = (int)decision_open; return(true); } bool TesterWarmupRequired(void) const {return(m_inference_split != 2);} //+-------------------------------------------------------------------+ //| PrepareZSnapshot allocates an owned buffer in the Encoder context | //+-------------------------------------------------------------------+ bool PrepareZSnapshot(CBufferFloat *destination, string &reason) { reason = ""; COpenCLMy *context = m_market.GetOpenCL(); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!m_encoder_only || !m_ready || m_failed || !m_restore_only || CheckPointer(context) == POINTER_INVALID || CheckPointer(destination) == POINTER_INVALID || !forecast || m_market.WeightsUpdateEnabled() || forecast.CodebookUpdate() || destination == forecast.GetZ() || destination.GetOpenCL() != NULL || destination.GetIndex() >= 0 || destination.Total() != 0) { reason = "study_encoder.snapshot.prepare_owner_context_or_shape"; return(false); } const uint count = 21 * 9 * 12 * 16; if(!destination.BufferInit(count, 0.0f)) { reason = "study_encoder.snapshot.prepare_init"; return(false); } if(!destination.BufferCreate(context)) { reason = "study_encoder.snapshot.prepare_create"; return(false); } return(true); } //+------------------------------------------------------------------+ //| CopyZ snapshots the last admitted Z in the Encoder context | //+------------------------------------------------------------------+ bool CopyZ(CBufferFloat *destination, string &reason) { CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(!m_encoder_only || !m_ready || m_failed || !m_restore_only || !m_z_available || !forecast || !destination || destination.GetOpenCL() != m_market.GetOpenCL() || destination.GetIndex() < 0 || destination.Total() != 21 * 9 * 12 * 16 || destination == forecast.GetZ() || m_market.WeightsUpdateEnabled() || forecast.CodebookUpdate()) { reason = "study_encoder.snapshot.owner_context_or_shape"; return(false); } if(!m_gradient_ops.Copy(forecast.GetZ(), destination, 21 * 9 * 12 * 16)) return(Fail("study_encoder.snapshot.copy", reason)); reason = ""; return(true); } COpenCLMy *EncoderContext(void) { if(!m_encoder_only || !m_ready || m_failed || !m_restore_only) return(NULL); return(m_market.GetOpenCL()); } //+------------------------------------------------------------------+ //| InferLiveZ commits one linked closed observation after Z publish | //+------------------------------------------------------------------+ bool InferLiveZ(const SDPCCClosedHistoryRow &captured, float &z[], string &reason) { //--- Refuse external input before changing the committed stream or native recurrent state. ArrayResize(z, 0); reason = ""; if(!m_encoder_only || !m_ready || m_failed || !m_restore_only) { reason = "study_encoder.live.lifecycle"; return(false); } const long ordinal = (long)ArraySize(m_bars) + m_live_count; if(ordinal >= INT_MAX) { reason = "study_encoder.live.capacity"; return(false); } SDPCCClosedHistoryRow row; if(!DPCCRebaseStudyLiveRow(captured, m_live_next_open, ordinal, row, reason)) return(false); SDPCCStudyForecastBar prior[]; float state_values[], normalization[]; if(!LivePredecessors(m_live_count, prior, reason)) return(false); if(!DPCCPrepareStudyLiveInput(prior, row, m_live_next_open, ordinal, state_values, normalization, reason)) return(false); CNeuronScenarioForecast *forecast = dynamic_cast(m_market.Layer(6)); if(m_market.WeightsUpdateEnabled() || !forecast || forecast.CodebookUpdate()) return(Fail("study_encoder.live.freeze_policy", reason)); //--- Reserve the next committed row before any stateful device operation. if(ArrayResize(m_live_bars, m_live_count + 1) != m_live_count + 1) { reason = "study_encoder.live.row_capacity"; return(false); } if(!PrepareLiveState(reason) || !ForwardInferenceState(state_values, reason) || !PublishZ(z, reason)) { ArrayResize(z, 0); ArrayResize(m_live_bars, m_live_count); return(false); } //--- Publish only after exact output shape, finiteness, and frozen-parameter checks. m_live_bars[m_live_count] = row.bar; m_live_count++; m_live_next_open = row.successor_open_server; m_inference_split = -1; m_inference_cursor = -1; return(true); } long LiveNextOrdinal(void) const { return((long)ArraySize(m_bars) + m_live_count); } long LiveNextServerOpen(void) const { return(m_live_next_open); } int LiveCount(void) const { return(m_live_count); } bool SaveCompleted(string &reason) { return(SaveEncoderCompleted(reason)); } }; //+------------------------------------------------------------------+ //| Rebase the restored Market on pre-start Tester chart history | //+------------------------------------------------------------------+ bool DPCCPrepareReadyMarketLiveStart(CDPCCStudyForecastCore &market, const string symbol, const int &handles[], const long first_closed_open, string &reason) { reason = ""; //--- Validate the first live bar. if(symbol == "" || first_closed_open <= 0 || ArraySize(handles) != 4) { reason = "ready_market.live.start.arguments"; return(false); } const int count = DPCC_ENCODER_WARMUP_HISTORY; const long current_open = (long)iTime(symbol, PERIOD_H1, 0); if(current_open <= first_closed_open || (long)iTime(symbol, PERIOD_H1, 1) != first_closed_open) { reason = "ready_market.live.start.bar_binding"; return(false); } SDPCCClosedHistoryRow history[]; if(ArrayResize(history, count) != count) { reason = "ready_market.live.start.capacity"; return(false); } //--- Capture the available pre-start bars in chronological order. for(int i = 0; i < count; i++) { const int shift = count + 1 - i; if(!DPCCCaptureTesterClosedRow(symbol, handles, shift, history[i], reason)) return(false); const long expected_successor = (i + 1 < count ? (long)iTime(symbol, PERIOD_H1, shift - 1) : first_closed_open); if(history[i].successor_open_server != expected_successor || history[i].bar.server_open >= first_closed_open || (i > 0 && (history[i - 1].bar.server_open >= history[i].bar.server_open || history[i - 1].successor_open_server != history[i].bar.server_open))) { reason = StringFormat("ready_market.live.start.history_link.%d", i); return(false); } } //--- Rebase the restored Market at the first live decision. return(market.RebaseReadyMarketLiveHistory(history, first_closed_open, reason)); } #endif #endif #undef DPCC_TRAJECTORY_INCLUDE_STUDY_FORECAST_CORE //+------------------------------------------------------------------+ //| Shared central model decoding for Study and Trading restoration | //+------------------------------------------------------------------+ struct SDPCCTrainingModelBinding { uint price_basis; float action_bias[6], action_scale[6], probability_tolerance; double volume_scale, price_scale, money_scale, schedule_s, beta_cap; double beta[], alpha_bar[], sigma[]; }; //+------------------------------------------------------------------+ //| Function DPCCRecordNonzeroMantissa | //+------------------------------------------------------------------+ bool DPCCRecordNonzeroMantissa(const string lexeme) { for(int i = 0; i < StringLen(lexeme); i++) { const ushort ch = StringGetCharacter(lexeme, i); if(ch == 'e' || ch == 'E') break; if(ch >= '1' && ch <= '9') return(true); } return(false); } //+------------------------------------------------------------------+ //| Function DPCCRecordFloatArray | //+------------------------------------------------------------------+ bool DPCCRecordFloatArray(CDPCCJsonDocument &document, const int node, const uint count, float &values[], const string path, string &reason, const int domain = 0, const double tolerance = 0) { ArrayResize(values, 0); if(document.Type(node) != DPCC_JSON_ARRAY || document.ChildCount(node) != (int)count || ArrayResize(values, (int)count) != (int)count) {reason = path + ".length"; return(false);} double original_sum = 0, converted_sum = 0; for(uint i = 0; i < count; i++) { double number = 0; if(!document.TryNumber(document.ChildAt(node, (int)i), number) || !MathIsValidNumber((float)number)) {reason = StringFormat("%s[%u].float32", path, i); return(false);} if(number == 0 && DPCCRecordNonzeroMantissa(document.Text(document.ChildAt(node, (int)i)))) {reason = StringFormat("%s[%u].double_underflow", path, i); return(false);} const float converted = (float)number; if((domain == 1 && (number <= 0 || converted <= 0)) || (domain == 2 && (number < 0 || converted < 0))) {reason = StringFormat("%s[%u].domain", path, i); return(false);} values[i] = converted; original_sum += number; converted_sum += (double)converted; } if(domain == 2 && (!MathIsValidNumber(original_sum) || !MathIsValidNumber(converted_sum) || MathAbs(original_sum - 1) > tolerance || MathAbs(converted_sum - 1) > tolerance)) {reason = path + ".simplex"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodePreparedActorConfig | //+------------------------------------------------------------------+ bool DPCCDecodePreparedActorConfig(const string source, const SDPCCConfig &training, SDPCCConfig &actor, float &limits[], string &reason) { reason = ""; CDPCCJsonDocument d; if(!d.Parse(source, reason)) return(false); actor = training; actor.shape.PlanBatch = actor.shape.M; actor.shape.Depth = 0; actor.mode = DPCC_P0; const int root = d.Root(), projection = d.FindMember(root, "projection"), risk = d.FindMember(root, "risk"), price = d.FindMember(root, "price_contract"), execution = d.FindMember(root, "execution_contract"), diffusion = d.FindMember(root, "diffusion"), bound = d.FindMember(risk, "limits"); long iterations = 0, evaluations = 0, lookback = 0; string name = "", quote = "", event = "", randomness = ""; double parameter = 0, mean = 0, gamma = 0; if(!DPCCResourceReadLong(d, projection, "max_iterations", iterations, reason, "actor.iterations") || !DPCCResourceReadLong(d, projection, "max_evaluations", evaluations, reason, "actor.evaluations") || !DPCCResourceReadLong(d, projection, "lookback_steps", lookback, reason, "actor.lookback") || iterations < 1 || iterations > UINT_MAX || evaluations < 1 || evaluations > UINT_MAX || lookback < 1 || lookback > actor.shape.H || !DPCCResourceReadDouble(d, projection, "improvement_atol", actor.projection_improvement_atol, reason, "actor.improvement") || !DPCCResourceReadDouble(d, projection, "distance_tie_atol", actor.projection_distance_tie_atol, reason, "actor.tie") || !DPCCResourceReadString(d, risk, "type", name, reason, "actor.risk") || !DPCCResourceReadDouble(d, risk, "parameter", parameter, reason, "actor.parameter") || !DPCCResourceReadDouble(d, risk, "mean_weight", mean, reason, "actor.mean") || !DPCCResourceReadDouble(d, risk, "gamma", gamma, reason, "actor.gamma") || !DPCCResourceReadString(d, price, "quote_adapter", quote, reason, "actor.quote") || !DPCCResourceReadString(d, execution, "event_model", event, reason, "actor.event") || event != "LOCAL_TWO_PATH_V1" || !DPCCResourceReadString(d, diffusion, "randomness_version", randomness, reason, "actor.randomness") || randomness != "INDEXED_NORMAL_V1" || !DPCCResourceReadDouble(d, price, "max_quote_age_seconds", actor.max_quote_age_seconds, reason, "actor.quote_age") || !DPCCResourceReadDouble(d, price, "max_last_age_seconds", actor.max_last_age_seconds, reason, "actor.last_age")) {if(StringLen(reason) == 0)reason = "actor.model_operational_profile"; return(false);} if(quote == "BID_IDENTITY_V1" && actor.price_basis == DPCC_PRICE_BID) actor.quote_adapter = DPCC_QUOTE_BID_IDENTITY_V1; else if(quote == "LAST_FROZEN_OFFSET_V1" && actor.price_basis == DPCC_PRICE_LAST) actor.quote_adapter = DPCC_QUOTE_LAST_FROZEN_OFFSET_V1; else { reason = "actor.model_quote_profile"; return(false); } actor.projection_max_iterations = (uint)iterations; actor.projection_max_evaluations = (uint)evaluations; actor.projection_lookback_steps = (uint)lookback; if(name == "MEAN") actor.risk_type = defSRM_MEAN; else if(name == "CVAR") actor.risk_type = defSRM_CVAR; else if(name == "MEAN_CVAR") actor.risk_type = defSRM_MEAN_CVAR; else if(name == "EXPONENTIAL") actor.risk_type = defSRM_EXPONENTIAL; else if(name == "DUAL_POWER") actor.risk_type = defSRM_DUAL_POWER; else { reason = "actor.model_risk_type"; return(false); } actor.risk_parameter = (float)parameter; actor.mean_weight = (float)mean; actor.gamma = (float)gamma; actor.event_model = DPCC_EVENT_LOCAL_TWO_PATH_V1; actor.randomness_version = DPCC_RANDOM_INDEXED_NORMAL_V1; //--- Missing extension denotes exactly the old pinned financial profile. //--- Partial explicit metadata cannot upgrade or silently downgrade a Model. const int margin_model = d.FindMember(execution, "margin_fx_model"), calc_version = d.FindMember(execution, "calc_contract_version"), money_digits = d.FindMember(execution, "account_currency_digits"), round_model = d.FindMember(execution, "margin_rounding_model"); actor.margin_fx_profile = DPCC_MARGIN_EXTERNAL_V2; actor.account_currency_digits = 0; if(margin_model >= 0 || calc_version >= 0 || money_digits >= 0 || round_model >= 0) { string margin_name = "", round_name = ""; long version = 0, digits = 0; if(!DPCCResourceReadString(d, execution, "margin_fx_model", margin_name, reason, "actor.margin_model") || !DPCCResourceReadLong(d, execution, "calc_contract_version", version, reason, "actor.calc_version") || !DPCCResourceReadLong(d, execution, "account_currency_digits", digits, reason, "actor.currency_digits") || !DPCCResourceReadString(d, execution, "margin_rounding_model", round_name, reason, "actor.margin_rounding")) return(false); if(margin_name == "EXTERNAL_CURRENCY_V2" && version == 2 && digits == 0 && round_name == "LEGACY_UNROUNDED_V2") actor.margin_fx_profile = DPCC_MARGIN_EXTERNAL_V2; else if(margin_name == "FOREX_BASE_MARGIN_ACCOUNT_PROFIT_SPOT_V1" && version == 3 && digits >= 0 && digits <= 8 && round_name == "ACCOUNT_CURRENCY_NEAREST_BOUNDED_FP32_V1") {actor.margin_fx_profile = DPCC_MARGIN_FOREX_SPOT_V1; actor.account_currency_digits = (uint)digits;} else if(margin_name == "HISTORICAL_CLOSE_FX_V1" && version == 4 && digits >= 0 && digits <= 8 && round_name == "ACCOUNT_CURRENCY_NEAREST_BOUNDED_FP32_V1") {actor.margin_fx_profile = DPCC_MARGIN_HISTORICAL_CLOSE_V1; actor.account_currency_digits = (uint)digits;} else { reason = "actor.model_financial_profile"; return(false); } } //--- Reuse exactly the native DPCC/FQF admission, including risk-specific domains. if((actor.risk_type == defSRM_DUAL_POWER && parameter < 1) || !DPCCValidateFixedConfig(actor.shape, actor)) { reason = "actor.model_native_config"; return(false); } const string names[] = {"max_abs_lots", "min_free_margin", "max_margin_fraction", "min_equity", "max_loss_from_snapshot", "max_drawdown_cash", "max_daily_loss", "min_sl_points", "max_sl_points", "min_tp_points", "max_tp_points", "free_margin_reserve", "loss_reserve", "extra_slippage_points", "volume_atol", "price_atol", "money_atol", "money_rtol", "probability_atol", "max_price_jump_diagnostic", "active_mask", "risk_scale_cash", "limit_version", "reserved_zero" }; if(ArrayResize(limits, 24) != 24) { reason = "actor.limits_capacity"; return(false); } for(int i = 0; i < 24; i++) { double value = 0; if(!DPCCResourceReadDouble(d, bound, names[i], value, reason, "actor.limit")) return(false); limits[i] = (float)value; if(!MathIsValidNumber(limits[i])) { reason = "actor.limit_float"; return(false); } } return(true); } //+------------------------------------------------------------------+ //| Function DPCCRunModelValues | //+------------------------------------------------------------------+ bool DPCCRunModelValues(CDPCCJsonDocument &d, const SDPCCResourceShape &shape, SDPCCTrainingModelBinding &binding, string &reason) { const int root = d.Root(), norm = d.FindMember(root, "normalization"), risk = d.FindMember(root, "risk"), scales = d.FindMember(risk, "technical_scales"), limits = d.FindMember(risk, "limits"), diff = d.FindMember(root, "diffusion"); string basis; const int price = d.FindMember(root, "price_contract"); if(!DPCCResourceReadString(d, price, "chart_price_basis", basis, reason, "model.chart_price_basis")) return(false); if(basis == "BID") binding.price_basis = DPCC_PRICE_BID; else if(basis == "LAST") binding.price_basis = DPCC_PRICE_LAST; else { reason = "training.model_price_basis"; return(false); } float bias[], scale[], gain[], nbias[]; if(!DPCCRecordFloatArray(d, d.FindMember(norm, "action_bias"), 6, bias, "model.action_bias", reason) || !DPCCRecordFloatArray(d, d.FindMember(norm, "action_scale"), 6, scale, "model.action_scale", reason, 1) || !DPCCRecordFloatArray(d, d.FindMember(norm, "gain"), 4, gain, "model.gain", reason) || !DPCCRecordFloatArray(d, d.FindMember(norm, "bias"), 4, nbias, "model.bias", reason)) return(false); for(int i = 0; i < 4; i++) if(gain[i] != 1 || nbias[i] != 0) { reason = "training.revin_gain1"; return(false); } for(int i = 0; i < 6; i++) { binding.action_bias[i] = bias[i]; binding.action_scale[i] = scale[i]; } double tolerance; if(!DPCCResourceReadDouble(d, scales, "volume_scale", binding.volume_scale, reason, "model.volume_scale") || !DPCCResourceReadDouble(d, scales, "price_scale", binding.price_scale, reason, "model.price_scale") || !DPCCResourceReadDouble(d, scales, "money_scale", binding.money_scale, reason, "model.money_scale") || !DPCCResourceReadDouble(d, limits, "probability_atol", tolerance, reason, "model.probability_atol") || !DPCCResourceReadDouble(d, diff, "s", binding.schedule_s, reason, "model.diffusion.s") || !DPCCResourceReadDouble(d, diff, "beta_cap", binding.beta_cap, reason, "model.diffusion.beta_cap")) return(false); binding.probability_tolerance = (float)tolerance; const int count = (int)shape.J + 1; if(shape.J < 1 || shape.J > 64 || ArrayResize(binding.beta, count) != count || ArrayResize(binding.alpha_bar, count) != count || ArrayResize(binding.sigma, count) != count) { reason = "training.model_schedule_capacity"; return(false); } int bn = d.FindMember(diff, "beta"), an = d.FindMember(diff, "alpha_bar"), sn = d.FindMember(diff, "sigma"); if(d.ChildCount(bn) != count || d.ChildCount(an) != count || d.ChildCount(sn) != count) { reason = "training.model_schedule_length"; return(false); } for(int i = 0; i < count; i++) if(!d.TryNumber(d.ChildAt(bn, i), binding.beta[i]) || !d.TryNumber(d.ChildAt(an, i), binding.alpha_bar[i]) || !d.TryNumber(d.ChildAt(sn, i), binding.sigma[i])) { reason = "training.model_schedule_numeric"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Decode the same central Action6 model for training and | //| restoration | //+------------------------------------------------------------------+ string DPCCAction6RuntimeDoubleArray(const double &values[]) { string json = "["; for(int i = 0; i < ArraySize(values); i++) { if(!MathIsValidNumber(values[i])) return(""); json += (i == 0 ? "" : ",") + StringFormat("%.17g", values[i]); } return(json + "]"); } //+------------------------------------------------------------------+ //| Function DPCCAction6RuntimeFloatArray | //+------------------------------------------------------------------+ string DPCCAction6RuntimeFloatArray(const float &values[]) { string json = "["; for(int i = 0; i < ArraySize(values); i++) { if(!MathIsValidNumber((double)values[i])) return(""); json += (i == 0 ? "" : ",") + StringFormat("%.9g", (double)values[i]); } return(json + "]"); } //+------------------------------------------------------------------+ //| Function DPCCAction6RuntimeReadArray | //+------------------------------------------------------------------+ bool DPCCAction6RuntimeReadArray(CDPCCJsonDocument &document, const int root, const string key, const int expected, double &values[], string &reason) { const int node = document.FindMember(root, key); if(node < 0 || document.Type(node) != DPCC_JSON_ARRAY || document.ChildCount(node) != expected || ArrayResize(values, expected) != expected) {reason = "action6.runtime." + key + ".shape"; return(false);} for(int i = 0; i < expected; i++) if(!document.TryNumber(document.ChildAt(node, i), values[i]) || !MathIsValidNumber(values[i])) {reason = StringFormat("action6.runtime.%s[%d].number", key, i); return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCEncodeAction6ModelRuntime | //+------------------------------------------------------------------+ bool DPCCEncodeAction6ModelRuntime(const SDPCCRuntimeConfig &runtime, const float &limits[], string &source, string &reason) { source = ""; reason = ""; const SDPCCConfig c = runtime.financial; if(ArraySize(limits) != 24 || !DPCCValidateFixedConfig(c.shape, c) || ArraySize(runtime.beta) != (int)c.shape.J + 1 || ArraySize(runtime.alpha_bar) != (int)c.shape.J + 1 || ArraySize(runtime.sigma) != (int)c.shape.J + 1) {reason = "action6.runtime.encode.shape"; return(false);} const long integers[] = {c.shape.K, c.shape.V, c.shape.H, c.shape.D, c.shape.M, c.shape.J, c.shape.E, c.shape.PlanBatch, c.shape.NActions, c.shape.StateSize, c.shape.Embedding, c.shape.Heads, c.shape.KeyDimension, c.shape.Depth, c.contract_version, c.binary_layer_version, ( long)c.mode, c.projection_max_iterations, c.projection_max_evaluations, c.projection_lookback_steps, c.price_basis, c.quote_adapter, c.event_model, c.risk_type, c.normalization_mode, c.loss_coordinates, c.randomness_version, c.margin_fx_profile, c.account_currency_digits }; const double reals[] = {c.projection_improvement_atol, c.projection_distance_tie_atol, c.volume_scale, c.price_scale, c.money_scale, c.max_quote_age_seconds, c.max_last_age_seconds }; const double scalars[] = {(double)c.risk_parameter, (double)c.mean_weight, (double)c.gamma, ( double)c.probability_tolerance}; string integer_json = "[", real_json = "[", scalar_json = "["; for(int i = 0; i < ArraySize(integers); i++) integer_json += (i == 0 ? "" : ",") + (string)integers[i]; for(int i = 0; i < ArraySize(reals); i++) { if(!MathIsValidNumber(reals[i])) { reason = "action6.runtime.encode.real"; return(false); } real_json += (i == 0 ? "" : ",") + StringFormat("%.17g", reals[i]); } for(int i = 0; i < ArraySize(scalars); i++) { if(!MathIsValidNumber(scalars[i])) { reason = "action6.runtime.encode.scalar"; return(false); } scalar_json += (i == 0 ? "" : ",") + StringFormat("%.9g", scalars[i]); } integer_json += "]"; real_json += "]"; scalar_json += "]"; string bias_json = "[", scale_json = "["; for(int i = 0; i < 6; i++) { if(!MathIsValidNumber((double)c.action_bias[i]) || !MathIsValidNumber((double)c.action_scale[i])) {reason = "action6.runtime.encode.action"; return(false);} bias_json += (i == 0 ? "" : ",") + StringFormat("%.9g", (double)c.action_bias[i]); scale_json += (i == 0 ? "" : ",") + StringFormat("%.9g", (double)c.action_scale[i]); } bias_json += "]"; scale_json += "]"; const string beta_json = DPCCAction6RuntimeDoubleArray(runtime.beta); const string alpha_json = DPCCAction6RuntimeDoubleArray(runtime.alpha_bar); const string sigma_json = DPCCAction6RuntimeDoubleArray(runtime.sigma); const string limits_json = DPCCAction6RuntimeFloatArray(limits); if(beta_json == "" || alpha_json == "" || sigma_json == "" || limits_json == "") {reason = "action6.runtime.encode.nonfinite"; return(false);} source = "{\"format\":\"DPCC_ACTION6_RUNTIME_V1\",\"integers\":" + integer_json + ",\"reals\":" + real_json + ",\"scalars\":" + scalar_json + ",\"action_bias\":" + bias_json + ",\"action_scale\":" + scale_json + ",\"beta\":" + beta_json + ",\"alpha_bar\":" + alpha_json + ",\"sigma\":" + sigma_json + ",\"limits\":" + limits_json + "}"; return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeAction6RuntimeEnvelope | //+------------------------------------------------------------------+ bool DPCCDecodeAction6RuntimeEnvelope(CDPCCJsonDocument &document, const int root, SDPCCRuntimeConfig &runtime, float &limits[], string &reason) { double integers[], reals[], scalars[], bias[], scale[], beta[], alpha[], sigma[], limit_values[]; if(!DPCCAction6RuntimeReadArray(document, root, "integers", 29, integers, reason) || !DPCCAction6RuntimeReadArray(document, root, "reals", 7, reals, reason) || !DPCCAction6RuntimeReadArray(document, root, "scalars", 4, scalars, reason) || !DPCCAction6RuntimeReadArray(document, root, "action_bias", 6, bias, reason) || !DPCCAction6RuntimeReadArray(document, root, "action_scale", 6, scale, reason) || !DPCCAction6RuntimeReadArray(document, root, "limits", 24, limit_values, reason)) return(false); for(int i = 0; i < 29; i++) if(!MathIsValidNumber(integers[i]) || integers[i] < 0 || MathFloor(integers[i]) != integers[i] || integers[i] > UINT_MAX) {reason = StringFormat("action6.runtime.integer_%d", i); return(false);} if(integers[16] > DPCC_P2 || integers[23] > INT_MAX) {reason = "action6.runtime.integer_domain"; return(false);} SDPCCConfig c; ZeroMemory(c); c.shape.K = (uint)integers[0]; c.shape.V = (uint)integers[1]; c.shape.H = (uint)integers[2]; c.shape.D = (uint)integers[3]; c.shape.M = (uint)integers[4]; c.shape.J = (uint)integers[5]; c.shape.E = (uint)integers[6]; c.shape.PlanBatch = (uint)integers[7]; c.shape.NActions = (uint)integers[8]; c.shape.StateSize = (uint)integers[9]; c.shape.Embedding = (uint)integers[10]; c.shape.Heads = (uint)integers[11]; c.shape.KeyDimension = (uint)integers[12]; c.shape.Depth = (uint)integers[13]; c.contract_version = (uint)integers[14]; c.binary_layer_version = (uint)integers[15]; c.mode = (ENUM_DPCC_MODE)(int)integers[16]; c.projection_max_iterations = (uint)integers[17]; c.projection_max_evaluations = (uint)integers[18]; c.projection_lookback_steps = (uint)integers[19]; c.price_basis = (uint)integers[20]; c.quote_adapter = (uint)integers[21]; c.event_model = (uint)integers[22]; c.risk_type = (int)integers[23]; c.normalization_mode = (uint)integers[24]; c.loss_coordinates = (uint)integers[25]; c.randomness_version = (uint)integers[26]; c.margin_fx_profile = (uint)integers[27]; c.account_currency_digits = (uint)integers[28]; for(int i = 0; i < 7; i++) if(!MathIsValidNumber(reals[i])) { reason = StringFormat("action6.runtime.real_%d", i); return(false); } for(int i = 0; i < 4; i++) if(!MathIsValidNumber(scalars[i]) || !MathIsValidNumber((float)scalars[i])) {reason = StringFormat("action6.runtime.scalar_%d", i); return(false);} c.projection_improvement_atol = reals[0]; c.projection_distance_tie_atol = reals[1]; c.volume_scale = reals[2]; c.price_scale = reals[3]; c.money_scale = reals[4]; c.max_quote_age_seconds = reals[5]; c.max_last_age_seconds = reals[6]; c.risk_parameter = (float)scalars[0]; c.mean_weight = (float)scalars[1]; c.gamma = (float)scalars[2]; c.probability_tolerance = (float)scalars[3]; for(int i = 0; i < 6; i++) { c.action_bias[i] = (float)bias[i]; c.action_scale[i] = (float)scale[i]; if(!MathIsValidNumber((double)c.action_bias[i]) || !MathIsValidNumber((double)c.action_scale[i])) {reason = "action6.runtime.action_float32"; return(false);} } if(!DPCCValidateFixedConfig(c.shape, c) || ArrayResize(limits, 24) != 24) {reason = "action6.runtime.config_or_limits"; return(false);} for(int i = 0; i < 24; i++) { if(!MathIsValidNumber(limit_values[i]) || !MathIsValidNumber((float)limit_values[i])) {reason = StringFormat("action6.runtime.limit_%d", i); return(false);} limits[i] = (float)limit_values[i]; } if(!DPCCAction6RuntimeReadArray(document, root, "beta", (int)c.shape.J + 1, beta, reason) || !DPCCAction6RuntimeReadArray(document, root, "alpha_bar", (int)c.shape.J + 1, alpha, reason) || !DPCCAction6RuntimeReadArray(document, root, "sigma", (int)c.shape.J + 1, sigma, reason)) return(false); if(ArrayResize(runtime.beta, ArraySize(beta)) != ArraySize(beta) || ArrayResize(runtime.alpha_bar, ArraySize(alpha)) != ArraySize(alpha) || ArrayResize(runtime.sigma, ArraySize(sigma)) != ArraySize(sigma)) {reason = "action6.runtime.schedule_capacity"; return(false);} for(int i = 0; i < ArraySize(beta); i++) { if(beta[i] < 0 || beta[i] >= 1 || alpha[i] <= 0 || alpha[i] > 1 || sigma[i] < 0) {reason = StringFormat("action6.runtime.schedule_%d", i); return(false);} runtime.beta[i] = beta[i]; runtime.alpha_bar[i] = alpha[i]; runtime.sigma[i] = sigma[i]; } runtime.financial = c; ArrayInitialize(runtime.condition_scale, 1.0f); ArrayInitialize(runtime.condition_bias, 0.0f); reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeAction6ModelRuntime | //+------------------------------------------------------------------+ bool DPCCDecodeAction6ModelRuntime(const string model_source, SDPCCRuntimeConfig &runtime, float &limits[], string &reason) { ZeroMemory(runtime); ArrayResize(limits, 0); CDPCCJsonDocument model_document; string parse_reason = ""; if(model_document.Parse(model_source, parse_reason)) { const int format = model_document.FindMember(model_document.Root(), "format"); if(format >= 0 && model_document.Type(format) == DPCC_JSON_STRING && model_document.Text(format) == "DPCC_ACTION6_RUNTIME_V1") return(DPCCDecodeAction6RuntimeEnvelope(model_document, model_document.Root(), runtime, limits, reason)); } reason = ""; SDPCCCentralModel model; SDPCCTrainingModelBinding binding; if(!model_document.Parse(model_source, reason) || !DPCCDecodeCentralModel(model_document, model, reason) || model.example_only || model.shapes.K != 21 || model.shapes.V != 9 || model.shapes.H != 12 || model.shapes.D != 16 || model.shapes.M < 1 || model.shapes.M > 64 || model.shapes.J < 1 || model.shapes.J > 64 || model.shapes.E < 1 || model.shapes.E > UINT_MAX || !DPCCRunModelValues(model_document, model.shapes, binding, reason)) {reason = "action6.central_model_profile:" + reason; return(false);} SDPCCConfig training; ZeroMemory(training); training.shape.K = 21; training.shape.V = 9; training.shape.H = 12; training.shape.D = 4; training.shape.M = (uint)model.shapes.M; training.shape.J = (uint)model.shapes.J; training.shape.E = (uint)model.shapes.E; training.shape.PlanBatch = 1; training.shape.NActions = 6; training.shape.StateSize = 13; training.shape.Embedding = 16; training.shape.Heads = 1; training.shape.KeyDimension = 16; training.shape.Depth = 0; training.contract_version = 2; training.binary_layer_version = 3; training.normalization_mode = DPCC_NORMALIZATION_CAUSAL_AFFINE_V2; training.loss_coordinates = DPCC_LOSS_NORMALIZED; training.price_basis = binding.price_basis; training.volume_scale = binding.volume_scale; training.price_scale = binding.price_scale; training.money_scale = binding.money_scale; training.probability_tolerance = binding.probability_tolerance; for(int i = 0; i < 6; i++) { training.action_bias[i] = binding.action_bias[i]; training.action_scale[i] = binding.action_scale[i]; } if(!DPCCDecodePreparedActorConfig(model_source, training, runtime.financial, limits, reason) || !DPCCBuildCosineDiffusionSchedule(training.shape.J, binding.schedule_s, binding.beta_cap, runtime.beta, runtime.alpha_bar, runtime.sigma, reason)) return(false); for(int i = 0; i < ArraySize(runtime.beta); i++) if(runtime.beta[i] != binding.beta[i] || runtime.alpha_bar[i] != binding.alpha_bar[i] || runtime.sigma[i] != binding.sigma[i]) {reason = "action6.central_model_schedule_mismatch"; return(false);} ArrayInitialize(runtime.condition_scale, 1.0f); ArrayInitialize(runtime.condition_bias, 0.0f); return(true); } //+------------------------------------------------------------------+ //| Build the existing Action6 runtime from EA inputs and terminal | //+------------------------------------------------------------------+ bool DPCCBuildAction6RuntimeFromInputs(const ENUM_DPCC_MODE mode, const ulong noise_seed, const double schedule_s, const double beta_cap, const int risk_type, const double risk_parameter, const double mean_weight, const double gamma, const uint projection_iterations, const uint projection_evaluations, const uint projection_lookback, const double improvement_atol, const double distance_tie_atol, const double min_equity, const double max_sl_points, const double max_tp_points, SDPCCRuntimeConfig &runtime, float &limits[], string &reason) { ZeroMemory(runtime); ArrayResize(limits, 0); reason = ""; const double input_numbers[] = {schedule_s, beta_cap, risk_parameter, mean_weight, gamma, improvement_atol, distance_tie_atol, min_equity, max_sl_points, max_tp_points }; for(int i = 0; i < ArraySize(input_numbers); i++) if(!MathIsValidNumber(input_numbers[i]) || !MathIsValidNumber((float)input_numbers[i])) { reason = "action6.inputs.nonfinite"; return(false); } if(min_equity < 0 || max_sl_points <= 0 || max_tp_points <= 0 || (risk_type == defSRM_DUAL_POWER && risk_parameter < 1)) { reason = "action6.inputs.risk_or_action_scale"; return(false); } SDPCCConfig config; ZeroMemory(config); //--- These dimensions are the existing Action6 architecture, not a data restriction. config.shape.K = 21; config.shape.V = 9; config.shape.H = 12; config.shape.D = 4; config.shape.M = 16; config.shape.J = 32; config.shape.E = DPCC_HISTORY_CALENDAR_MIN_EVENTS; config.shape.PlanBatch = config.shape.M; config.shape.NActions = 6; config.shape.StateSize = 13; config.shape.Embedding = 16; config.shape.Heads = 1; config.shape.KeyDimension = 16; config.shape.Depth = 0; config.contract_version = DPCC_CONTRACT_VERSION; config.binary_layer_version = DPCC_BINARY_LAYER_VERSION; config.mode = mode; config.projection_max_iterations = projection_iterations; config.projection_max_evaluations = projection_evaluations; config.projection_lookback_steps = projection_lookback; config.projection_improvement_atol = improvement_atol; config.projection_distance_tie_atol = distance_tie_atol; config.volume_scale = 1.0; config.price_scale = 1.0; config.money_scale = 1000.0; config.risk_type = risk_type; config.risk_parameter = (float)risk_parameter; config.mean_weight = (float)mean_weight; config.gamma = (float)gamma; config.probability_tolerance = 0.00001f; config.event_model = DPCC_EVENT_LOCAL_TWO_PATH_V1; config.normalization_mode = DPCC_NORMALIZATION_CAUSAL_AFFINE_V2; config.loss_coordinates = DPCC_LOSS_NORMALIZED; config.randomness_version = DPCC_RANDOM_INDEXED_NORMAL_V1; for(int i = 0; i < 6; i++) config.action_scale[i] = 1.0f; long chart_mode = 0, calculation_mode = 0; if(!SymbolInfoInteger(_Symbol, SYMBOL_CHART_MODE, chart_mode) || !SymbolInfoInteger(_Symbol, SYMBOL_TRADE_CALC_MODE, calculation_mode)) { reason = "action6.inputs.symbol_properties"; return(false); } config.price_basis = (chart_mode == SYMBOL_CHART_MODE_LAST ? DPCC_PRICE_LAST : DPCC_PRICE_BID); config.quote_adapter = config.price_basis; config.max_quote_age_seconds = 3600.0; config.max_last_age_seconds = 3600.0; const bool forex = (calculation_mode == SYMBOL_CALC_MODE_FOREX || calculation_mode == SYMBOL_CALC_MODE_FOREX_NO_LEVERAGE); config.margin_fx_profile = (forex && config.price_basis == DPCC_PRICE_BID ? DPCC_MARGIN_FOREX_SPOT_V1 : DPCC_MARGIN_EXTERNAL_V2); config.account_currency_digits = (config.margin_fx_profile == DPCC_MARGIN_EXTERNAL_V2 ? 0 : (uint)AccountInfoInteger(ACCOUNT_CURRENCY_DIGITS)); if(!DPCCValidateFixedConfig(config.shape, config) || !DPCCBuildCosineDiffusionSchedule(config.shape.J, schedule_s, beta_cap, runtime.beta, runtime.alpha_bar, runtime.sigma, reason)) { if(reason == "") reason = "action6.inputs.native_runtime"; return(false); } //--- Preserve the former 24-slot direct-lot profile and its numeric tolerances. const float values[] = {0, 0, 0, (float)min_equity, 0, 0, 0, 0, (float)max_sl_points, 0, (float)max_tp_points, 0, 0, 0, 0.000001f, 0.00001f, 0.01f, 0.00001f, 0.00001f, 0, 8, 1000, 2, 0 }; if(ArrayCopy(limits, values) != 24) { reason = "action6.inputs.limits_capacity"; return(false); } runtime.financial = config; //--- The caller replaces these neutral coordinates with the loaded FIT normalization. ArrayInitialize(runtime.condition_scale, 1.0f); ArrayInitialize(runtime.condition_bias, 0.0f); return(true); } #ifdef DPCC_TRAJECTORY_ENTRY_STUDY #ifndef DPCC_STUDY_STAGE_LOADER_MQH #define DPCC_STUDY_STAGE_LOADER_MQH #ifndef DPCC_STUDY_STAGE_INPUT_MQH #define DPCC_STUDY_STAGE_INPUT_MQH #ifndef DPCC_EPOCH_OWNER_MQH #define DPCC_EPOCH_OWNER_MQH #ifndef DPCC_RUN_PLAN_MQH #define DPCC_RUN_PLAN_MQH #ifndef DPCC_TRAINING_MQH #define DPCC_TRAINING_MQH //--- Canonical destination uses Model.mqh and BankStaging.mqh in this directory. #ifndef DPCC_MODEL_MQH #define DPCC_MODEL_MQH //--- Bounded deployment preparation, never a readiness gate. #endif #ifndef DPCC_BANK_STAGING_CANDIDATE_MQH #define DPCC_BANK_STAGING_CANDIDATE_MQH //--- WORK CANDIDATE. Includes resolve from the canonical consumer directory. #ifndef DPCC_BANK_READER_MQH #define DPCC_BANK_READER_MQH #define DPCC_BANK_PREFIX_BYTES 20 #define DPCC_BANK_FOOTER_BYTES 48 #define DPCC_BANK_RECORD_FRAME_BYTES 36 #define DPCC_BANK_DIGEST_BYTES DPCC_SHA256_DIGEST_BYTES #define DPCC_BANK_MAX_JSON_BYTES 4194304 #define DPCC_BANK_STREAM_CHUNK_BYTES 65536 struct SDPCCBankIntegrityResult { uchar header_bytes[]; int header_length; string header_sha256; string prefix_sha256; string bank_sha256; ulong file_bytes; ulong record_count; }; //+------------------------------------------------------------------+ //| Function DPCCBankIntegrityClear | //+------------------------------------------------------------------+ void DPCCBankIntegrityClear(SDPCCBankIntegrityResult &result) { ArrayResize(result.header_bytes, 0); result.header_length = 0; result.header_sha256 = ""; result.prefix_sha256 = ""; result.bank_sha256 = ""; result.file_bytes = 0; result.record_count = 0; } //+------------------------------------------------------------------+ //| Function DPCCBankIntegrityFail | //+------------------------------------------------------------------+ bool DPCCBankIntegrityFail(SDPCCBankIntegrityResult &result, string &reason, const string code, const int handle = INVALID_HANDLE) { if(handle != INVALID_HANDLE) FileClose(handle); DPCCBankIntegrityClear(result); reason = code; return(false); } //+------------------------------------------------------------------+ //| Function DPCCBankExpectedSHAValid | //+------------------------------------------------------------------+ bool DPCCBankExpectedSHAValid(const string digest) { if(StringLen(digest) != 64) return(false); for(int i = 0; i < 64; i++) { const ushort ch = StringGetCharacter(digest, i); if(!((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'))) return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankReadUnsignedLE | //+------------------------------------------------------------------+ ulong DPCCBankReadUnsignedLE(const uchar &bytes[], const int offset, const int width) { ulong value = 0; for(int i = 0; i < width; i++) value |= ((ulong)bytes[offset + i] << (i * 8)); return(value); } //+------------------------------------------------------------------+ //| Function DPCCBankRawDigestHex | //+------------------------------------------------------------------+ string DPCCBankRawDigestHex(const uchar &bytes[], const int offset, const int length) { const string alphabet = "0123456789abcdef"; string hex = ""; for(int i = 0; i < length; i++) { hex += StringSubstr(alphabet, (int)(bytes[offset + i] >> 4), 1); hex += StringSubstr(alphabet, (int)(bytes[offset + i] & 0x0F), 1); } return(hex); } //+------------------------------------------------------------------+ //| Function DPCCBankUpdate | //+------------------------------------------------------------------+ bool DPCCBankUpdate(CDPCCIncrementalSHA256 &hash, const uchar &bytes[], const int count, string &reason) { return(hash.Update(bytes, 0, count, reason)); } //+------------------------------------------------------------------+ //| Function DPCCBankReadPrefixRegion | //+------------------------------------------------------------------+ bool DPCCBankReadPrefixRegion(const int handle, uchar &scratch[], const ulong length, CDPCCIncrementalSHA256 &whole_hash, CDPCCIncrementalSHA256 &prefix_hash, ulong &position, string &reason) { ulong remaining = length; while(remaining > 0) { const int requested = (int)MathMin((double)ArraySize(scratch), (double)remaining); ResetLastError(); const uint read = FileReadArray(handle, scratch, 0, requested); if(read != (uint)requested) { reason = StringFormat("bank.read.short.%u.%d", read, GetLastError()); return(false); } if(!DPCCBankUpdate(whole_hash, scratch, requested, reason) || !DPCCBankUpdate(prefix_hash, scratch, requested, reason)) return(false); position += (ulong)requested; remaining -= (ulong)requested; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankReadHeader | //+------------------------------------------------------------------+ bool DPCCBankReadHeader(const int handle, uchar &scratch[], uchar &header[], const int length, CDPCCIncrementalSHA256 &whole_hash, CDPCCIncrementalSHA256 &prefix_hash, CDPCCIncrementalSHA256 &header_hash, ulong &position, string &reason) { if(ArrayResize(header, length) != length) { reason = "bank.header.capacity"; return(false); } int copied = 0; while(copied < length) { const int requested = MathMin(ArraySize(scratch), length - copied); ResetLastError(); const uint read = FileReadArray(handle, scratch, 0, requested); if(read != (uint)requested) { reason = StringFormat("bank.header.short.%u.%d", read, GetLastError()); return(false); } if(!DPCCBankUpdate(whole_hash, scratch, requested, reason) || !DPCCBankUpdate(prefix_hash, scratch, requested, reason) || !DPCCBankUpdate(header_hash, scratch, requested, reason)) return(false); if(ArrayCopy(header, scratch, copied, 0, requested) != requested) { reason = "bank.header.copy"; return(false); } copied += requested; position += (ulong)requested; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankReadRecordPayload | //+------------------------------------------------------------------+ bool DPCCBankReadRecordPayload(const int handle, uchar &scratch[], const ulong length, CDPCCIncrementalSHA256 &whole_hash, CDPCCIncrementalSHA256 &prefix_hash, CDPCCIncrementalSHA256 &record_hash, ulong &position, uchar &captured_payload[], const bool capture_payload, string &reason) { if(capture_payload && ArrayResize(captured_payload, (int)length) != (int)length) { reason = "bank.record.capture.capacity"; return(false); } ulong copied = 0; ulong remaining = length; while(remaining > 0) { const int requested = (int)MathMin((double)ArraySize(scratch), (double)remaining); ResetLastError(); const uint read = FileReadArray(handle, scratch, 0, requested); if(read != (uint)requested) { reason = StringFormat("bank.record.short.%u.%d", read, GetLastError()); return(false); } if(!DPCCBankUpdate(whole_hash, scratch, requested, reason) || !DPCCBankUpdate(prefix_hash, scratch, requested, reason) || !DPCCBankUpdate(record_hash, scratch, requested, reason)) return(false); if(capture_payload && ArrayCopy(captured_payload, scratch, (int)copied, 0, requested) != requested) { reason = "bank.record.capture.copy"; return(false); } copied += (ulong)requested; position += (ulong)requested; remaining -= (ulong)requested; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankVerifyFooterAtEOF | //+------------------------------------------------------------------+ bool DPCCBankVerifyFooterAtEOF(const int handle, const ulong footer_start, const uchar &expected_footer[], string &reason) { if(!FileSeek(handle, (long)footer_start, SEEK_SET)) { reason = "bank.footer.eof_seek"; return(false); } uchar actual_footer[]; if(!DPCCBankReadExact(handle, actual_footer, DPCC_BANK_FOOTER_BYTES, reason)) { reason = "bank.footer.eof_read"; return(false); } if(ArraySize(actual_footer) != ArraySize(expected_footer)) { reason = "bank.footer.eof_size"; return(false); } for(int i = 0; i < DPCC_BANK_FOOTER_BYTES; i++) if(actual_footer[i] != expected_footer[i]) { reason = "bank.footer.changed"; return(false); } if(FileTell(handle) != (long)(footer_start + DPCC_BANK_FOOTER_BYTES)) { reason = "bank.file.not_eof"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCBankReadExact | //+------------------------------------------------------------------+ bool DPCCBankReadExact(const int handle, uchar &bytes[], const int count, string &reason) { if(ArrayResize(bytes, count) != count) { reason = "bank.buffer.capacity"; return(false); } ResetLastError(); const uint read = FileReadArray(handle, bytes, 0, count); if(read != (uint)count) { reason = StringFormat("bank.read.fixed_short.%u.%d", read, GetLastError()); return(false); } return(true); } //--- Validation-only contract: implementations may update provisional in-memory state. //--- They must not write externally, submit trades, or make training decisions. class CDPCCBankRecordValidationObserver { public: virtual bool BeginRecord(const ulong record_index, const ulong payload_bytes, string &reason) = 0; virtual bool ConsumeChunk(const ulong record_index, const uchar &chunk[], const int count, string &reason) = 0; virtual bool EndRecord(const ulong record_index, string &reason) = 0; virtual bool Complete(string &reason) = 0; virtual void Abort(void) = 0; }; //+------------------------------------------------------------------+ //| Function DPCCBankObserverPointerValid | //+------------------------------------------------------------------+ bool DPCCBankObserverPointerValid(CDPCCBankRecordValidationObserver *observer) { const ENUM_POINTER_TYPE pointer_type = CheckPointer(observer); return(pointer_type == POINTER_AUTOMATIC || pointer_type == POINTER_DYNAMIC); } //+------------------------------------------------------------------+ //| Function DPCCBankScanHandle | //+------------------------------------------------------------------+ bool DPCCBankScanHandle(const int handle, const string expected_bank_sha256, SDPCCBankIntegrityResult &result, string &reason, CDPCCBankRecordValidationObserver *observer, bool &callback_active, bool &callback_mutation, ulong &session_generation, const ulong scan_generation) { DPCCBankIntegrityClear(result); reason = ""; ResetLastError(); const ulong file_bytes = FileSize(handle); if(GetLastError() != 0) return(DPCCBankIntegrityFail(result, reason, "bank.file.size")); if(file_bytes > (ulong)LONG_MAX) return(DPCCBankIntegrityFail(result, reason, "bank.file.seek_range")); if(file_bytes < (ulong)(DPCC_BANK_PREFIX_BYTES + 1 + DPCC_BANK_FOOTER_BYTES)) return(DPCCBankIntegrityFail(result, reason, "bank.file.too_short")); const ulong footer_start = file_bytes - DPCC_BANK_FOOTER_BYTES; if(!FileSeek(handle, (long)footer_start, SEEK_SET)) return(DPCCBankIntegrityFail(result, reason, "bank.footer.seek")); uchar footer[]; if(!DPCCBankReadExact(handle, footer, DPCC_BANK_FOOTER_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.footer.short_read")); if(footer[0] != 'D' || footer[1] != 'P' || footer[2] != 'C' || footer[3] != 'C' || footer[4] != 'E' || footer[5] != 'N' || footer[6] != 'D' || footer[7] != '2') return(DPCCBankIntegrityFail(result, reason, "bank.footer.magic")); const ulong declared_records = DPCCBankReadUnsignedLE(footer, 8, 8); if(!FileSeek(handle, 0, SEEK_SET)) return(DPCCBankIntegrityFail(result, reason, "bank.file.seek_start")); uchar scratch[]; if(ArrayResize(scratch, DPCC_BANK_STREAM_CHUNK_BYTES) != DPCC_BANK_STREAM_CHUNK_BYTES) return(DPCCBankIntegrityFail(result, reason, "bank.scratch.capacity")); uchar fixed_prefix[]; if(!DPCCBankReadExact(handle, fixed_prefix, DPCC_BANK_PREFIX_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.prefix.short_read")); if(fixed_prefix[0] != 'D' || fixed_prefix[1] != 'P' || fixed_prefix[2] != 'C' || fixed_prefix[3] != 'C' || fixed_prefix[4] != 'B' || fixed_prefix[5] != 'K' || fixed_prefix[6] != '0' || fixed_prefix[7] != '2') return(DPCCBankIntegrityFail(result, reason, "bank.magic")); if(DPCCBankReadUnsignedLE(fixed_prefix, 8, 2) != 2 || DPCCBankReadUnsignedLE(fixed_prefix, 10, 2) != 0) return(DPCCBankIntegrityFail(result, reason, "bank.version")); if(DPCCBankReadUnsignedLE(fixed_prefix, 12, 4) != 0x01020304) return(DPCCBankIntegrityFail(result, reason, "bank.endian")); const ulong header_length = DPCCBankReadUnsignedLE(fixed_prefix, 16, 4); if(header_length == 0) return(DPCCBankIntegrityFail(result, reason, "bank.header.length.zero")); if(header_length > DPCC_BANK_MAX_JSON_BYTES) return(DPCCBankIntegrityFail(result, reason, "bank.header.length.limit")); if((ulong)DPCC_BANK_PREFIX_BYTES + header_length > footer_start) return(DPCCBankIntegrityFail(result, reason, "bank.header.length.file_bounds")); CDPCCIncrementalSHA256 whole_hash, prefix_hash, header_hash; if(!whole_hash.Init(reason) || !prefix_hash.Init(reason) || !header_hash.Init(reason)) return(DPCCBankIntegrityFail(result, reason, "bank.hash.init")); ulong position = 0; if(!DPCCBankUpdate(whole_hash, fixed_prefix, DPCC_BANK_PREFIX_BYTES, reason) || !DPCCBankUpdate(prefix_hash, fixed_prefix, DPCC_BANK_PREFIX_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.hash.prefix")); position = DPCC_BANK_PREFIX_BYTES; uchar header_bytes[]; if(!DPCCBankReadHeader(handle, scratch, header_bytes, (int)header_length, whole_hash, prefix_hash, header_hash, position, reason)) return(DPCCBankIntegrityFail(result, reason, reason)); if(position > footer_start || footer_start - position < DPCC_BANK_RECORD_FRAME_BYTES + 1) { if(declared_records != 0 || position != footer_start) return(DPCCBankIntegrityFail(result, reason, "bank.footer.count.impossible")); } if(declared_records > (footer_start - position) / (DPCC_BANK_RECORD_FRAME_BYTES + 1)) return(DPCCBankIntegrityFail(result, reason, "bank.footer.count.impossible")); ulong actual_records = 0; uchar captured_payload[]; while(position < footer_start) { if(footer_start - position < DPCC_BANK_RECORD_FRAME_BYTES + 1) return(DPCCBankIntegrityFail(result, reason, "bank.record.frame.truncated")); uchar length_bytes[]; if(!DPCCBankReadExact(handle, length_bytes, 4, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.record.length.short")); if(!DPCCBankUpdate(whole_hash, length_bytes, 4, reason) || !DPCCBankUpdate(prefix_hash, length_bytes, 4, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.hash.record_length")); position += 4; const ulong payload_length = DPCCBankReadUnsignedLE(length_bytes, 0, 4); if(payload_length == 0) return(DPCCBankIntegrityFail(result, reason, "bank.record.length.zero")); if(payload_length > DPCC_BANK_MAX_JSON_BYTES) return(DPCCBankIntegrityFail(result, reason, "bank.record.length.limit")); if(payload_length > footer_start - position - DPCC_BANK_DIGEST_BYTES) return(DPCCBankIntegrityFail(result, reason, "bank.record.frame.truncated")); uchar record_digest[]; if(!DPCCBankReadExact(handle, record_digest, DPCC_BANK_DIGEST_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.record.digest.short")); if(!DPCCBankUpdate(whole_hash, record_digest, DPCC_BANK_DIGEST_BYTES, reason) || !DPCCBankUpdate(prefix_hash, record_digest, DPCC_BANK_DIGEST_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.hash.record_digest")); position += DPCC_BANK_DIGEST_BYTES; CDPCCIncrementalSHA256 record_hash; if(!record_hash.Init(reason)) return(DPCCBankIntegrityFail(result, reason, "bank.record.hash.init")); if(!DPCCBankReadRecordPayload(handle, scratch, payload_length, whole_hash, prefix_hash, record_hash, position, captured_payload, observer != NULL, reason)) return(DPCCBankIntegrityFail(result, reason, reason)); uchar record_hash_bytes[]; string record_hash_hex = ""; if(!record_hash.Final(record_hash_bytes, record_hash_hex, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.record.hash.final")); if(record_hash_hex != DPCCBankRawDigestHex(record_digest, 0, DPCC_BANK_DIGEST_BYTES)) return(DPCCBankIntegrityFail(result, reason, "bank.record.digest")); if(observer != NULL) { if(!DPCCBankObserverPointerValid(observer)) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.invalid_pointer")); callback_active = true; const bool begin_ok = observer.BeginRecord(actual_records, payload_length, reason); callback_active = false; if(callback_mutation || session_generation != scan_generation) return(DPCCBankIntegrityFail(result, reason, "bank.session.callback_mutation")); if(!begin_ok) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.begin")); ulong delivered = 0; while(delivered < payload_length) { const int delivery_count = (int)MathMin((double)ArraySize(scratch), (double)(payload_length - delivered)); if(ArrayCopy(scratch, captured_payload, 0, (int)delivered, delivery_count) != delivery_count) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.copy")); if(!DPCCBankObserverPointerValid(observer)) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.invalid_pointer")); callback_active = true; const bool chunk_ok = observer.ConsumeChunk(actual_records, scratch, delivery_count, reason); callback_active = false; if(callback_mutation || session_generation != scan_generation) return(DPCCBankIntegrityFail(result, reason, "bank.session.callback_mutation")); if(!chunk_ok) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.chunk")); delivered += (ulong)delivery_count; } if(!DPCCBankObserverPointerValid(observer)) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.invalid_pointer")); callback_active = true; const bool end_ok = observer.EndRecord(actual_records, reason); callback_active = false; if(callback_mutation || session_generation != scan_generation) return(DPCCBankIntegrityFail(result, reason, "bank.session.callback_mutation")); if(!end_ok) return(DPCCBankIntegrityFail(result, reason, "bank.record.observer.end")); ArrayResize(captured_payload, 0); } actual_records++; } if(position != footer_start) return(DPCCBankIntegrityFail(result, reason, "bank.frame.trailing_or_truncated")); if(actual_records != declared_records) return(DPCCBankIntegrityFail(result, reason, "bank.footer.count.mismatch")); uchar prefix_digest[]; string prefix_hex = ""; if(!prefix_hash.Final(prefix_digest, prefix_hex, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.prefix.hash.final")); if(prefix_hex != DPCCBankRawDigestHex(footer, 16, 32)) return(DPCCBankIntegrityFail(result, reason, "bank.footer.prefix_sha256")); if(!DPCCBankVerifyFooterAtEOF(handle, footer_start, footer, reason)) return(DPCCBankIntegrityFail(result, reason, reason)); if(!whole_hash.Update(footer, 0, DPCC_BANK_FOOTER_BYTES, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.full.hash.footer")); uchar whole_digest[]; string whole_hex = ""; if(!whole_hash.Final(whole_digest, whole_hex, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.full.hash.final")); if(whole_hex != expected_bank_sha256) return(DPCCBankIntegrityFail(result, reason, "bank.file.sha256")); uchar header_digest[]; string header_hex = ""; if(!header_hash.Final(header_digest, header_hex, reason)) return(DPCCBankIntegrityFail(result, reason, "bank.header.hash.final")); if(ArrayResize(result.header_bytes, ArraySize(header_bytes)) != ArraySize(header_bytes)) return(DPCCBankIntegrityFail(result, reason, "bank.result.header.capacity")); if(ArrayCopy(result.header_bytes, header_bytes, 0, 0, ArraySize(header_bytes)) != ArraySize(header_bytes)) return(DPCCBankIntegrityFail(result, reason, "bank.result.header.copy")); result.header_length = (int)header_length; result.header_sha256 = header_hex; result.prefix_sha256 = prefix_hex; result.bank_sha256 = whole_hex; result.file_bytes = file_bytes; result.record_count = actual_records; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Declaration CDPCCBankIntegritySession | //+------------------------------------------------------------------+ class CDPCCBankIntegritySession { private: int m_handle; string m_expected_sha256; ulong m_file_bytes; SDPCCBankIntegrityResult m_pass_a; bool m_pass_a_valid; bool m_record_pass_verified; CDPCCBankRecordValidationObserver *m_observer; ulong m_generation; bool m_callback_active; bool m_callback_mutation; void BumpGeneration(void) { m_generation++; if(m_generation == 0) m_generation = 1; } bool RejectMutationDuringCallback(string &reason) { if(!m_callback_active) return(false); m_callback_mutation = true; reason = "bank.session.callback_mutation"; return(true); } void AbortBorrowedObserver(CDPCCBankRecordValidationObserver *observer) { const bool previously_active = m_callback_active; m_callback_active = true; if(DPCCBankObserverPointerValid(observer)) observer.Abort(); m_callback_active = previously_active; } void AbortObserver(void) { CDPCCBankRecordValidationObserver *observer = m_observer; m_observer = NULL; AbortBorrowedObserver(observer); } bool Fail(string &reason, const string code) { AbortObserver(); Close(); DPCCBankIntegrityClear(m_pass_a); m_pass_a_valid = false; m_record_pass_verified = false; reason = code; return(false); } bool CopyPassResult(SDPCCBankIntegrityResult &result, string &reason) { if(ArrayResize(result.header_bytes, ArraySize(m_pass_a.header_bytes)) != ArraySize(m_pass_a.header_bytes) || ArrayCopy(result.header_bytes, m_pass_a.header_bytes, 0, 0, ArraySize(m_pass_a.header_bytes)) != ArraySize( m_pass_a.header_bytes)) { reason = "bank.result.header.copy"; return(false); } result.header_length = m_pass_a.header_length; result.header_sha256 = m_pass_a.header_sha256; result.prefix_sha256 = m_pass_a.prefix_sha256; result.bank_sha256 = m_pass_a.bank_sha256; result.file_bytes = m_pass_a.file_bytes; result.record_count = m_pass_a.record_count; return(true); } public: void CDPCCBankIntegritySession(void) { m_handle = INVALID_HANDLE; m_expected_sha256 = ""; m_file_bytes = 0; m_pass_a_valid = false; m_record_pass_verified = false; m_observer = NULL; m_generation = 0; m_callback_active = false; m_callback_mutation = false; DPCCBankIntegrityClear(m_pass_a); } ~CDPCCBankIntegritySession(void) { Close(); } void Close(void) { if(m_callback_active) { m_callback_mutation = true; return; } BumpGeneration(); AbortObserver(); if(m_handle != INVALID_HANDLE) { FileClose(m_handle); m_handle = INVALID_HANDLE; } m_expected_sha256 = ""; m_file_bytes = 0; m_pass_a_valid = false; m_record_pass_verified = false; m_observer = NULL; m_callback_active = false; m_callback_mutation = false; DPCCBankIntegrityClear(m_pass_a); } bool IsOpen(void) const { return(m_handle != INVALID_HANDLE); } bool CopyVerifiedPassA(SDPCCBankIntegrityResult &result, string &reason) { DPCCBankIntegrityClear(result); reason = ""; if(RejectMutationDuringCallback(reason)) return(false); if(m_handle == INVALID_HANDLE || !m_pass_a_valid) { reason = "bank.session.pass_a_required"; return(false); } if(!CopyPassResult(result, reason)) { DPCCBankIntegrityClear(result); if(reason == "") reason = "bank.result.copy"; return(false); } return(true); } bool OpenAndVerify(const string resolved_sandbox_path, const bool common_files, const string expected_bank_sha256, string &reason) { if(RejectMutationDuringCallback(reason)) return(false); BumpGeneration(); Close(); reason = ""; if(!DPCCBankExpectedSHAValid(expected_bank_sha256)) return(Fail(reason, "bank.expected_sha256")); string lowered_path = resolved_sandbox_path; StringToLower(lowered_path); const int path_length = StringLen(lowered_path); if(path_length >= 5 && StringSubstr(lowered_path, path_length - 5) == ".part") return(Fail(reason, "bank.path.part")); if(resolved_sandbox_path == "") return(Fail(reason, "bank.path.empty")); int flags = FILE_READ | FILE_BIN | FILE_SHARE_READ; if(common_files) flags |= FILE_COMMON; ResetLastError(); m_handle = FileOpen(resolved_sandbox_path, flags); if(m_handle == INVALID_HANDLE) return(Fail(reason, StringFormat("bank.file.open.%d", GetLastError()))); m_expected_sha256 = expected_bank_sha256; ResetLastError(); m_file_bytes = FileSize(m_handle); if(GetLastError() != 0) return(Fail(reason, "bank.file.size")); if(!DPCCBankScanHandle(m_handle, m_expected_sha256, m_pass_a, reason, NULL, m_callback_active, m_callback_mutation, m_generation, m_generation)) return(Fail(reason, reason)); if(FileTell(m_handle) != (long)m_file_bytes) return(Fail(reason, "bank.file.not_eof")); m_pass_a_valid = true; reason = ""; return(true); } //--- Pass-B bytes are provisional inputs for internal semantic validation. //--- This session has no API that exposes records to training. The borrowed observer must //--- outlive the session until FinishRecordPass or Close; every deferred dereference is guarded. bool BeginRecordPass(CDPCCBankRecordValidationObserver *observer, string &reason) { if(RejectMutationDuringCallback(reason)) return(false); BumpGeneration(); m_callback_mutation = false; reason = ""; if(m_handle == INVALID_HANDLE || !m_pass_a_valid) return(Fail(reason, "bank.session.pass_a_required")); if(m_record_pass_verified) return(Fail(reason, "bank.session.record_pass_repeated")); const ENUM_POINTER_TYPE observer_type = CheckPointer(observer); if(observer_type != POINTER_AUTOMATIC && observer_type != POINTER_DYNAMIC) return(Fail(reason, "bank.record.observer.invalid_pointer")); m_observer = observer; SDPCCBankIntegrityResult pass_b; DPCCBankIntegrityClear(pass_b); const ulong pass_generation = m_generation; if(!DPCCBankScanHandle(m_handle, m_expected_sha256, pass_b, reason, m_observer, m_callback_active, m_callback_mutation, m_generation, pass_generation)) return(Fail(reason, reason)); if(m_generation != pass_generation || m_callback_mutation) return(Fail(reason, "bank.session.callback_mutation")); if(FileTell(m_handle) != (long)m_file_bytes || pass_b.file_bytes != m_pass_a.file_bytes || pass_b.record_count != m_pass_a.record_count || pass_b.bank_sha256 != m_pass_a.bank_sha256 || pass_b.prefix_sha256 != m_pass_a.prefix_sha256 || pass_b.header_sha256 != m_pass_a.header_sha256) { DPCCBankIntegrityClear(pass_b); return(Fail(reason, "bank.record_pass.changed")); } DPCCBankIntegrityClear(pass_b); m_record_pass_verified = true; return(true); } bool BeginRecordPass(CDPCCBankRecordValidationObserver &observer, string &reason) { return(BeginRecordPass(GetPointer(observer), reason)); } bool FinishRecordPass(SDPCCBankIntegrityResult &result, string &reason) { DPCCBankIntegrityClear(result); if(RejectMutationDuringCallback(reason)) return(false); BumpGeneration(); reason = ""; if(m_handle == INVALID_HANDLE || !m_pass_a_valid || !m_record_pass_verified) return(Fail(reason, "bank.session.record_pass_incomplete")); if(!CopyPassResult(result, reason)) { DPCCBankIntegrityClear(result); return(Fail(reason, reason)); } CDPCCBankRecordValidationObserver *observer = m_observer; m_observer = NULL; if(!DPCCBankObserverPointerValid(observer)) { DPCCBankIntegrityClear(result); return(Fail(reason, "bank.record.observer.invalid_pointer")); } const ulong complete_generation = m_generation; m_callback_mutation = false; m_callback_active = true; const bool complete_ok = observer.Complete(reason); m_callback_active = false; if(m_generation != complete_generation || m_callback_mutation) { AbortBorrowedObserver(observer); Close(); DPCCBankIntegrityClear(result); reason = "bank.session.reentered"; return(false); } if(!complete_ok) { AbortBorrowedObserver(observer); DPCCBankIntegrityClear(result); return(Fail(reason, "bank.record.observer.complete")); } if(!m_pass_a_valid || !m_record_pass_verified || m_handle == INVALID_HANDLE) { AbortBorrowedObserver(observer); Close(); DPCCBankIntegrityClear(result); reason = "bank.session.reentered"; return(false); } reason = ""; Close(); return(true); } bool FinishScanOnly(SDPCCBankIntegrityResult &result, string &reason) { DPCCBankIntegrityClear(result); if(RejectMutationDuringCallback(reason)) return(false); BumpGeneration(); reason = ""; if(m_handle == INVALID_HANDLE || !m_pass_a_valid) return(Fail(reason, "bank.session.pass_a_required")); if(!CopyPassResult(result, reason)) { DPCCBankIntegrityClear(result); return(Fail(reason, reason)); } reason = ""; Close(); return(true); } }; //+------------------------------------------------------------------+ //| Function DPCCScanPublishedBankIntegrity | //+------------------------------------------------------------------+ bool DPCCScanPublishedBankIntegrity(const string resolved_sandbox_path, const bool common_files, const string expected_bank_sha256, SDPCCBankIntegrityResult &result, string &reason) { DPCCBankIntegrityClear(result); reason = ""; CDPCCBankIntegritySession session; if(!session.OpenAndVerify(resolved_sandbox_path, common_files, expected_bank_sha256, reason)) return(false); return(session.FinishScanOnly(result, reason)); } #endif //+------------------------------------------------------------------+ //| TrajectoryRecord.mqh: VALID-record identity and time admission | //+------------------------------------------------------------------+ #ifndef DPCC_TRAJECTORY_RECORD_MQH #define DPCC_TRAJECTORY_RECORD_MQH struct SDPCCRecordAdmission { bool admitted; string split; long decision_bar_index; long decision_time; long horizon; }; //+------------------------------------------------------------------+ //| Function DPCCClearRecordAdmission | //+------------------------------------------------------------------+ void DPCCClearRecordAdmission(SDPCCRecordAdmission &result) { result.admitted = false; result.split = ""; result.decision_bar_index = 0; result.decision_time = 0; result.horizon = 0; } //+------------------------------------------------------------------+ //| Function DPCCRecordFail | //+------------------------------------------------------------------+ bool DPCCRecordFail(SDPCCRecordAdmission &result, string &reason, const string code) { DPCCClearRecordAdmission(result); reason = code; return(false); } //--- Exact JSON decimal-to-signed-long conversion. Integral decimal/exponent //--- spellings are accepted; fractional values and signed overflow fail closed. bool DPCCRecordReadSignedLong(CDPCCJsonDocument &document, const int node, long &value) { value = 0; if(node < 0 || document.Type(node) != DPCC_JSON_NUMBER) return(false); const string lexeme = document.Text(node); const int length = StringLen(lexeme); if(length <= 0) return(false); int position = 0; bool negative = false; if(StringGetCharacter(lexeme, position) == '-') { negative = true; position++; } string digits = ""; int fraction = 0; bool decimal = false; bool any_digit = false; for(; position < length; position++) { const ushort ch = StringGetCharacter(lexeme, position); if(ch >= '0' && ch <= '9') { digits += ShortToString(ch); if(decimal) fraction++; any_digit = true; continue; } if(ch == '.' && !decimal) { decimal = true; continue; } break; } if(!any_digit) return(false); int exponent = 0; if(position < length && (StringGetCharacter(lexeme, position) == 'e' || StringGetCharacter(lexeme, position) == 'E')) { position++; bool exponent_negative = false; if(position < length && (StringGetCharacter(lexeme, position) == '+' || StringGetCharacter(lexeme, position) == '-')) { exponent_negative = (StringGetCharacter(lexeme, position) == '-'); position++; } if(position >= length) return(false); for(; position < length; position++) { const ushort ch = StringGetCharacter(lexeme, position); if(ch < '0' || ch > '9') return(false); const int digit = (int)(ch - '0'); if(exponent > DPCC_JSON_MAX_SOURCE_CHARS) exponent = DPCC_JSON_MAX_SOURCE_CHARS + 1; else exponent = exponent * 10 + digit; } if(exponent_negative) exponent = -exponent; } else if(position != length) return(false); int first = 0; while(first < StringLen(digits) && StringGetCharacter(digits, first) == '0') first++; if(first == StringLen(digits)) return(true); digits = StringSubstr(digits, first); int scale = exponent - fraction; if(scale < 0) { const int remove = -scale; if(remove > StringLen(digits)) return(false); for(int i = StringLen(digits) - remove; i < StringLen(digits); i++) if(StringGetCharacter(digits, i) != '0') return(false); digits = StringSubstr(digits, 0, StringLen(digits) - remove); scale = 0; if(StringLen(digits) == 0) return(true); } if(scale > 19 || StringLen(digits) + scale > 19) return(false); const ulong positive_limit = (ulong)LONG_MAX; const ulong negative_limit = positive_limit + 1; const ulong limit = (negative ? negative_limit : positive_limit); ulong magnitude = 0; const int total = StringLen(digits) + scale; for(int i = 0; i < total; i++) { const int digit = (i < StringLen(digits) ? (int)(StringGetCharacter(digits, i) - '0') : 0); if(magnitude > (limit - (ulong)digit) / 10) return(false); magnitude = magnitude * 10 + (ulong)digit; } if(negative) { if(magnitude == negative_limit) value = LONG_MIN; else value = -(long)magnitude; } else value = (long)magnitude; return(true); } //+------------------------------------------------------------------+ //| Function DPCCRecordReadLongField | //+------------------------------------------------------------------+ bool DPCCRecordReadLongField(CDPCCJsonDocument &document, const int object_node, const string field, long &value, const string path, string &reason) { const int node = document.FindMember(object_node, field); if(!DPCCRecordReadSignedLong(document, node, value)) { reason = path + ".integer"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCRecordReadStringField | //+------------------------------------------------------------------+ bool DPCCRecordReadStringField(CDPCCJsonDocument &document, const int object_node, const string field, string &value, const string path, string &reason) { const int node = document.FindMember(object_node, field); if(node < 0 || document.Type(node) != DPCC_JSON_STRING) { reason = path; return(false); } value = document.Text(node); return(true); } //+------------------------------------------------------------------+ //| Function DPCCRecordSplit | //+------------------------------------------------------------------+ bool DPCCRecordSplit(const SDPCCDatasetManifest &dataset, const string split, SDPCCDatasetInterval &interval, string &reason) { if(split == "fit") interval = dataset.fit; else if(split == "validation") interval = dataset.validation; else if(split == "test") interval = dataset.test; else { reason = "record.split"; return(false); } if(interval.start_utc < 0 || interval.end_utc <= interval.start_utc || interval.bar_start < 0 || interval.bar_end <= interval.bar_start) { reason = "dataset.splits." + split; return(false); } return(true); } //--- Admission boundary: input Dataset/links/Profile/Bank/Header/TimeIndex values //--- must already have passed their respective linked metadata gates. The typed //--- TimeIndex arrays are structurally revalidated here, but the DTO carries no //--- content digest, so this function cannot authenticate caller-mutated arrays. //--- INVALID records are rejected for training admission; bank-level policy must //--- skip them without rejecting an otherwise complete published bank. bool DPCCAdmitValidTrajectoryRecord(const string record_source, const string record_schema, const string accepted_header_source, const string bank_header_schema, const SDPCCDatasetManifest &accepted_dataset, const SDPCCDatasetLinkResult &accepted_links, const SDPCCDataProfile &accepted_profile, const SDPCCBankMetadataResult &accepted_bank, const SDPCCDatasetTimeIndex &time_index, SDPCCRecordAdmission &result, string &reason) { DPCCClearRecordAdmission(result); reason = ""; if(accepted_dataset.dataset_id == "" || accepted_dataset.normalization_id == "" || accepted_dataset.time_index.id == "" || accepted_dataset.time_index.sha256 == "" || accepted_links.dataset_id != accepted_dataset.dataset_id || accepted_links.time_index_id != accepted_dataset.time_index.id || accepted_links.time_index_sha256 != accepted_dataset.time_index.sha256 || accepted_profile.dataset_manifest.id != accepted_links.dataset_id || accepted_profile.dataset_manifest.sha256 != accepted_links.dataset_sha256 || accepted_profile.forecast_id == "" || accepted_bank.bank_id == "" || accepted_bank.dataset_id != accepted_dataset.dataset_id || accepted_bank.time_index_id != accepted_links.time_index_id) return(DPCCRecordFail(result, reason, "record.accepted_contract")); CDPCCJsonDocument header_document; if(!DPCCCentralSchemaValidate(header_document, accepted_header_source, bank_header_schema, reason, "record.header")) return(DPCCRecordFail(result, reason, reason)); const int header_root = header_document.Root(); string header_bank_id = "", header_dataset_id = "", header_forecast_id = "", header_normalization_id = ""; if(!DPCCRecordReadStringField(header_document, header_root, "bank_id", header_bank_id, "record.header.bank_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "dataset_id", header_dataset_id, "record.header.dataset_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "forecast_id", header_forecast_id, "record.header.forecast_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "normalization_id", header_normalization_id, "record.header.normalization_id", reason)) return(DPCCRecordFail(result, reason, reason)); if(header_bank_id != accepted_bank.bank_id || header_dataset_id != accepted_dataset.dataset_id || header_forecast_id != accepted_profile.forecast_id || header_normalization_id != accepted_dataset.normalization_id) return(DPCCRecordFail(result, reason, "record.header.accepted_contract")); const int header_shapes = header_document.FindMember(header_root, "shapes"); long header_horizon = 0; if(!DPCCRecordReadLongField(header_document, header_shapes, "H", header_horizon, "record.header.shapes.H", reason) || header_horizon <= 0 || (ulong)header_horizon != (ulong)accepted_dataset.shapes.H) return(DPCCRecordFail(result, reason, "record.header.shapes.H")); string time_reason = ""; if(!DPCCValidateDatasetTimeIndex(time_index, accepted_dataset.dataset_id, time_reason)) return(DPCCRecordFail(result, reason, "record.time_index." + time_reason)); if(time_index.time_index_id != accepted_links.time_index_id || time_index.dataset_id != accepted_links.dataset_id) return(DPCCRecordFail(result, reason, "record.time_index.identity")); CDPCCJsonDocument record_document; if(!DPCCCentralSchemaValidate(record_document, record_source, record_schema, reason, "record")) return(DPCCRecordFail(result, reason, reason)); const int root = record_document.Root(); string state = "", record_dataset_id = "", record_forecast_id = "", record_normalization_id = ""; if(!DPCCRecordReadStringField(record_document, root, "record_state", state, "record.record_state", reason) || !DPCCRecordReadStringField(record_document, root, "dataset_id", record_dataset_id, "record.dataset_id", reason) || !DPCCRecordReadStringField(record_document, root, "forecast_id", record_forecast_id, "record.forecast_id", reason) || !DPCCRecordReadStringField(record_document, root, "normalization_id", record_normalization_id, "record.normalization_id", reason)) return(DPCCRecordFail(result, reason, reason)); if(state != "VALID") return(DPCCRecordFail(result, reason, "record.state.not_valid")); if(record_dataset_id != accepted_links.dataset_id) return(DPCCRecordFail(result, reason, "record.identity.dataset_id")); if(record_forecast_id != accepted_profile.forecast_id) return(DPCCRecordFail(result, reason, "record.identity.forecast_id")); if(record_normalization_id != accepted_dataset.normalization_id) return(DPCCRecordFail(result, reason, "record.identity.normalization_id")); long decision_bar = 0, decision_time = 0, forecast_asof = 0, normalization_asof = 0; if(!DPCCRecordReadLongField(record_document, root, "decision_bar_index", decision_bar, "record.decision_bar_index", reason) || !DPCCRecordReadLongField(record_document, root, "decision_time", decision_time, "record.decision_time", reason) || !DPCCRecordReadLongField(record_document, root, "forecast_asof", forecast_asof, "record.forecast_asof", reason) || !DPCCRecordReadLongField(record_document, root, "normalization_asof", normalization_asof, "record.normalization_asof", reason)) return(DPCCRecordFail(result, reason, reason)); if(decision_bar < 0) return(DPCCRecordFail(result, reason, "record.decision_bar_index.negative")); if(forecast_asof > decision_time) return(DPCCRecordFail(result, reason, "record.forecast_asof.future")); if(normalization_asof > decision_time) return(DPCCRecordFail(result, reason, "record.normalization_asof.future")); string split = ""; if(!DPCCRecordReadStringField(record_document, root, "split", split, "record.split", reason)) return(DPCCRecordFail(result, reason, reason)); SDPCCDatasetInterval interval; if(!DPCCRecordSplit(accepted_dataset, split, interval, reason)) return(DPCCRecordFail(result, reason, reason)); const int count = ArraySize(time_index.bar_index); if(decision_bar > (long)count) return(DPCCRecordFail(result, reason, "record.decision_bar_index.range")); const ulong available = (ulong)(count - (int)decision_bar); const ulong horizon = (ulong)accepted_dataset.shapes.H; if(horizon == 0) return(DPCCRecordFail(result, reason, "dataset.shapes.H")); if(horizon > available) return(DPCCRecordFail(result, reason, "record.horizon.range")); if(decision_bar < interval.bar_start || decision_bar >= interval.bar_end || interval.bar_end > count) return(DPCCRecordFail(result, reason, "record.split.bar_bounds")); const int bar_times = record_document.FindMember(root, "bar_times"); const int action_times = record_document.FindMember(root, "source_action_times"); if(bar_times < 0 || record_document.Type(bar_times) != DPCC_JSON_ARRAY || record_document.ChildCount(bar_times) != (int)horizon) return(DPCCRecordFail(result, reason, "record.bar_times.length")); if(action_times < 0 || record_document.Type(action_times) != DPCC_JSON_ARRAY || record_document.ChildCount(action_times) != (int)horizon) return(DPCCRecordFail(result, reason, "record.source_action_times.length")); if(decision_time != time_index.start_utc[(int)decision_bar]) return(DPCCRecordFail(result, reason, "record.decision_time.index")); for(int h = 0; h < (int)horizon; h++) { const long ordinal = decision_bar + h; const int bar_node = record_document.ChildAt(bar_times, h); long open_utc = 0, close_utc = 0, bar_ordinal = 0, action_time = 0; if(!DPCCRecordReadLongField(record_document, bar_node, "open_utc", open_utc, StringFormat("record.bar_times[%d].open_utc", h), reason) || !DPCCRecordReadLongField(record_document, bar_node, "close_utc", close_utc, StringFormat("record.bar_times[%d].close_utc", h), reason) || !DPCCRecordReadLongField(record_document, bar_node, "bar_index", bar_ordinal, StringFormat("record.bar_times[%d].bar_index", h), reason) || !DPCCRecordReadSignedLong(record_document, record_document.ChildAt(action_times, h), action_time)) { if(reason == "") reason = StringFormat("record.source_action_times[%d].integer", h); return(DPCCRecordFail(result, reason, reason)); } if(bar_ordinal < 0) return(DPCCRecordFail(result, reason, StringFormat("record.bar_times[%d].bar_index.negative", h))); if(bar_ordinal != ordinal || time_index.bar_index[(int)ordinal] != ordinal) return(DPCCRecordFail(result, reason, StringFormat("record.bar_times[%d].bar_index", h))); if(open_utc != time_index.start_utc[(int)ordinal]) return(DPCCRecordFail(result, reason, StringFormat("record.bar_times[%d].open_utc", h))); if(close_utc != time_index.end_utc[(int)ordinal]) return(DPCCRecordFail(result, reason, StringFormat("record.bar_times[%d].close_utc", h))); if(ordinal < interval.bar_start || ordinal >= interval.bar_end || open_utc < interval.start_utc || close_utc > interval.end_utc) return(DPCCRecordFail(result, reason, StringFormat("record.split.bounds[%d]", h))); if(action_time > open_utc) return(DPCCRecordFail(result, reason, StringFormat("record.source_action_times[%d].future", h))); } result.admitted = true; result.split = split; result.decision_bar_index = decision_bar; result.decision_time = decision_time; result.horizon = (long)horizon; reason = ""; return(true); } //--- Provisional CPU materialization only. Whole-bank observer authorization is //--- deliberately not represented by this per-record value. struct SDPCCMaterializedRecord { SDPCCRecordAdmission admission; string window_id, dataset_id, forecast_id, normalization_id, source_id, noise_seed, execution_label; long episode_id, forecast_asof, normalization_asof; ulong noise_seed_u64; float account_features[], context_without_noise[], target_native_ohlc[], rho[], actions[], executed_actions[], rewards[]; uint terminal_flags[]; long bar_open_times[], bar_close_times[], bar_indices[], source_action_times[]; }; //+------------------------------------------------------------------+ //| Function DPCCClearMaterializedRecord | //+------------------------------------------------------------------+ void DPCCClearMaterializedRecord(SDPCCMaterializedRecord &record) { DPCCClearRecordAdmission(record.admission); record.window_id = ""; record.dataset_id = ""; record.forecast_id = ""; record.normalization_id = ""; record.source_id = ""; record.noise_seed = ""; record.execution_label = ""; record.episode_id = 0; record.forecast_asof = 0; record.normalization_asof = 0; record.noise_seed_u64 = 0; ArrayResize(record.account_features, 0); ArrayResize(record.context_without_noise, 0); ArrayResize(record.target_native_ohlc, 0); ArrayResize(record.rho, 0); ArrayResize(record.actions, 0); ArrayResize(record.executed_actions, 0); ArrayResize(record.rewards, 0); ArrayResize(record.terminal_flags, 0); ArrayResize(record.bar_open_times, 0); ArrayResize(record.bar_close_times, 0); ArrayResize(record.bar_indices, 0); ArrayResize(record.source_action_times, 0); } //--- A record-array decoder, not a second general JSON validator. Both double //--- and final float32 probability sums are checked without renormalization. //--- The JSON parser already guarantees numeric syntax. Inspect only the mantissa: //--- StringToDouble can silently turn a nonzero tiny decimal into double zero. //+------------------------------------------------------------------+ //| Function DPCCMaterializeValidTrajectoryRecord | //+------------------------------------------------------------------+ bool DPCCMaterializeValidTrajectoryRecord(const string record_source, const string record_schema, const string accepted_header_source, const string bank_header_schema, const SDPCCDatasetManifest &accepted_dataset, const SDPCCDatasetLinkResult &accepted_links, const SDPCCDataProfile &accepted_profile, const SDPCCBankMetadataResult &accepted_bank, const SDPCCDatasetTimeIndex &time_index, const SDPCCConfig &frozen_config, SDPCCMaterializedRecord &result, string &reason) { DPCCClearMaterializedRecord(result); reason = ""; SDPCCMaterializedRecord candidate; DPCCClearMaterializedRecord(candidate); if(!DPCCAdmitValidTrajectoryRecord(record_source, record_schema, accepted_header_source, bank_header_schema, accepted_dataset, accepted_links, accepted_profile, accepted_bank, time_index, candidate.admission, reason)) return(false); const SDPCCShape shape = frozen_config.shape; const SDPCCResourceShape expected = accepted_dataset.shapes; if(shape.K != (ulong)expected.K || shape.V != (ulong)expected.V || shape.H != (ulong)expected.H || shape.D != (ulong)expected.D || shape.M != (ulong)expected.M || shape.J != (ulong)expected.J || shape.E != (ulong)expected.E || shape.NActions != 6 || shape.StateSize != 13 || expected.A != 6 || expected.B != 1 || expected.account_features != 13 || frozen_config.normalization_mode != DPCC_NORMALIZATION_CAUSAL_AFFINE_V2 || !MathIsValidNumber(frozen_config.probability_tolerance) || frozen_config.probability_tolerance < 0 || frozen_config.probability_tolerance >= 1) {reason = "record.materialization.frozen_config"; return(false);} SDPCCLayout layout, rollout; uint tokens; if(!DPCCCompositeLayouts(shape, layout, rollout, tokens) || layout.offsets[10] > (ulong)INT_MAX) {reason = "record.materialization.layout"; return(false);} CDPCCJsonDocument document; if(!document.Parse(record_source, reason)) { reason = "record.materialization.parse"; return(false); } const int root = document.Root(), context = document.FindMember(root, "context"); const string identities[] = {"window_id", "dataset_id", "forecast_id", "normalization_id", "source_id", "noise_seed", "execution_label"}; string texts[]; ArrayResize(texts, 7); for(uint i = 0; i < 7; i++) if(!DPCCRecordReadStringField(document, root, identities[i], texts[i], "record." + identities[i], reason)) return(false); candidate.window_id = texts[0]; candidate.dataset_id = texts[1]; candidate.forecast_id = texts[2]; candidate.normalization_id = texts[3]; candidate.source_id = texts[4]; candidate.noise_seed = texts[5]; candidate.execution_label = texts[6]; if(!DPCCRecordReadLongField(document, root, "episode_id", candidate.episode_id, "record.episode_id", reason) || !DPCCRecordReadLongField(document, root, "forecast_asof", candidate.forecast_asof, "record.forecast_asof", reason) || !DPCCRecordReadLongField(document, root, "normalization_asof", candidate.normalization_asof, "record.normalization_asof", reason)) return(false); for(int i = 0; i < 16; i++) { const int ch = (int)StringGetCharacter(candidate.noise_seed, i); const uint digit = (uint)(ch >= '0' && ch <= '9' ? ch - '0' : ch - 'a' + 10); candidate.noise_seed_u64 = (candidate.noise_seed_u64 << 4) | (ulong)digit; } const string hash_source = candidate.dataset_id + "\n" + candidate.source_id + "\n" + (string)candidate.episode_id + "\n" + (string)candidate.admission.decision_time + "\n" + candidate.noise_seed; uchar bytes[]; const int encoded = StringToCharArray(hash_source, bytes, 0, WHOLE_ARRAY, CP_UTF8); string digest = ""; if(encoded <= 0 || ArrayResize(bytes, encoded - 1) != encoded - 1 || !DPCCSHA256Lower(bytes, digest, reason) || digest != candidate.window_id) {reason = "record.window_id.derivation"; return(false);} if(!DPCCRecordFloatArray(document, document.FindMember(root, "account_features"), 13, candidate.account_features, "record.account_features", reason) || ArrayResize(candidate.context_without_noise, ( int)layout.offsets[10]) != (int)layout.offsets[10]) {if(reason == "")reason = "record.context.capacity"; return(false);} const string names[] = {"Z", "U", "Pi", "PriceNorm", "Account", "Instrument", "Limits", "Execution", "CalendarCount", "Calendar"}; float values[]; for(uint section = 0; section < 10; section++) { const int node = document.FindMember(context, names[section]); const int domain = (section == 1 ? 1 : (section == 2 ? 2 : 0)); if(!DPCCRecordFloatArray(document, node, (uint)layout.lengths[section], values, "record.context." + names[section], reason, domain, ( double)frozen_config.probability_tolerance)) return(false); for(uint i = 0; i < (uint)layout.lengths[section]; i++) candidate.context_without_noise[(int)layout.offsets[section] + (int)i] = values[i]; if(section == 8) for(uint i = 0; i < shape.H; i++) { long count = 0; if(!DPCCRecordReadSignedLong(document, document.ChildAt(node, (int)i), count) || count < 0 || count > (long)shape.E || (double)(float)count != (double)count) {reason = StringFormat("record.context.CalendarCount[%u].integer", i); return(false);} } if(section == 7) for(uint row = 0; row < shape.K * shape.H; row++) { long count = 0; if(!DPCCRecordReadSignedLong(document, document.ChildAt(node, (int)(row * 24 + 21)), count) || count < 0 || count > (long)shape.E || (double)(float)count != (double)count) {reason = StringFormat("record.context.Execution[%u].event_count", row); return(false);} } } for(uint channel = 0; channel < 4; channel++) { const int p = (int)layout.offsets[3] + (int)channel * 4; long exact_gain = 0; double raw_bias = 0; const int price_node = document.FindMember(context, "PriceNorm"); if(candidate.context_without_noise[p + 1] <= 0 || candidate.context_without_noise[p + 2] != 1 || candidate.context_without_noise[p + 3] != 0 || !DPCCRecordReadSignedLong(document, document.ChildAt(price_node, (int)channel * 4 + 2), exact_gain) || !document.TryNumber(document.ChildAt(price_node, (int)channel * 4 + 3), raw_bias) || exact_gain != 1 || raw_bias != 0) {reason = "record.context.PriceNorm.profile"; return(false);} } if(!DPCCRecordFloatArray(document, document.FindMember(root, "target_native_ohlc"), shape.H * 4, candidate.target_native_ohlc, "record.target_native_ohlc", reason) || !DPCCRecordFloatArray(document, document.FindMember(root, "rho"), shape.K, candidate.rho, "record.rho", reason, 2, (double)frozen_config.probability_tolerance) || !DPCCRecordFloatArray(document, document.FindMember(root, "actions"), shape.H * 6, candidate.actions, "record.actions", reason) || !DPCCRecordFloatArray(document, document.FindMember(root, "executed_actions"), shape.H * 6, candidate.executed_actions, "record.executed_actions", reason) || !DPCCRecordFloatArray(document, document.FindMember(root, "rewards"), shape.H, candidate.rewards, "record.rewards", reason)) return(false); if(ArrayResize(candidate.terminal_flags, (int)shape.H) != (int)shape.H || ArrayResize(candidate.bar_open_times, (int)shape.H) != (int)shape.H || ArrayResize(candidate.bar_close_times, (int)shape.H) != (int)shape.H || ArrayResize(candidate.bar_indices, (int)shape.H) != (int)shape.H || ArrayResize(candidate.source_action_times, (int)shape.H) != (int)shape.H) {reason = "record.targets.capacity"; return(false);} const int flags = document.FindMember(root, "terminal_flags"), bars = document.FindMember(root, "bar_times"), action_times = document.FindMember(root, "source_action_times"); if(document.ChildCount(flags) != (int)shape.H) { reason = "record.terminal_flags.length"; return(false); } bool terminal = false; for(uint h = 0; h < shape.H; h++) { long flag = 0; const int bar = document.ChildAt(bars, (int)h); if(!DPCCRecordReadSignedLong(document, document.ChildAt(flags, (int)h), flag) || (flag != 0 && flag != 1) || !DPCCRecordReadLongField(document, bar, "open_utc", candidate.bar_open_times[h], "record.bar_times.open_utc", reason) || !DPCCRecordReadLongField(document, bar, "close_utc", candidate.bar_close_times[h], "record.bar_times.close_utc", reason) || !DPCCRecordReadLongField(document, bar, "bar_index", candidate.bar_indices[h], "record.bar_times.bar_index", reason) || !DPCCRecordReadSignedLong(document, document.ChildAt(action_times, (int)h), candidate.source_action_times[h])) {if(reason == "")reason = "record.targets.integer"; return(false);} if(terminal) { double raw_value = 0; if(flag != 1 || candidate.rewards[h] != 0 || !document.TryNumber(document.ChildAt(document.FindMember(root, "rewards"), (int)h), raw_value) || raw_value != 0) {reason = "record.terminal.absorbing"; return(false);} for(uint a = 0; a < 6; a++) { if(candidate.actions[h * 6 + a] != 0 || candidate.executed_actions[h * 6 + a] != 0 || !document.TryNumber(document.ChildAt(document.FindMember(root, "actions"), (int)(h * 6 + a)), raw_value) || raw_value != 0 || !document.TryNumber(document.ChildAt(document.FindMember(root, "executed_actions"), (int)(h * 6 + a)), raw_value) || raw_value != 0) {reason = "record.terminal.absorbing"; return(false);} } } candidate.terminal_flags[h] = (uint)flag; if(flag == 1) terminal = true; } result = candidate; reason = ""; return(true); } #endif //--- Materialized records stay private through every callback. Complete means //--- semantic staging finished, not DATA readiness or permission to train. class CDPCCBankMaterializationObserver : public CDPCCBankRecordValidationObserver { private: string m_record_schema, m_header, m_header_schema; SDPCCDatasetManifest m_dataset; SDPCCDatasetLinkResult m_links; SDPCCDataProfile m_profile; SDPCCBankMetadataResult m_bank; SDPCCDatasetTimeIndex m_time_index; SDPCCConfig m_config; SDPCCMaterializedRecord m_records[]; string m_window_ids[]; ulong m_invalid; uchar m_payload[]; ulong m_next, m_expected; uint m_limit; bool m_active, m_complete, m_configured; bool Fail(string &reason, const string code) { Abort(); reason = code; return(false); } public: CDPCCBankMaterializationObserver(void) {m_configured = false; m_limit = 0; Abort();} bool Configure(const string record_schema, const string header, const string header_schema, const SDPCCDatasetManifest &dataset, const SDPCCDatasetLinkResult &links, const SDPCCDataProfile &profile, const SDPCCBankMetadataResult &bank, const SDPCCDatasetTimeIndex &time_index, const SDPCCConfig &config, const uint maximum_records, string &reason) { Abort(); m_configured = false; reason = ""; if(maximum_records == 0 || maximum_records > (uint)INT_MAX) return(Fail(reason, "staging.capacity.profile")); if(dataset.dataset_id == "" || dataset.normalization_id == "" || dataset.time_index.id == "" || dataset.time_index.sha256 == "" || links.dataset_id != dataset.dataset_id || links.time_index_id != dataset.time_index.id || links.time_index_sha256 != dataset.time_index.sha256 || profile.dataset_manifest.id != links.dataset_id || profile.dataset_manifest.sha256 != links.dataset_sha256 || profile.forecast_id == "" || bank.bank_id == "" || bank.dataset_id != dataset.dataset_id || bank.time_index_id != links.time_index_id || bank.record_count < 0 || !DPCCBankExpectedSHAValid(bank.bank_sha256) || !DPCCBankExpectedSHAValid(bank.header_sha256)) return(Fail(reason, "staging.accepted_contract")); if(!DPCCValidateDatasetTimeIndex(time_index, dataset.dataset_id, reason) || time_index.time_index_id != links.time_index_id || time_index.dataset_id != links.dataset_id) return(Fail(reason, "staging.time_index.identity")); CDPCCJsonDocument header_document; if(!DPCCCentralSchemaValidate(header_document, header, header_schema, reason, "staging.header")) {Abort(); return(false);} const int header_root = header_document.Root(); string bank_id = "", dataset_id = "", forecast_id = "", normalization_id = ""; if(!DPCCRecordReadStringField(header_document, header_root, "bank_id", bank_id, "staging.header.bank_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "dataset_id", dataset_id, "staging.header.dataset_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "forecast_id", forecast_id, "staging.header.forecast_id", reason) || !DPCCRecordReadStringField(header_document, header_root, "normalization_id", normalization_id, "staging.header.normalization_id", reason)) {Abort(); return(false);} if(bank_id != bank.bank_id || dataset_id != dataset.dataset_id || forecast_id != profile.forecast_id || normalization_id != dataset.normalization_id) return(Fail(reason, "staging.header.accepted_contract")); if(!DPCCBankCheckShape(header_document, header_root, dataset.shapes, reason)) {Abort(); return(false);} uchar header_bytes[]; string header_digest = ""; const int encoded = StringToCharArray(header, header_bytes, 0, WHOLE_ARRAY, CP_UTF8); if(encoded <= 0 || ArrayResize(header_bytes, encoded - 1) != encoded - 1 || !DPCCSHA256Lower(header_bytes, header_digest, reason) || header_digest != bank.header_sha256) return(Fail(reason, "staging.header.sha256")); m_record_schema = record_schema; m_header = header; m_header_schema = header_schema; m_dataset = dataset; m_links = links; m_profile = profile; m_bank = bank; m_time_index = time_index; m_config = config; m_limit = maximum_records; m_configured = true; return(true); } virtual bool BeginRecord(const ulong record_index, const ulong payload_bytes, string &reason) { reason = ""; if(!m_configured || m_active || m_complete || record_index != m_next || m_next >= m_limit || payload_bytes == 0 || payload_bytes > DPCC_BANK_MAX_JSON_BYTES) return(Fail(reason, "staging.record.begin")); m_expected = payload_bytes; if(ArrayResize(m_payload, 0) != 0) return(Fail(reason, "staging.payload.reset")); m_active = true; return(true); } virtual bool ConsumeChunk(const ulong record_index, const uchar &chunk[], const int count, string &reason) { reason = ""; const int old = ArraySize(m_payload); if(!m_active || record_index != m_next || count <= 0 || count > ArraySize(chunk) || (ulong)old + (ulong)count > m_expected) return(Fail(reason, "staging.payload.chunk")); if(ArrayResize(m_payload, old + count) != old + count || ArrayCopy(m_payload, chunk, old, 0, count) != count) return(Fail(reason, "staging.payload.capacity")); return(true); } virtual bool EndRecord(const ulong record_index, string &reason) { reason = ""; if(!m_active || record_index != m_next || (ulong)ArraySize(m_payload) != m_expected) return(Fail(reason, "staging.record.end")); const string source = CharArrayToString(m_payload, 0, ArraySize(m_payload), CP_UTF8); //--- Byte-exact UTF8 roundtrip rejects replacement characters / embedded NUL. uchar encoded[]; const int size = StringToCharArray(source, encoded, 0, WHOLE_ARRAY, CP_UTF8); if(size != ArraySize(m_payload) + 1) return(Fail(reason, "staging.payload.utf8")); for(int i = 0; i < ArraySize(m_payload); i++) if(encoded[i] != m_payload[i]) return(Fail(reason, "staging.payload.utf8")); //--- Envelope admission applies to ALL records. INVALID records have no //--- fabricated tensor values; their identity still participates in uniqueness. CDPCCJsonDocument document; if(!DPCCCentralSchemaValidate(document, source, m_record_schema, reason, "staging.record")) {Abort(); return(false);} const int root = document.Root(); string state = "", window = "", dataset = "", forecast = "", normalization = "", source_id = "", seed = ""; if(!DPCCRecordReadStringField(document, root, "record_state", state, "record.record_state", reason) || !DPCCRecordReadStringField(document, root, "window_id", window, "record.window_id", reason) || !DPCCRecordReadStringField(document, root, "dataset_id", dataset, "record.dataset_id", reason) || !DPCCRecordReadStringField(document, root, "forecast_id", forecast, "record.forecast_id", reason) || !DPCCRecordReadStringField(document, root, "normalization_id", normalization, "record.normalization_id", reason) || !DPCCRecordReadStringField(document, root, "source_id", source_id, "record.source_id", reason) || !DPCCRecordReadStringField(document, root, "noise_seed", seed, "record.noise_seed", reason)) {Abort(); return(false);} if(dataset != m_dataset.dataset_id || dataset != m_links.dataset_id || forecast != m_profile.forecast_id || normalization != m_dataset.normalization_id) return(Fail(reason, "staging.record.identity")); long episode = 0, time = 0, forecast_asof = 0, normalization_asof = 0; if(!DPCCRecordReadLongField(document, root, "episode_id", episode, "record.episode_id", reason) || !DPCCRecordReadLongField(document, root, "decision_time", time, "record.decision_time", reason) || !DPCCRecordReadLongField(document, root, "forecast_asof", forecast_asof, "record.forecast_asof", reason) || !DPCCRecordReadLongField(document, root, "normalization_asof", normalization_asof, "record.normalization_asof", reason)) {Abort(); return(false);} if(forecast_asof > time || normalization_asof > time) return(Fail(reason, "staging.record.future_statistics")); const string hash_source = dataset + "\n" + source_id + "\n" + (string)episode + "\n" + (string)time + "\n" + seed; uchar hash_bytes[]; string digest = ""; const int hash_size = StringToCharArray(hash_source, hash_bytes, 0, WHOLE_ARRAY, CP_UTF8); if(hash_size <= 0 || ArrayResize(hash_bytes, hash_size - 1) != hash_size - 1 || !DPCCSHA256Lower(hash_bytes, digest, reason) || digest != window) return(Fail(reason, "staging.window_id.derivation")); for(int i = 0; i < ArraySize(m_window_ids); i++) if(m_window_ids[i] == window) return(Fail(reason, "staging.window_id.duplicate")); const int identity_count = ArraySize(m_window_ids); if(ArrayResize(m_window_ids, identity_count + 1) != identity_count + 1) return(Fail(reason, "staging.identities.capacity")); m_window_ids[identity_count] = window; if(state == "INVALID") { string invalid_reason = ""; if(!DPCCRecordReadStringField(document, root, "invalid_reason", invalid_reason, "record.invalid_reason", reason)) {Abort(); return(false);} if(StringLen(invalid_reason) == 0) return(Fail(reason, "staging.invalid.reason_empty")); m_invalid++; m_next++; m_expected = 0; m_active = false; ArrayResize(m_payload, 0); return(true); } if(state != "VALID") return(Fail(reason, "staging.record.state")); SDPCCMaterializedRecord candidate; if(!DPCCMaterializeValidTrajectoryRecord(source, m_record_schema, m_header, m_header_schema, m_dataset, m_links, m_profile, m_bank, m_time_index, m_config, candidate, reason)) {Abort(); return(false);} const int next = ArraySize(m_records); if(ArrayResize(m_records, next + 1) != next + 1) return(Fail(reason, "staging.records.capacity")); m_records[next] = candidate; m_next++; m_expected = 0; m_active = false; ArrayResize(m_payload, 0); return(true); } virtual bool Complete(string &reason) { reason = ""; if(!m_configured || m_active || m_complete || m_next == 0) return(Fail(reason, "staging.complete.state")); m_complete = true; return(true); } virtual void Abort(void) { for(int i = 0; i < ArraySize(m_records); i++) DPCCClearMaterializedRecord(m_records[i]); ArrayResize(m_records, 0); ArrayResize(m_payload, 0); ArrayResize(m_window_ids, 0); m_next = 0; m_invalid = 0; m_expected = 0; m_active = false; m_complete = false; } //--- Caller uses this only AFTER FinishRecordPass returned true. Copies protect //--- stored rho from consumer writes; no mutable record/array pointer escapes. bool CopyFinishedRecords(SDPCCMaterializedRecord &result[], string &reason) { ArrayResize(result, 0); reason = ""; if(!m_complete) return(Fail(reason, "staging.not_complete")); const int count = ArraySize(m_records); if(ArrayResize(result, count) != count) { reason = "staging.output.capacity"; return(false); } for(int i = 0; i < count; i++) result[i] = m_records[i]; return(true); } ulong InvalidCount(void) const {return(m_complete ? m_invalid : 0);} ulong TotalCount(void) const {return(m_complete ? m_next : 0);} bool BindExpectedSHA(const string expected_sha, string &reason) { if(!m_configured || expected_sha != m_bank.bank_sha256) return(Fail(reason, "staging.bank.expected_sha256")); reason = ""; return(true); } bool BindFinishedIntegrity(const SDPCCBankIntegrityResult &integrity, string &reason) { if(!m_complete || integrity.bank_sha256 != m_bank.bank_sha256 || integrity.header_sha256 != m_bank.header_sha256 || integrity.record_count != (ulong)m_bank.record_count || integrity.record_count != m_next) return(Fail(reason, "staging.bank.finished_binding")); //--- Hash binding plus exact raw-byte comparison against the configured header. uchar expected[]; const int encoded = StringToCharArray(m_header, expected, 0, WHOLE_ARRAY, CP_UTF8); if(encoded != integrity.header_length + 1 || ArraySize(integrity.header_bytes) != integrity.header_length) return(Fail(reason, "staging.header.finished_bytes")); for(int i = 0; i < integrity.header_length; i++) if(expected[i] != integrity.header_bytes[i]) return(Fail(reason, "staging.header.finished_bytes")); reason = ""; return(true); } }; //--- This outer transaction is the only consumer entry: nothing is published if //--- either pass, any callback, footer/hash comparison or FinishRecordPass fails. bool DPCCStageMaterializedBank(CDPCCBankMaterializationObserver &observer, const string path, const bool common_files, const string expected_sha, SDPCCMaterializedRecord &records[], SDPCCBankIntegrityResult &integrity, ulong &invalid_records, string &reason) { invalid_records = 0; ArrayResize(records, 0); DPCCBankIntegrityClear(integrity); CDPCCBankIntegritySession session; if(!observer.BindExpectedSHA(expected_sha, reason) || !session.OpenAndVerify(path, common_files, expected_sha, reason) || !session.BeginRecordPass(observer, reason) || !session.FinishRecordPass(integrity, reason) || !observer.BindFinishedIntegrity(integrity, reason) || !observer.CopyFinishedRecords(records, reason)) { observer.Abort(); session.Close(); ArrayResize(records, 0); DPCCBankIntegrityClear(integrity); return(false); } if(observer.TotalCount() != integrity.record_count || (ulong)ArraySize(records) + observer.InvalidCount() != integrity.record_count) { observer.Abort(); ArrayResize(records, 0); DPCCBankIntegrityClear(integrity); reason = "staging.record_count.reconciliation"; return(false); } invalid_records = observer.InvalidCount(); observer.Abort(); return(true); } #endif enum ENUM_DPCC_TRAIN_OPERATION { DPCC_TRAIN_UPDATE = 0, DPCC_TRAIN_FROZEN = 1, DPCC_TRAIN_VALIDATE = 2 }; struct SDPCCTrainingBankSource { string record_schema, header, header_schema, path; bool common_files; uint maximum_records; ulong maximum_staged_tensor_bytes; SDPCCDatasetManifest dataset; SDPCCDatasetLinkResult links; SDPCCDataProfile profile; SDPCCBankMetadataResult bank; SDPCCDatasetTimeIndex time_index; }; struct SDPCCTrainingExampleKeys { uint j, repeat; ulong seed, decision_identity, context_identity, owner_identity; long decision_time; }; struct SDPCCTrainingFrozenRecordKeys { string window_id; long decision_time; ulong decision_identity, context_identity, owner_identity; uint validation_j; }; struct SDPCCTrainingExampleResult { bool completed, update_attempted; string stage, reason, window_id, source_id, split, bank_sha256, header_sha256; long decision_time; uint j, repeat, updates_before, updates_after; ulong seed; double loss; }; struct SDPCCDecoderValidationObservation { int record_index; string window_id, selected_manifest_sha; double normalized_loss; float normalized[], native_ohlc[], causal_price_norm[]; }; struct SDPCCDecoderInferenceObservation { int record_index; string window_id, selected_manifest_sha; long decision_time; float normalized[]; }; //--- Owns the factories and their frozen bindings; does not adopt arbitrary CNet. #ifndef DPCC_RUN_IDENTITY_MQH #define DPCC_RUN_IDENTITY_MQH struct SDPCCTrainingRecordIdentity { int record_index; string window_id, source_id, split, bank_sha256, header_sha256; long decision_time, decision_bar_index; SDPCCTrainingFrozenRecordKeys keys; }; #endif //--- Staging is semantic/integrity admission, never DATA/RESOURCES/RESEARCH READY. class CDPCCTrainingExamples { private: //--- Reverse destruction order keeps borrowed denoiser context alive. CNet m_decoder_net, m_denoiser_net; CNeuronScenarioDecoderOCL *m_decoder; CNeuronPlanDenoiserOCL *m_denoiser; SDPCCConfig m_config; string m_epoch_time_index_id; SDPCCMaterializedRecord m_records[]; SDPCCTrainingFrozenRecordKeys m_record_keys[]; SDPCCBankIntegrityResult m_integrity; double m_alpha[]; ulong m_training_seed, m_validation_seed; uint m_validation_repeat; ulong m_decoder_contract, m_denoiser_contract; bool m_initialized, m_poisoned, m_decoder_initial_saved, m_denoiser_initial_saved; int m_loaded; SDPCCLayout m_c, m_r; uint m_tokens, m_decoder_stream_native_loads; CBufferFloat m_z, m_norm, m_target, m_rho, m_context, m_account, m_actions, m_epsilon, m_noisy, m_condition; CNet *m_checkpoint_decoder_net; CNet *m_checkpoint_denoiser_net; CNet *m_inference_decoder_net; CBufferFloat *m_inference_input; SDPCCCheckpointArtifact m_inference_artifact; SDPCCDecoderComposition m_checkpoint_decoder_composition; SDPCCDenoiserComposition m_checkpoint_denoiser_composition; SDPCCCheckpointBinding m_checkpoint_binding; string m_checkpoint_dataset_id; string m_checkpoint_normalization_id; string m_checkpoint_bank_id; string m_checkpoint_dataset_sha; string m_checkpoint_timeindex_sha; string m_checkpoint_timeindex_table_sha; string m_checkpoint_binding_sha; SDPCCCheckpointArtifact m_checkpoint_initial; bool m_checkpoint_metadata_bound; bool m_checkpoint_restored; ulong m_checkpoint_max_file_bytes; CNet *DecoderNet(void) { return(m_checkpoint_decoder_net ? m_checkpoint_decoder_net : GetPointer(m_decoder_net)); } void CloseDecoderInference(void) { delete m_inference_input; m_inference_input = NULL; delete m_inference_decoder_net; m_inference_decoder_net = NULL; ZeroMemory(m_inference_artifact); } bool DecoderInferenceFailure(const string cause, string &reason) { CloseDecoderInference(); reason = cause; return(false); } bool DecoderInferenceArtifact(const SDPCCCheckpointArtifact &artifact, string &reason) { string sha = ""; if(!DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, sha, reason) || sha != artifact.manifest_sha256) { reason = "decoder.infer.manifest_artifact"; return(false); } if(!DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.decoder_sha256) { reason = "decoder.infer.decoder_artifact"; return(false); } return(true); } CNet *DenoiserNet(void) { return(m_checkpoint_denoiser_net ? m_checkpoint_denoiser_net : GetPointer(m_denoiser_net)); } //+------------------------------------------------------------------+ //| Computes a complete immutable owner/profile/table SHA | //+------------------------------------------------------------------+ bool CheckpointBindingSHA(const SDPCCCheckpointBinding &binding, string &sha, string &reason) { const string receipts[] = {binding.training_config_sha256, binding.model_manifest_sha256, binding.bank_manifest_sha256, binding.dataset_manifest_sha256, binding.time_index_sha256, binding.normalization_section_sha256, binding.decoder_plan_sha256, binding.denoiser_plan_sha256 }; string text = "DPCC_CHECKPOINT_BINDING_V1\n"; for(uint i = 0; i < (uint)ArraySize(receipts); i++) { if(!DPCCResolverLowerSha256(receipts[i])) { reason = "checkpoint.binding.sha256"; return(false); } text += receipts[i] + "\n"; } text += m_integrity.bank_sha256 + "\n" + m_integrity.header_sha256 + "\n"; text += (string)StringLen(m_checkpoint_dataset_id) + ":" + m_checkpoint_dataset_id + "\n"; text += (string)StringLen(m_checkpoint_bank_id) + ":" + m_checkpoint_bank_id + "\n"; text += (string)StringLen(m_checkpoint_normalization_id) + ":" + m_checkpoint_normalization_id + "\n"; text += m_checkpoint_timeindex_table_sha + "\n"; text += (string)m_decoder_contract + "\n" + (string)m_denoiser_contract + "\n"; text += (string)m_training_seed + "\n" + (string)m_validation_seed + "\n" + (string)m_validation_repeat + "\n"; const SDPCCShape s = m_config.shape; const uint words[] = {s.K, s.V, s.H, s.D, s.M, s.J, s.E, s.PlanBatch, s.NActions, s.StateSize, s.Embedding, s.Heads, s.KeyDimension, s.Depth, m_config.contract_version, m_config.binary_layer_version, (uint)m_config.mode, m_config.projection_max_iterations, m_config.projection_max_evaluations, m_config.projection_lookback_steps, m_config.price_basis, m_config.quote_adapter, m_config.event_model, (uint)m_config.risk_type, m_config.normalization_mode, m_config.loss_coordinates, m_config.randomness_version }; for(uint i = 0; i < (uint)ArraySize(words); i++) text += (string)words[i] + "\n"; const double scalars[] = {m_config.projection_improvement_atol, m_config.projection_distance_tie_atol, m_config.volume_scale, m_config.price_scale, m_config.money_scale, m_config.max_quote_age_seconds, m_config.max_last_age_seconds, ( double)m_config.risk_parameter, (double)m_config.mean_weight, (double)m_config.gamma, ( double)m_config.probability_tolerance }; for(uint i = 0; i < (uint)ArraySize(scalars); i++) { if(!MathIsValidNumber(scalars[i])) { reason = "checkpoint.binding.numeric"; return(false); } text += StringFormat("%.17g", scalars[i]) + "\n"; } for(uint i = 0; i < 6; i++) text += StringFormat("%.17g\n%.17g\n", (double)m_config.action_bias[i], (double)m_config.action_scale[i]); for(int i = 0; i < ArraySize(m_alpha); i++) text += StringFormat("%.17g\n", m_alpha[i]); for(int i = 0; i < ArraySize(m_record_keys); i++) { const SDPCCTrainingFrozenRecordKeys k = m_record_keys[i]; text += (string)StringLen(k.window_id) + ":" + k.window_id + "\n"; text += (string)k.decision_time + "\n" + (string)k.decision_identity + "\n"; text += (string)k.context_identity + "\n" + (string)k.owner_identity + "\n" + (string)k.validation_j + "\n"; text += (string)StringLen(m_records[i].source_id) + ":" + m_records[i].source_id + "\n"; text += m_records[i].admission.split + "\n"; } return(DPCCCheckpointTextSHA(text, sha, reason)); } bool CheckpointPath(const string path, string &canonical, string &reason) { string prepared = ""; if(!DPCCResolverPreparePath(path, prepared, reason, "checkpoint.path")) return(false); string segments[]; if(!DPCCResolverAppendSegments(prepared, segments, 0, reason, "checkpoint.path")) return(false); canonical = DPCCResolverJoinSegments(segments); if(StringLen(canonical) == 0 || canonical != prepared) { reason = "checkpoint.path.canonical"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Writes one fresh full native artifact and keeps failed bytes | //+------------------------------------------------------------------+ bool CheckpointSaveNative(CNet *net, const string path, const bool common_files, string &sha, string &reason) { if(FileIsExist(path, common_files ? FILE_COMMON : 0)) { reason = "checkpoint.destination_exists"; return(false); } const int writer = FileOpen(path, FILE_WRITE | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(writer == INVALID_HANDLE) { reason = "checkpoint.native.open"; return(false); } ResetLastError(); bool saved = net.Save(writer); FileFlush(writer); saved = (saved && GetLastError() == 0); FileClose(writer); if(!saved) { reason = "checkpoint.native.save_or_flush"; return(false); } return(DPCCCheckpointFileSHA(path, common_files, m_checkpoint_max_file_bytes, sha, reason)); } //+------------------------------------------------------------------+ //| Creates a native candidate borrowing the permanent decoder owner | //+------------------------------------------------------------------+ bool CheckpointCandidate(const bool decoder_role, const string path, const bool common_files, const uint expected_updates, const ulong expected_state, CNet *&candidate, string &reason) { candidate = new CNet; if(!candidate) { reason = "checkpoint.candidate.capacity"; return(false); } bool created = false; if(decoder_role) { SDPCCConfig config = m_config; config.shape.Depth = 0; CNeuronScenarioDecoderOCL *layer = NULL; created = DPCCCreateDecoderNet(candidate, config, m_checkpoint_decoder_composition, 1, m_decoder_net.GetOpenCL(), layer, reason); } else { CNeuronPlanDenoiserOCL *layer = NULL; created = DPCCCreateDenoiserNet(candidate, m_config, m_checkpoint_denoiser_composition, m_checkpoint_denoiser_composition, 1, m_decoder_net.GetOpenCL(), layer, reason); } if(!created) { delete candidate; candidate = NULL; return(false); } float error = 0, undefined = 0, forecast = 0; datetime time = 0; //--- Native filename overload owns the open/close lifecycle, including failure. if(!candidate.Load(path, error, undefined, forecast, time, common_files)) { delete candidate; candidate = NULL; reason = "checkpoint.candidate.native_load"; return(false); } bool admitted = false; if(decoder_role) { CNeuronScenarioDecoderOCL *layer = dynamic_cast(candidate.Layer(1)); admitted = (layer && layer.Type() == defNeuronScenarioDecoderOCL && candidate.Layer(2) == NULL && layer.ContractSignature() == m_decoder_contract && layer.UpdateCount() == expected_updates && layer.GenerationPhase() == 0 && layer.TrainableStateFingerprint() == expected_state); } else { CNeuronPlanDenoiserOCL *layer = dynamic_cast(candidate.Layer(1)); admitted = (layer && layer.Type() == defNeuronPlanDenoiserOCL && candidate.Layer(2) == NULL && layer.ContractSignature() == m_denoiser_contract && layer.UpdateCount() == expected_updates && layer.GenerationPhase() == 0 && layer.TrainableStateFingerprint() == expected_state); } if(!admitted || expected_state == 0 || candidate.GetOpenCL() != m_decoder_net.GetOpenCL()) { delete candidate; candidate = NULL; reason = "checkpoint.candidate.contract_or_full_state"; return(false); } CLayerDescription *input_info = candidate.Layer(0).GetLayerInfo(); CLayerDescription *role = candidate.Layer(1).GetLayerInfo(); const uint input_count = (decoder_role ? m_config.shape.K * m_config.shape.V * m_config.shape.H*m_config.shape.D : m_config.shape.H * 6); const bool descriptor_ok = (input_info && role && input_info.type == defNeuronBaseOCL && input_info.count == input_count && input_info.batch == 1 && input_info.optimization == ADAM && role.batch == 1 && role.optimization == ADAM && role.layers == (decoder_role ? 0 : 1) && ArraySize(role.windows) == 9 && role.windows[8] == 1); delete role; delete input_info; if(!descriptor_ok) { delete candidate; candidate = NULL; reason = "checkpoint.candidate.optimizer_or_input_descriptor"; return(false); } if(!decoder_role) { CNeuronPlanDenoiserOCL *layer = dynamic_cast(candidate.Layer(1)); const ulong state_before = layer.TrainableStateFingerprint(); const uint updates_before = layer.UpdateCount(); //--- Native bytes carry graph/affine/Adam state; the bound owner supplies condition scales. const bool configured = layer.Configure(m_config, m_checkpoint_denoiser_composition, m_checkpoint_denoiser_composition); const int configuration_status = (int)layer.LastStatus(); const ulong state_after = layer.TrainableStateFingerprint(); const uint updates_after = layer.UpdateCount(); const bool unchanged = (state_before == expected_state && state_after == state_before && updates_before == expected_updates && updates_after == updates_before && layer.ContractSignature() == m_denoiser_contract && layer.GenerationPhase() == 0 && candidate.GetOpenCL() == m_decoder_net.GetOpenCL()); if(!configured || configuration_status != (int)DPCC_OK || !unchanged) { reason = StringFormat("checkpoint.candidate.denoiser_configuration:status=%d:unchanged=%d", configuration_status, (int)unchanged); delete candidate; candidate = NULL; return(false); } PrintFormat("DPCC_CHECKPOINT_NATIVE_CONFIG_BOUND role=denoiser updates_before=%u updates_after=%u " "state_before=%s state_after=%s status=%d", updates_before, updates_after, (string)state_before, (string)state_after, configuration_status); } return(true); } //+-------------------------------------------------------------------+ //| Loss-only native probe reuses owned staged tensors and fixed keys | //+-------------------------------------------------------------------+ bool CheckpointProbeFailure(const string stage, CNeuronScenarioDecoderOCL *decoder, CNeuronPlanDenoiserOCL *denoiser, const bool candidate, string &reason) { const int terminal_error = GetLastError(); reason = stage + StringFormat(":decoder_status=%d:denoiser_status=%d:terminal_error=%d", (int)decoder.LastStatus(), (int)denoiser.LastStatus(), terminal_error); PrintFormat("DPCC_CHECKPOINT_PROBE_FAILURE candidate=%d stage=%s decoder_status=%d denoiser_status=%d " "decoder_phase=%u denoiser_phase=%u terminal_error=%d", (int)candidate, stage, (int)decoder.LastStatus(), (int)denoiser.LastStatus(), decoder.GenerationPhase(), denoiser.GenerationPhase(), terminal_error); return(false); } bool CheckpointProbe(CNet *decoder_net, CNet *denoiser_net, const int index, float &decoder_output[], float &denoiser_output[], double &decoder_loss, double &denoiser_loss, string &reason) { SDPCCTrainingExampleResult admission; if(!Begin(index, DPCC_TRAIN_VALIDATE, admission)) { reason = admission.reason; return(false); } CNeuronScenarioDecoderOCL *decoder = dynamic_cast(decoder_net.Layer(1)); CNeuronPlanDenoiserOCL *denoiser = dynamic_cast(denoiser_net.Layer(1)); if(!decoder || !denoiser || !decoder_net.TrainMode(false) || !denoiser_net.TrainMode(false)) { reason = "checkpoint.probe.model_or_mode"; return(false); } decoder_net.SetWeightsUpdate(false); denoiser_net.SetWeightsUpdate(false); const bool candidate = (denoiser_net != DenoiserNet()); ResetLastError(); if(!decoder_net.feedForward(GetPointer(m_z), 1, false, (CBufferFloat*)NULL)) return(CheckpointProbeFailure("checkpoint.probe.decoder.forward", decoder, denoiser, candidate, reason)); if(!decoder.BuildNativeBankRhoLoss(GetPointer(m_target), GetPointer(m_norm), GetPointer(m_rho), m_config.probability_tolerance, false, decoder_loss)) return(CheckpointProbeFailure("checkpoint.probe.decoder.loss", decoder, denoiser, candidate, reason)); if(decoder.getOutputVal(decoder_output) != (int)(m_config.shape.K * m_config.shape.H * 4)) return(CheckpointProbeFailure("checkpoint.probe.decoder.output", decoder, denoiser, candidate, reason)); const SDPCCTrainingFrozenRecordKeys k = m_record_keys[index]; float indexed[], epsilon[]; if(!DPCCBuildIndexedNoiseV1(m_validation_seed, k.decision_time, m_validation_repeat, 1, m_config.shape.J, m_config.shape.H, indexed) || ArrayResize(epsilon, m_config.shape.H * 6) != (int)(m_config.shape.H * 6)) { reason = "checkpoint.probe.noise"; return(false); } for(uint i = 0; i < m_config.shape.H * 6; i++) epsilon[i] = indexed[k.validation_j * m_config.shape.H * 6 + i]; if(!Upload(m_epsilon, epsilon)) return(CheckpointProbeFailure("checkpoint.probe.denoiser.upload", decoder, denoiser, candidate, reason)); if(!denoiser.PrepareTrainingExample(GetPointer(m_actions), GetPointer(m_epsilon), GetPointer(m_context), GetPointer(m_account), k.validation_j, m_alpha[k.validation_j], GetPointer(m_noisy), GetPointer(m_condition))) return(CheckpointProbeFailure("checkpoint.probe.denoiser.prepare", decoder, denoiser, candidate, reason)); if(!denoiser.SetContextIdentity(k.decision_identity, k.context_identity, k.owner_identity)) return(CheckpointProbeFailure("checkpoint.probe.denoiser.identity", decoder, denoiser, candidate, reason)); if(!denoiser_net.feedForward(GetPointer(m_noisy), 1, false, GetPointer(m_condition))) return(CheckpointProbeFailure("checkpoint.probe.denoiser.forward", decoder, denoiser, candidate, reason)); if(!denoiser.BuildEpsilonMSELoss(GetPointer(m_epsilon), false, denoiser_loss)) return(CheckpointProbeFailure("checkpoint.probe.denoiser.loss", decoder, denoiser, candidate, reason)); if(denoiser.getOutputVal(denoiser_output) != (int)(m_config.shape.H * 6)) return(CheckpointProbeFailure("checkpoint.probe.denoiser.output", decoder, denoiser, candidate, reason)); if(!MathIsValidNumber(decoder_loss) || !MathIsValidNumber(denoiser_loss)) return(CheckpointProbeFailure("checkpoint.probe.loss_nonfinite", decoder, denoiser, candidate, reason)); return(true); } bool CheckpointSameOutputs(const float &first[], const float &second[]) { if(ArraySize(first) != ArraySize(second)) return(false); for(int i = 0; i < ArraySize(first); i++) if(!MathIsValidNumber(first[i]) || !MathIsValidNumber(second[i]) || first[i] != second[i]) return(false); return(true); } //+------------------------------------------------------------------+ //| Resolves retained typed initial manifest and both native bytes | //+------------------------------------------------------------------+ bool CheckpointInitialArtifacts(const string path, const string expected_sha, const bool common_files, SDPCCCheckpointArtifact &initial, string &reason) { ZeroMemory(initial); if(!DPCCReadCheckpointArtifact(path, common_files, expected_sha, initial, reason) || initial.binding_sha256 != m_checkpoint_binding_sha || initial.cursor.stage != 0 || initial.cursor.decoder_updates != 0 || initial.cursor.denoiser_updates != 0) { reason = "checkpoint.initial.manifest_binding"; return(false); } string observed = ""; if(!DPCCCheckpointFileSHA(initial.decoder_path, common_files, m_checkpoint_max_file_bytes, observed, reason) || observed != initial.decoder_sha256) { reason = "checkpoint.initial.decoder_artifact"; return(false); } if(!DPCCCheckpointFileSHA(initial.denoiser_path, common_files, m_checkpoint_max_file_bytes, observed, reason) || observed != initial.denoiser_sha256) { reason = "checkpoint.initial.denoiser_artifact"; return(false); } return(true); } bool CheckpointFirstUpdateGate(string &reason) { SDPCCCheckpointArtifact initial; return(CheckpointInitialArtifacts(m_checkpoint_initial.manifest_path, m_checkpoint_initial.manifest_sha256, m_checkpoint_initial.common_files, initial, reason)); } bool Allocate(CBufferFloat &buffer, const uint count) {return(buffer.BufferInit(count, 0) && buffer.BufferCreate(DecoderNet().GetOpenCL()));} bool Upload(CBufferFloat &buffer, const float &values[]) { if(buffer.Total() != ArraySize(values)) return(false); for(int i = 0; i < ArraySize(values); i++) if(!buffer.Update(i, values[i])) return(false); return(buffer.BufferWrite()); } bool Fail(SDPCCTrainingExampleResult &result, const string stage, const string reason) {result.completed = false; result.stage = stage; result.reason = reason; return(false);} bool Begin(const int record_index, const ENUM_DPCC_TRAIN_OPERATION operation, SDPCCTrainingExampleResult &result) { ZeroMemory(result); result.stage = "admission"; result.bank_sha256 = m_integrity.bank_sha256; result.header_sha256 = m_integrity.header_sha256; if(!m_initialized || m_poisoned || record_index < 0 || record_index >= ArraySize(m_records)) return(Fail(result, "admission", "training.instance_or_record")); result.window_id = m_records[record_index].window_id; result.source_id = m_records[record_index].source_id; result.split = m_records[record_index].admission.split; result.decision_time = m_records[record_index].admission.decision_time; if(operation != DPCC_TRAIN_UPDATE && operation != DPCC_TRAIN_FROZEN && operation != DPCC_TRAIN_VALIDATE) return(Fail(result, "admission", "training.operation")); const string required = operation == DPCC_TRAIN_VALIDATE ? "validation" : "fit"; if(result.split != required) return(Fail(result, "admission", "training.split")); if(m_decoder.ContractSignature() != m_decoder_contract || m_denoiser.ContractSignature() != m_denoiser_contract) return(Fail(result, "admission", "training.factory_binding")); if(m_loaded == record_index) return(true); const SDPCCMaterializedRecord record = m_records[record_index]; float z[], norm[]; string packing_reason = ""; if(!DPCCPackDecoderInput(m_config.shape, record.context_without_noise, z, packing_reason)) return(Fail(result, "upload", packing_reason)); if(ArrayResize(norm, 16) != 16) return(Fail(result, "upload", "training.host_slice_capacity")); for(uint i = 0; i < 16; i++) norm[i] = record.context_without_noise[(uint)m_c.offsets[3] + i]; if(!Upload(m_z, z) || !Upload(m_norm, norm) || !Upload(m_target, record.target_native_ohlc) || !Upload(m_rho, record.rho) || !Upload(m_context, record.context_without_noise) || !Upload(m_account, record.account_features) || !Upload( m_actions, record.actions)) {m_loaded = -1; return(Fail(result, "upload", "training.original_tensor_upload"));} m_loaded = record_index; return(true); } bool ObserveDecoderCandidate(CNet *decoder, CNet *denoiser, const SDPCCCheckpointArtifact &artifact, const int index, SDPCCDecoderValidationObservation &out, string &reason) { if(CheckPointer(decoder) == POINTER_INVALID || CheckPointer(denoiser) == POINTER_INVALID || index < 0 || index >= ArraySize(m_records) || m_records[index].admission.split != "validation") {reason = "decoder.observe.candidate_or_record"; return(false);} float normalized[], discard[]; double loss = 0, unused = 0; bool ok = CheckpointProbe(decoder, denoiser, index, normalized, discard, loss, unused, reason); CNeuronScenarioDecoderOCL *layer = dynamic_cast(decoder.Layer(1)); CNeuronPlanDenoiserOCL *noise = dynamic_cast(denoiser.Layer(1)); SDPCCDecoderValidationObservation candidate; candidate.record_index = index; candidate.window_id = m_records[index].window_id; candidate.selected_manifest_sha = artifact.manifest_sha256; candidate.normalized_loss = loss; { const uint count = m_config.shape.K * m_config.shape.H * 4; CNeuronBatchNormOCL holder; CNeuronRevINDenormOCL restoration; ok = ok && layer != NULL && noise != NULL && ArraySize(normalized) == (int)count && ArrayResize(candidate.causal_price_norm, 16) == 16 && holder.Init(0, 2, decoder.GetOpenCL(), uint(4), uint(2), ADAM) && restoration.InitOwned(1, decoder.GetOpenCL(), count); if(ok) { holder.TrainMode(false); restoration.TrainMode(false); for(int i = 0; i < 16; i++) candidate.causal_price_norm[i] = m_records[index].context_without_noise[(int)m_c.offsets[3] + i]; CBufferFloat *ho = holder.getBatchOptions(), *ro = restoration.getBatchOptions(); ok = ho != NULL && ro != NULL && ho.Total() == 36 && ro.Total() == (int)(count * 9); for(uint row = 0; ok && row < count; row++) { const int channel = (int)(row % 4) * 4; const float mean = candidate.causal_price_norm[channel], std = candidate.causal_price_norm[channel + 1]; const float variance = (float)((double)std * std); ok = MathIsValidNumber(mean) && MathIsValidNumber(std) && std > 0 && MathIsValidNumber(variance) && variance > 0 && candidate.causal_price_norm[channel + 2] == 1 && candidate.causal_price_norm[channel + 3] == 0 && MathIsValidNumber(normalized[(int)row]); //--- Finite preflight only: observable restoration comes exclusively from native RevIN. const float product = (float)((double)(float)MathSqrt(variance) * normalized[(int)row]); ok = ok && MathIsValidNumber(product) && MathIsValidNumber((float)((double)mean + product)); for(int j = 0; ok && j < 9; j++) ok = ro.Update((int)row * 9 + j, 0); ok = ok && ro.Update((int)row * 9, mean) && ro.Update((int)row * 9 + 1, variance) && ro.Update(( int)row * 9 + 3, 1); if(row < 4) { for(int j = 0; ok && j < 9; j++) ok = ho.Update((int)row * 9 + j, 0); ok = ok && ho.Update((int)row * 9, mean) && ho.Update((int)row * 9 + 1, variance) && ho.Update(( int)row * 9 + 3, 1); } } ok = ok && ho.BufferWrite() && holder.SetBatchCount(1) && ro.BufferWrite() && restoration.FeedForward( layer.AsObject()) && restoration.getOutput() != NULL && restoration.getOutput().BufferRead() && ArrayResize( candidate.native_ohlc, (int)count) == (int)count; for(uint i = 0; ok && i < count; i++) {candidate.native_ohlc[(int)i] = restoration.getOutput().At((int)i); ok = MathIsValidNumber( candidate.native_ohlc[(int)i]);} if(ok) ok = ArrayCopy(candidate.normalized, normalized) == (int)count; } //--- Owning normalization dies while native candidate context still exists. } ok = ok && layer != NULL && noise != NULL && layer.TrainableStateFingerprint() == artifact.decoder_state && noise.TrainableStateFingerprint() == artifact.denoiser_state && layer.UpdateCount() == artifact.cursor.decoder_updates && noise.UpdateCount() == artifact.cursor.denoiser_updates; if(!ok) { if(StringLen(reason) == 0) reason = "decoder.observe.native_restoration_or_state"; return(false); } out = candidate; return(true); } bool WriteDecoderObservation(const int handle, const string value, const ulong budget, ulong &written, string &reason) { if((ulong)StringLen(value) > budget - written) {reason = "decoder.stream.budget"; return(false); } uchar bytes[]; const int count = StringToCharArray(value, bytes, 0, WHOLE_ARRAY, CP_UTF8); if(count <= 0 || (ulong)(count - 1) > budget - written || ArrayResize(bytes, count - 1) != count - 1 || FileWriteArray(handle, bytes, 0, count - 1) != (uint)(count - 1)) {reason = "decoder.stream.write_or_budget"; return(false);} written += (ulong)(count - 1); return(true); } public: CDPCCTrainingExamples(void) { m_decoder = NULL; m_denoiser = NULL; m_initialized = false; m_poisoned = false; m_decoder_initial_saved = false; m_denoiser_initial_saved = false; m_loaded = -1; m_tokens = 0; m_decoder_stream_native_loads = 0; m_checkpoint_decoder_net = NULL; m_checkpoint_denoiser_net = NULL; m_inference_decoder_net = NULL; m_inference_input = NULL; ZeroMemory(m_inference_artifact); m_checkpoint_metadata_bound = false; m_checkpoint_restored = false; m_checkpoint_max_file_bytes = 0; ZeroMemory(m_checkpoint_initial); } ~CDPCCTrainingExamples(void) { CloseDecoderInference(); delete m_checkpoint_denoiser_net; delete m_checkpoint_decoder_net; } bool Initialize(const SDPCCTrainingBankSource &source, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder_composition, const SDPCCDenoiserComposition &model_composition, const SDPCCDenoiserComposition &training_composition, const double schedule_s, const double beta_cap, const double &frozen_alpha[], const SDPCCTrainingFrozenRecordKeys &frozen_keys[], const ulong training_seed, const ulong validation_seed, const uint validation_repeat, string &reason) { reason = ""; if(m_initialized || m_poisoned || DecoderNet().Layer(0) != NULL || DenoiserNet().Layer(0) != NULL) {reason = "training.destination_not_empty"; return(false);} if(config.shape.PlanBatch != 1 || config.shape.J == 0 || config.shape.J > 64 || ArraySize(frozen_alpha) != (int)config.shape.J + 1 || frozen_alpha[0] != 1 || !DPCCCompositeLayouts(config.shape, m_c, m_r, m_tokens)) {reason = "training.frozen_shape_or_schedule"; return(false);} double canonical_beta[], canonical_alpha[], canonical_sigma[]; if(schedule_s != 0.008 || beta_cap != 0.999 || !DPCCBuildCosineDiffusionSchedule(config.shape.J, schedule_s, beta_cap, canonical_beta, canonical_alpha, canonical_sigma, reason)) {reason = "training.canonical_cosine_parameters"; return(false);} for(uint j = 0; j <= config.shape.J; j++) if(frozen_alpha[j] != canonical_alpha[j]) {reason = "training.frozen_cosine_schedule_mismatch"; return(false);} for(uint j = 1; j <= config.shape.J; j++) if(!MathIsValidNumber(frozen_alpha[j]) || frozen_alpha[j] <= 0 || frozen_alpha[j] >= frozen_alpha[j - 1] || (float)frozen_alpha[j] <= 0 || (float)frozen_alpha[j] >= 1) {reason = "training.alpha_numeric_or_order"; return(false);} //--- Bound two simultaneous materialization copies plus fixed-width arrays. //--- This is a tensor payload budget, not a measured total process RAM gate. const ulong H = config.shape.H, K = config.shape.K; const ulong floats = (ulong)m_c.offsets[10] + 13 + H * 4 + K + H * 12 + H; const ulong per_record = floats * sizeof(float) + H * sizeof(uint) + H * 4 * sizeof(long); if(source.maximum_records == 0 || source.maximum_staged_tensor_bytes == 0 || per_record > source.maximum_staged_tensor_bytes / 2 / source.maximum_records) {reason = "training.staged_tensor_budget"; return(false);} CDPCCBankMaterializationObserver observer; if(!observer.Configure(source.record_schema, source.header, source.header_schema, source.dataset, source.links, source.profile, source.bank, source.time_index, config, source.maximum_records, reason)) return(false); ulong invalid = 0; if(!DPCCStageMaterializedBank(observer, source.path, source.common_files, source.bank.bank_sha256, m_records, m_integrity, invalid, reason)) return(false); //--- Invalid records remain in bank integrity count and are not fabricated as examples. if(ArraySize(m_records) == 0) { reason = "training.no_valid_examples"; return(false); } if(ArraySize(frozen_keys) != ArraySize(m_records) || ArrayResize(m_record_keys, ArraySize(m_records)) != ArraySize(m_records)) {reason = "training.frozen_record_keys_capacity"; return(false);} for(int i = 0; i < ArraySize(m_records); i++) { const SDPCCTrainingFrozenRecordKeys key = frozen_keys[i]; if(key.window_id != m_records[i].window_id || key.decision_time != m_records[i].admission.decision_time || key.decision_identity == 0 || key.context_identity == 0 || key.owner_identity == 0 || (m_records[i].admission.split == "validation" ? (key.validation_j == 0 || key.validation_j > config.shape.J) : key.validation_j != 0)) {reason = "training.frozen_record_keys_binding"; return(false);} for(int k = 0; k < i; k++) if(frozen_keys[k].decision_identity == key.decision_identity) {reason = "training.frozen_decision_identity_duplicate"; return(false);} m_record_keys[i] = key; } m_config = config; m_training_seed = training_seed; m_validation_seed = validation_seed; m_validation_repeat = validation_repeat; if(ArrayCopy(m_alpha, frozen_alpha) != ArraySize(frozen_alpha)) { reason = "training.alpha_copy"; return(false); } //--- Decoder has no UShape depth; retain the denoiser role geometry unchanged. SDPCCConfig decoder_config = config; decoder_config.shape.Depth = 0; if(!DPCCCreateDecoderNet(m_decoder_net, decoder_config, decoder_composition, 1, NULL, m_decoder, reason) || !DPCCCreateDenoiserNet(m_denoiser_net, config, model_composition, training_composition, 1, DecoderNet().GetOpenCL(), m_denoiser, reason)) {m_poisoned = true; return(false);} m_decoder_contract = m_decoder.ContractSignature(); m_denoiser_contract = m_denoiser.ContractSignature(); if(m_decoder_contract == 0 || m_denoiser_contract == 0 || !Allocate(m_z, (uint)m_c.lengths[0]) || !Allocate( m_norm, 16) || !Allocate(m_target, config.shape.H * 4) || !Allocate(m_rho, config.shape.K) || !Allocate(m_context, (uint)m_c.offsets[10]) || !Allocate(m_account, 13) || !Allocate(m_actions, config.shape.H * 6) || !Allocate(m_epsilon, config.shape.H * 6) || !Allocate(m_noisy, config.shape.H * 6) || !Allocate(m_condition, m_tokens + 61)) {m_poisoned = true; reason = "training.device_allocation"; return(false);} m_checkpoint_decoder_composition = decoder_composition; m_checkpoint_denoiser_composition = model_composition; m_checkpoint_dataset_id = source.dataset.dataset_id; m_epoch_time_index_id = source.links.time_index_id; m_checkpoint_normalization_id = source.dataset.normalization_id; m_checkpoint_bank_id = source.bank.bank_id; m_checkpoint_dataset_sha = source.links.dataset_sha256; m_checkpoint_timeindex_sha = source.links.time_index_sha256; string index_text = "DPCC_TIME_INDEX_TYPED_V1\n"; for(int i = 0; i < ArraySize(source.time_index.bar_index); i++) index_text += (string)source.time_index.bar_index[i] + ":" + (string)source.time_index.start_utc[i] + ":" + (string)source.time_index.end_utc[i] + "\n"; if(!DPCCCheckpointTextSHA(index_text, m_checkpoint_timeindex_table_sha, reason)) return(false); m_initialized = true; return(true); } int RecordCount(void) const {return(m_initialized ? ArraySize(m_records) : 0);} //--- Read-only access to one admitted row; the full bank remains private. bool AdmittedRecordCount(int &count, string &reason) const { count = 0; reason = ""; if(!m_initialized) {reason = "training.records_not_admitted"; return(false);} if(m_poisoned) {reason = "training.records_poisoned"; return(false);} count = ArraySize(m_records); if(count <= 0) {count = 0; reason = "training.records_empty"; return(false);} return(true); } bool CopyAdmittedRecord(const int index, SDPCCMaterializedRecord &record, string &reason) const { DPCCClearMaterializedRecord(record); reason = ""; if(!m_initialized) {reason = "training.records_not_admitted"; return(false);} if(m_poisoned) {reason = "training.records_poisoned"; return(false);} if(index < 0 || index >= ArraySize(m_records)) {reason = "training.record_index"; return(false);} record = m_records[index]; return(true); } //--- Read-only validation of EVERY admitted original record before the owner can UPDATE. //--- Exact float bits use the native holder's effective std, never a tolerance. bool CheckFixedPriceNorm(const float &profile[], const long cutoff, string &reason) { if(!m_initialized || m_poisoned || ArraySize(profile) != 16 || cutoff < 0 || ArraySize(m_records) == 0) {reason = "BLOCKED.training.fixed_profile_admission_required"; return(false);} for(int record = 0; record < ArraySize(m_records); record++) { if(m_records[record].normalization_asof < cutoff || m_records[record].normalization_asof > m_records[record].admission.decision_time) {reason = "BLOCKED.training.fixed_normalization_cutoff.record_" + (string)record; return(false);} for(int field = 0; field < 16; field++) { URAGFloatBits expected, actual; expected.value = profile[field]; actual.value = m_records[record].context_without_noise[(int)m_c.offsets[3] + field]; if(expected.bits != actual.bits) {reason = "BLOCKED.training.fixed_PriceNorm.record_" + (string)record + ".field_" + (string)field; return( false);} } } reason = ""; return(true); } //--- Contract-only copy: FIT_FIXED never obtains statistics from a bank record. bool PreparedActorContract(SDPCCConfig &config, SDPCCDecoderComposition &decoder, SDPCCDenoiserComposition &denoiser, string &reason) { if(!m_initialized || m_poisoned) {reason = "actor.examples_contract_not_admitted"; return(false);} config = m_config; decoder = m_checkpoint_decoder_composition; denoiser = m_checkpoint_denoiser_composition; reason = ""; return(true); } bool PreparedActorProfile(const int index, SDPCCConfig &config, SDPCCDecoderComposition &decoder, SDPCCDenoiserComposition &denoiser, float &norm[], string &reason) { if(!m_initialized || m_poisoned || index < 0 || index >= ArraySize(m_records) || ArrayResize(norm, 16) != 16) { reason = "actor.owner_causal_record"; return(false); } config = m_config; decoder = m_checkpoint_decoder_composition; denoiser = m_checkpoint_denoiser_composition; for(int i = 0; i < 16; i++) norm[i] = m_records[index].context_without_noise[(int)m_c.offsets[3] + i]; return(true); } bool RecordIdentity(const int index, SDPCCTrainingRecordIdentity &result) { ZeroMemory(result); if(!m_initialized || m_poisoned || index < 0 || index >= ArraySize(m_records)) return(false); result.record_index = index; result.window_id = m_records[index].window_id; result.source_id = m_records[index].source_id; result.split = m_records[index].admission.split; result.decision_time = m_records[index].admission.decision_time; result.decision_bar_index = m_records[index].admission.decision_bar_index; result.bank_sha256 = m_integrity.bank_sha256; result.header_sha256 = m_integrity.header_sha256; result.keys = m_record_keys[index]; return(true); } bool RunBinding(const string dataset_id, const string bank_id, const string normalization_id, const string dataset_sha256, const SDPCCBankMetadataResult &bank, const SDPCCTrainingModelBinding &model, const string time_index_sha256, const SDPCCDatasetTimeIndex &time_index, const SDPCCArchitectureContract &a, const string training_seed, const string validation_seed, const uint validation_repeat, const uint J, const bool denoiser_keys, string &reason) { if(!m_initialized || m_poisoned || dataset_id != m_checkpoint_dataset_id || bank_id != m_checkpoint_bank_id || normalization_id != m_checkpoint_normalization_id || dataset_sha256 != m_checkpoint_dataset_sha || J != m_config.shape.J || (denoiser_keys && (training_seed != StringFormat("%016I64x", m_training_seed) || validation_seed != StringFormat("%016I64x", m_validation_seed) || validation_repeat != m_validation_repeat))) {reason = "training.owner_run_binding"; return(false);} if(bank.bank_id != m_checkpoint_bank_id || bank.dataset_id != m_checkpoint_dataset_id || bank.time_index_id != m_epoch_time_index_id || bank.bank_sha256 != m_integrity.bank_sha256 || bank.header_sha256 != m_integrity.header_sha256 || bank.record_count < 0 || (ulong)bank.record_count != m_integrity.record_count) {reason = "training.owner_bank_receipt"; return(false);} if(time_index_sha256 != m_checkpoint_timeindex_sha || time_index.time_index_id != m_epoch_time_index_id || !DPCCValidateDatasetTimeIndex(time_index, m_checkpoint_dataset_id, reason)) return(false); for(int r = 0; r < ArraySize(m_records); r++) { long decision = m_records[r].admission.decision_bar_index; if(decision < 0 || decision >= ArraySize(time_index.bar_index) || m_records[r].admission.decision_time != time_index.start_utc[(int)decision]) {reason = "training.owner_decision_time_index"; return(false);} for(int t = 0; t < ArraySize(m_records[r].bar_indices); t++) { long bar = m_records[r].bar_indices[t]; if(bar < 0 || bar >= ArraySize(time_index.bar_index) || time_index.bar_index[(int)bar] != bar || time_index.start_utc[(int)bar] != m_records[r].bar_open_times[t] || time_index.end_utc[(int)bar] != m_records[r].bar_close_times[t]) {reason = "training.owner_target_time_index"; return(false);} } } if(model.price_basis != m_config.price_basis || model.volume_scale != m_config.volume_scale || model.price_scale != m_config.price_scale || model.money_scale != m_config.money_scale || model.probability_tolerance != m_config.probability_tolerance || model.schedule_s != 0.008 || model.beta_cap != 0.999) {reason = "training.owner_model_scales"; return(false);} for(int i = 0; i < 6; i++) if(model.action_bias[i] != m_config.action_bias[i] || model.action_scale[i] != m_config.action_scale[i]) {reason = "training.owner_action_affine"; return(false);} double beta[], alpha[], sigma[]; if(!DPCCBuildCosineDiffusionSchedule(J, model.schedule_s, model.beta_cap, beta, alpha, sigma, reason) || ArraySize(model.beta) != (int)J + 1 || ArraySize(model.alpha_bar) != (int)J + 1 || ArraySize(model.sigma) != ( int)J + 1) {reason = "training.owner_schedule_shape"; return(false);} for(uint i = 0; i <= J; i++) if(model.beta[i] != beta[i] || model.alpha_bar[i] != m_alpha[i] || model.sigma[i] != sigma[i]) {reason = "training.owner_schedule_values"; return(false);} SDPCCDecoderArchitecture d; d.architecture = m_checkpoint_decoder_composition.architecture; d.embedding = m_checkpoint_decoder_composition.embedding; d.activation = m_checkpoint_decoder_composition.activation_profile; d.backward_profile = m_checkpoint_decoder_composition.backward_profile; return(DPCCCompareDecoderArchitecture(d, a.training_decoder, "training.owner_decoder", reason) && DPCCCompareDenoiserArchitecture(m_checkpoint_denoiser_composition.graph, a.training_denoiser, "training.owner_denoiser", reason) && DPCCCompareParallelExecution(m_checkpoint_denoiser_composition.parallel, a.training_parallel, "training.owner_parallel", reason)); } bool SaveDecoderCheckpoint(const int writer, string &reason) { reason = ""; if(!m_initialized || m_poisoned || writer == INVALID_HANDLE || m_decoder.GenerationPhase() != 0 || !DecoderNet( ).Save(writer)) {reason = "training.decoder_checkpoint"; return(false);} ResetLastError(); FileFlush(writer); if(GetLastError() != 0) { reason = "training.decoder_checkpoint_flush"; return(false); } //--- Legacy writer success does not authorize the first Adam update. return(true); } bool SaveDenoiserCheckpoint(const int writer, string &reason) { reason = ""; if(!m_initialized || m_poisoned || writer == INVALID_HANDLE || m_denoiser.GenerationPhase() != 0 || !DenoiserNet( ).Save(writer)) {reason = "training.denoiser_checkpoint"; return(false);} ResetLastError(); FileFlush(writer); if(GetLastError() != 0) { reason = "training.denoiser_checkpoint_flush"; return(false); } //--- Legacy writer success does not authorize the first Adam update. return(true); } //+------------------------------------------------------------------+ //| Binds external admitted metadata without granting READY | //+------------------------------------------------------------------+ bool BindCheckpointMetadata(const SDPCCCheckpointBinding &binding, const ulong maximum_file_bytes, string &reason) { reason = ""; if(!m_initialized || m_poisoned || m_checkpoint_metadata_bound || maximum_file_bytes == 0 || m_decoder.UpdateCount() != 0 || m_denoiser.UpdateCount() != 0 || binding.dataset_manifest_sha256 != m_checkpoint_dataset_sha || binding.time_index_sha256 != m_checkpoint_timeindex_sha) { reason = "checkpoint.binding.owner_state"; return(false); } string digest = ""; if(!CheckpointBindingSHA(binding, digest, reason)) return(false); m_checkpoint_binding = binding; m_checkpoint_binding_sha = digest; m_checkpoint_max_file_bytes = maximum_file_bytes; m_checkpoint_metadata_bound = true; return(true); } bool NativeState(uint &decoder_updates, uint &denoiser_updates, ulong &decoder_state, ulong &denoiser_state) { decoder_updates = 0; denoiser_updates = 0; decoder_state = 0; denoiser_state = 0; if(!m_initialized || m_poisoned || m_decoder.GenerationPhase() != 0 || m_denoiser.GenerationPhase() != 0) return(false); decoder_updates = m_decoder.UpdateCount(); denoiser_updates = m_denoiser.UpdateCount(); decoder_state = m_decoder.TrainableStateFingerprint(); denoiser_state = m_denoiser.TrainableStateFingerprint(); return(decoder_state != 0 && denoiser_state != 0); } //+-------------------------------------------------------------------+ //| Saves both native roles; publishes manifest and initial gate last | //+-------------------------------------------------------------------+ bool SaveVerifiedCheckpointArtifact(const string stem, const bool common_files, const SDPCCCheckpointCursor &cursor, SDPCCCheckpointArtifact &artifact, string &reason) { ZeroMemory(artifact); reason = ""; if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || cursor.stage > 2 || cursor.next_record != 0 || cursor.completed_epoch == UINT_MAX || cursor.next_epoch != cursor.completed_epoch + 1 || (cursor.stage == 0 && (cursor.completed_epoch != 0 || cursor.decoder_updates != 0 || cursor.denoiser_updates != 0))) { reason = "checkpoint.save.owner_or_epoch_boundary"; return(false); } uint decoder_updates = 0, denoiser_updates = 0; ulong decoder_state = 0, denoiser_state = 0; if(!NativeState(decoder_updates, denoiser_updates, decoder_state, denoiser_state) || decoder_updates != cursor.decoder_updates || denoiser_updates != cursor.denoiser_updates) { reason = "checkpoint.save.counter_binding"; return(false); } const bool initial = (cursor.stage == 0); if(initial ? StringLen(m_checkpoint_initial.manifest_sha256) > 0 : StringLen( m_checkpoint_initial.manifest_sha256) == 0) { reason = "checkpoint.save.initial_artifact_required_or_already_preserved"; PrintFormat("DPCC_CHECKPOINT_INITIAL_GATE initial=%d retained_sha_len=%d eq_empty=%d eq_null=%d", (int)initial, StringLen(m_checkpoint_initial.manifest_sha256), (int)(m_checkpoint_initial.manifest_sha256 == ""), (int)(m_checkpoint_initial.manifest_sha256 == NULL)); return(false); } if(!initial && common_files != m_checkpoint_initial.common_files) { reason = "checkpoint.save.storage_namespace"; return(false); } if(!initial) { SDPCCCheckpointArtifact retained_initial; if(!CheckpointInitialArtifacts(m_checkpoint_initial.manifest_path, m_checkpoint_initial.manifest_sha256, common_files, retained_initial, reason)) return(false); } string canonical = ""; if(!CheckpointPath(stem, canonical, reason)) return(false); artifact.manifest_path = canonical + ".dpccstate"; artifact.decoder_path = canonical + ".decoder.bin"; artifact.denoiser_path = canonical + ".denoiser.bin"; artifact.common_files = common_files; artifact.cursor = cursor; artifact.binding_sha256 = m_checkpoint_binding_sha; artifact.decoder_state = decoder_state; artifact.denoiser_state = denoiser_state; if(!initial) { artifact.initial_manifest_path = m_checkpoint_initial.manifest_path; artifact.initial_manifest_sha256 = m_checkpoint_initial.manifest_sha256; } if(FileIsExist(artifact.manifest_path, common_files ? FILE_COMMON : 0) || FileIsExist(artifact.decoder_path, common_files ? FILE_COMMON : 0) || FileIsExist(artifact.denoiser_path, common_files ? FILE_COMMON : 0)) { reason = "checkpoint.destination_exists"; return(false); } int validation_index = -1; for(int i = 0; i < ArraySize(m_records); i++) if(m_records[i].admission.split == "validation") { validation_index = i; break; } float reference_decoder[], reference_denoiser[]; if(validation_index < 0 || !CheckpointProbe(DecoderNet(), DenoiserNet(), validation_index, reference_decoder, reference_denoiser, artifact.decoder_probe_loss, artifact.denoiser_probe_loss, reason)) { m_poisoned = true; if(StringLen(reason) == 0) reason = "checkpoint.probe.validation_missing"; return(false); } if(!DPCCCheckpointOutputSHA(reference_decoder, artifact.decoder_probe_sha256, reason) || !DPCCCheckpointOutputSHA(reference_denoiser, artifact.denoiser_probe_sha256, reason)) return(false); if(!CheckpointSaveNative(DecoderNet(), artifact.decoder_path, common_files, artifact.decoder_sha256, reason)) return(false); if(!CheckpointSaveNative(DenoiserNet(), artifact.denoiser_path, common_files, artifact.denoiser_sha256, reason)) return(false); CNet *decoder_candidate = NULL; CNet *denoiser_candidate = NULL; if(!CheckpointCandidate(true, artifact.decoder_path, common_files, decoder_updates, decoder_state, decoder_candidate, reason)) return(false); if(!CheckpointCandidate(false, artifact.denoiser_path, common_files, denoiser_updates, denoiser_state, denoiser_candidate, reason)) { delete decoder_candidate; return(false); } float candidate_decoder[], candidate_denoiser[]; double decoder_loss = 0, denoiser_loss = 0; bool verified = CheckpointProbe(decoder_candidate, denoiser_candidate, validation_index, candidate_decoder, candidate_denoiser, decoder_loss, denoiser_loss, reason); if(verified) verified = (decoder_loss == artifact.decoder_probe_loss && denoiser_loss == artifact.denoiser_probe_loss && CheckpointSameOutputs(reference_decoder, candidate_decoder) && CheckpointSameOutputs(reference_denoiser, candidate_denoiser)); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(decoder_candidate.Layer(1)); CNeuronPlanDenoiserOCL *denoiser = dynamic_cast(denoiser_candidate.Layer(1)); if(verified) verified = (decoder.TrainableStateFingerprint() == decoder_state && denoiser.TrainableStateFingerprint() == denoiser_state && m_decoder.TrainableStateFingerprint() == decoder_state && m_denoiser.TrainableStateFingerprint() == denoiser_state); delete denoiser_candidate; delete decoder_candidate; if(!verified) { if(StringLen(reason) == 0) reason = "checkpoint.readback.native_state_or_output"; return(false); } string actual = ""; if(!DPCCCheckpointFileSHA(artifact.decoder_path, common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.decoder_sha256) { reason = "checkpoint.readback.decoder_changed"; return(false); } if(!DPCCCheckpointFileSHA(artifact.denoiser_path, common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.denoiser_sha256) { reason = "checkpoint.readback.denoiser_changed"; return(false); } const string text = DPCCCheckpointManifest(artifact); if(!DPCCCheckpointWriteText(artifact.manifest_path, common_files, text, reason)) return(false); string expected = ""; if(!DPCCCheckpointTextSHA(text, expected, reason) || !DPCCCheckpointFileSHA(artifact.manifest_path, common_files, 65536, actual, reason) || actual != expected) { reason = "checkpoint.manifest.readback"; return(false); } artifact.manifest_sha256 = actual; if(initial) { m_checkpoint_initial = artifact; m_decoder_initial_saved = true; m_denoiser_initial_saved = true; } return(true); } //+------------------------------------------------------------------+ //| Admits every artifact before loading any candidate native state | //+------------------------------------------------------------------+ bool RestoreTrainingCheckpoint(const SDPCCCheckpointArtifact &artifact, SDPCCCheckpointCursor &cursor, string &reason) { ZeroMemory(cursor); reason = ""; if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || m_checkpoint_restored || m_decoder.UpdateCount() != 0 || m_denoiser.UpdateCount() != 0 || artifact.binding_sha256 != m_checkpoint_binding_sha || artifact.cursor.stage > 2 || artifact.cursor.next_record != 0 || artifact.cursor.completed_epoch == UINT_MAX || artifact.cursor.next_epoch != artifact.cursor.completed_epoch + 1 || !DPCCResolverLowerSha256(artifact.manifest_sha256) || !DPCCResolverLowerSha256(artifact.decoder_sha256) || !DPCCResolverLowerSha256(artifact.denoiser_sha256) || !DPCCResolverLowerSha256(artifact.decoder_probe_sha256) || !DPCCResolverLowerSha256( artifact.denoiser_probe_sha256)) { reason = "checkpoint.restore.binding_or_destination"; return(false); } string canonical = ""; if(!CheckpointPath(artifact.manifest_path, canonical, reason) || !CheckpointPath(artifact.decoder_path, canonical, reason) || !CheckpointPath(artifact.denoiser_path, canonical, reason) || artifact.decoder_path == artifact.denoiser_path || artifact.decoder_path == artifact.manifest_path || artifact.denoiser_path == artifact.manifest_path) return(false); string expected = "", actual = ""; if(!DPCCCheckpointTextSHA(DPCCCheckpointManifest(artifact), expected, reason) || expected != artifact.manifest_sha256 || !DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, actual, reason) || actual != expected) { reason = "checkpoint.restore.manifest"; return(false); } if(artifact.cursor.stage == 0) { if(artifact.cursor.completed_epoch != 0 || artifact.cursor.decoder_updates != 0 || artifact.cursor.denoiser_updates != 0 || StringLen(artifact.initial_manifest_path) != 0 || StringLen(artifact.initial_manifest_sha256) != 0) { reason = "checkpoint.restore.initial_state"; return(false); } } SDPCCCheckpointArtifact retained_initial; if(artifact.cursor.stage == 0) { if(!CheckpointInitialArtifacts(artifact.manifest_path, artifact.manifest_sha256, artifact.common_files, retained_initial, reason)) return(false); } else { if(!CheckpointInitialArtifacts(artifact.initial_manifest_path, artifact.initial_manifest_sha256, artifact.common_files, retained_initial, reason)) return(false); } if(!DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.decoder_sha256) { reason = "checkpoint.restore.decoder_artifact"; return(false); } if(!DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.denoiser_sha256) { reason = "checkpoint.restore.denoiser_artifact"; return(false); } CNet *decoder_candidate = NULL; CNet *denoiser_candidate = NULL; if(!CheckpointCandidate(true, artifact.decoder_path, artifact.common_files, artifact.cursor.decoder_updates, artifact.decoder_state, decoder_candidate, reason)) { m_poisoned = true; return(false); } if(!CheckpointCandidate(false, artifact.denoiser_path, artifact.common_files, artifact.cursor.denoiser_updates, artifact.denoiser_state, denoiser_candidate, reason)) { delete decoder_candidate; m_poisoned = true; return(false); } int validation_index = -1; for(int i = 0; i < ArraySize(m_records); i++) if(m_records[i].admission.split == "validation") { validation_index = i; break; } float decoder_output[], denoiser_output[]; double decoder_loss = 0, denoiser_loss = 0; string decoder_sha = "", denoiser_sha = ""; bool verified = (validation_index >= 0 && CheckpointProbe(decoder_candidate, denoiser_candidate, validation_index, decoder_output, denoiser_output, decoder_loss, denoiser_loss, reason)); if(verified) verified = (DPCCCheckpointOutputSHA(decoder_output, decoder_sha, reason) && DPCCCheckpointOutputSHA(denoiser_output, denoiser_sha, reason) && decoder_sha == artifact.decoder_probe_sha256 && denoiser_sha == artifact.denoiser_probe_sha256 && decoder_loss == artifact.decoder_probe_loss && denoiser_loss == artifact.denoiser_probe_loss); CNeuronScenarioDecoderOCL *decoder = dynamic_cast(decoder_candidate.Layer(1)); CNeuronPlanDenoiserOCL *denoiser = dynamic_cast(denoiser_candidate.Layer(1)); if(verified) verified = (decoder.TrainableStateFingerprint() == artifact.decoder_state && denoiser.TrainableStateFingerprint() == artifact.denoiser_state); if(verified) verified = (DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) && actual == artifact.decoder_sha256); if(verified) verified = (DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) && actual == artifact.denoiser_sha256); if(verified) verified = (DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, actual, reason) && actual == artifact.manifest_sha256); if(verified) { SDPCCCheckpointArtifact final_initial; verified = CheckpointInitialArtifacts(retained_initial.manifest_path, retained_initial.manifest_sha256, artifact.common_files, final_initial, reason); } if(!verified) { delete denoiser_candidate; delete decoder_candidate; m_poisoned = true; if(StringLen(reason) == 0) reason = "checkpoint.restore.native_readback"; return(false); } //--- Permanent original decoder retains the shared context for every borrowed tensor/net. m_checkpoint_decoder_net = decoder_candidate; m_checkpoint_denoiser_net = denoiser_candidate; m_decoder = decoder; m_denoiser = denoiser; m_decoder_initial_saved = true; m_denoiser_initial_saved = true; m_checkpoint_restored = true; m_loaded = -1; m_checkpoint_initial = retained_initial; cursor = artifact.cursor; return(true); } //--- C2 read-only candidate verification; never adopts either native model. bool VerifyRunnerArtifact(const SDPCCCheckpointArtifact &artifact, double &decoder_loss, double &denoiser_loss, string &reason) { decoder_loss = 0; denoiser_loss = 0; if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || artifact.binding_sha256 != m_checkpoint_binding_sha) {reason = "epoch.artifact.binding"; return(false);} string actual = ""; if(!DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.denoiser_sha256) {reason = "epoch.artifact.native_sha"; return(false);} CNet *decoder = NULL, *denoiser = NULL; if(!CheckpointCandidate(true, artifact.decoder_path, artifact.common_files, artifact.cursor.decoder_updates, artifact.decoder_state, decoder, reason)) return(false); if(!CheckpointCandidate(false, artifact.denoiser_path, artifact.common_files, artifact.cursor.denoiser_updates, artifact.denoiser_state, denoiser, reason)) {delete decoder; return(false);} bool ok = true; uint count = 0; for(int i = 0; ok && i < ArraySize(m_records); i++) if(m_records[i].admission.split == "validation") { float dout[], nout[]; double dl = 0, nl = 0; ok = CheckpointProbe(decoder, denoiser, i, dout, nout, dl, nl, reason); if(ok) { decoder_loss += dl; denoiser_loss += nl; count++; } } CNeuronScenarioDecoderOCL *d = dynamic_cast(decoder.Layer(1)); CNeuronPlanDenoiserOCL *n = dynamic_cast(denoiser.Layer(1)); ok = ok && count > 0 && MathIsValidNumber(decoder_loss) && MathIsValidNumber(denoiser_loss) && d.TrainableStateFingerprint() == artifact.decoder_state && n.TrainableStateFingerprint() == artifact.denoiser_state && d.UpdateCount() == artifact.cursor.decoder_updates && n.UpdateCount() == artifact.cursor.denoiser_updates; delete denoiser; delete decoder; if(!ok) { if(StringLen(reason) == 0) reason = "epoch.artifact.validation"; return(false); } if(!DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, actual, reason) || actual != artifact.denoiser_sha256 || !DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, actual, reason) || actual != artifact.manifest_sha256) {reason = "epoch.artifact.changed_after_probe"; return(false);} decoder_loss /= count; denoiser_loss /= count; return(true); } //+------------------------------------------------------------------+ //| Opens one frozen selected decoder for admitted causal rows | //+------------------------------------------------------------------+ bool BeginSelectedDecoderInference(const SDPCCCheckpointArtifact &artifact, string &reason) { reason = ""; if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || artifact.binding_sha256 != m_checkpoint_binding_sha || artifact.cursor.stage != 1) return(DecoderInferenceFailure("decoder.infer.admission", reason)); if(m_inference_decoder_net != NULL || m_inference_input != NULL) return(DecoderInferenceFailure("decoder.infer.session_already_open", reason)); if(!DecoderInferenceArtifact(artifact, reason)) { CloseDecoderInference(); return(false); } SDPCCCheckpointArtifact actual; if(!DPCCReadCheckpointArtifact(artifact.manifest_path, artifact.common_files, artifact.manifest_sha256, actual, reason) || DPCCCheckpointManifest(actual) != DPCCCheckpointManifest(artifact)) return(DecoderInferenceFailure("decoder.infer.manifest_binding", reason)); if(!CheckpointCandidate(true, artifact.decoder_path, artifact.common_files, artifact.cursor.decoder_updates, artifact.decoder_state, m_inference_decoder_net, reason)) { CloseDecoderInference(); return(false); } if(!m_inference_decoder_net.TrainMode(false)) return(DecoderInferenceFailure("decoder.infer.frozen_mode", reason)); if(!m_inference_decoder_net.SetWeightsUpdate(false)) return(DecoderInferenceFailure("decoder.infer.freeze_weights", reason)); m_inference_input = new CBufferFloat; if(m_inference_input == NULL) return(DecoderInferenceFailure("decoder.infer.input_capacity", reason)); if(!Allocate(m_inference_input, (uint)m_c.lengths[0])) return(DecoderInferenceFailure("decoder.infer.input_allocate", reason)); m_inference_artifact = artifact; return(true); } //+------------------------------------------------------------------+ //| Copies one frozen decoder output without targets or denoiser | //+------------------------------------------------------------------+ bool InferSelectedDecoderRow(const int index, SDPCCDecoderInferenceObservation &out, string &reason) { out.record_index = -1; out.window_id = ""; out.selected_manifest_sha = ""; out.decision_time = 0; ArrayResize(out.normalized, 0); reason = ""; if(!m_initialized || m_poisoned || m_inference_decoder_net == NULL || m_inference_input == NULL || index < 0 || index >= ArraySize(m_records)) return(DecoderInferenceFailure("decoder.infer.row_admission", reason)); if(m_records[index].admission.split != "fit") return(DecoderInferenceFailure("decoder.infer.fit_row_required", reason)); CNeuronScenarioDecoderOCL *layer = dynamic_cast(m_inference_decoder_net.Layer(1)); if(layer == NULL || layer.ContractSignature() != m_decoder_contract || layer.TrainableStateFingerprint() != m_inference_artifact.decoder_state || layer.UpdateCount() != m_inference_artifact.cursor.decoder_updates || layer.GenerationPhase() != 0) return(DecoderInferenceFailure("decoder.infer.state_before", reason)); //--- Upload only the causal section into the persistent session buffer. float causal_values[]; if(!DPCCPackDecoderInput(m_config.shape, m_records[index].context_without_noise, causal_values, reason)) { CloseDecoderInference(); return(false); } if(!Upload(m_inference_input, causal_values)) return(DecoderInferenceFailure("decoder.infer.input_upload", reason)); if(!m_inference_decoder_net.feedForward(m_inference_input, 1, false, (CBufferFloat*)NULL)) return(DecoderInferenceFailure("decoder.infer.forward", reason)); SDPCCDecoderInferenceObservation observed; const int count = (int)(m_config.shape.K * m_config.shape.H * 4); if(layer.getOutputVal(observed.normalized) != count) return(DecoderInferenceFailure("decoder.infer.output", reason)); for(int i = 0; i < ArraySize(observed.normalized); i++) if(!MathIsValidNumber((double)observed.normalized[i])) return(DecoderInferenceFailure("decoder.infer.output_nonfinite", reason)); if(layer.ContractSignature() != m_decoder_contract || layer.TrainableStateFingerprint() != m_inference_artifact.decoder_state || layer.UpdateCount() != m_inference_artifact.cursor.decoder_updates || layer.GenerationPhase() != 0) return(DecoderInferenceFailure("decoder.infer.state_after", reason)); //--- Return an owned copy before the next row replaces the native output. observed.record_index = index; observed.window_id = m_records[index].window_id; observed.selected_manifest_sha = m_inference_artifact.manifest_sha256; observed.decision_time = m_records[index].admission.decision_time; out = observed; return(true); } //+------------------------------------------------------------------+ //| Verifies artifacts and releases the frozen inference session | //+------------------------------------------------------------------+ bool EndSelectedDecoderInference(string &reason) { reason = ""; if(m_inference_decoder_net == NULL || m_inference_input == NULL) return(DecoderInferenceFailure("decoder.infer.session_not_open", reason)); const bool ok = DecoderInferenceArtifact(m_inference_artifact, reason); CloseDecoderInference(); return(ok); } bool ObserveSelectedDecoder(const SDPCCCheckpointArtifact &artifact, const int index, SDPCCDecoderValidationObservation &out, string &reason) { out.record_index = -1; out.window_id = ""; out.selected_manifest_sha = ""; out.normalized_loss = 0; ArrayResize(out.normalized, 0); ArrayResize(out.native_ohlc, 0); ArrayResize(out.causal_price_norm, 0); if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || artifact.binding_sha256 != m_checkpoint_binding_sha || artifact.cursor.stage != 1 || index < 0 || index >= ArraySize(m_records) || m_records[index].admission.split != "validation") {reason = "decoder.observe.admission"; return(false);} string sha = ""; if(!DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, sha, reason) || sha != artifact.manifest_sha256 || !DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.denoiser_sha256) return(false); CNet *decoder = NULL, *denoiser = NULL; if(!CheckpointCandidate(true, artifact.decoder_path, artifact.common_files, artifact.cursor.decoder_updates, artifact.decoder_state, decoder, reason)) return(false); if(!CheckpointCandidate(false, artifact.denoiser_path, artifact.common_files, artifact.cursor.denoiser_updates, artifact.denoiser_state, denoiser, reason)) {delete decoder; return(false);} SDPCCDecoderValidationObservation observed; const bool ok = ObserveDecoderCandidate(decoder, denoiser, artifact, index, observed, reason); delete denoiser; delete decoder; if(!ok) return(false); if(!DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, sha, reason) || sha != artifact.manifest_sha256 || !DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.denoiser_sha256) {reason = "decoder.observe.artifact_changed"; return(false);} out = observed; return(true); } bool StreamSelectedDecoder(const SDPCCCheckpointArtifact &artifact, const int &indices[], const string path, const ulong output_budget, double &mean, uint &records, ulong &written, string &output_sha, string &reason) { mean = 0; records = 0; written = 0; output_sha = ""; reason = ""; m_decoder_stream_native_loads = 0; if(!m_initialized || m_poisoned || !m_checkpoint_metadata_bound || artifact.binding_sha256 != m_checkpoint_binding_sha || artifact.cursor.stage != 1 || output_budget == 0) { reason = "decoder.stream.admission"; return(false); } int expected = 0; for(int i = 0; i < ArraySize(m_records); i++) if(m_records[i].admission.split == "validation") expected++; if(expected <= 0 || ArraySize(indices) != expected) { reason = "decoder.stream.complete_validation_required"; return(false); } bool seen[]; if(ArrayResize(seen, ArraySize(m_records)) != ArraySize(m_records)) { reason = "decoder.stream.seen_capacity"; return(false); } ArrayInitialize(seen, false); for(int i = 0; i < expected; i++) { if(indices[i] < 0 || indices[i] >= ArraySize(m_records) || m_records[indices[i]].admission.split != "validation") {reason = "decoder.stream.record"; return(false);} if(seen[indices[i]]) { reason = "decoder.stream.duplicate"; return(false); } seen[indices[i]] = true; } string canonical = "", sha = ""; SDPCCCheckpointArtifact actual; if(!DPCCCheckpointCanonicalPath(path, canonical, reason) || FileIsExist(canonical, artifact.common_files ? FILE_COMMON : 0) || !DPCCReadCheckpointArtifact(artifact.manifest_path, artifact.common_files, artifact.manifest_sha256, actual, reason) || DPCCCheckpointManifest(actual) != DPCCCheckpointManifest(artifact) || !DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.denoiser_sha256) return(false); CNet *decoder = NULL, *denoiser = NULL; if(!CheckpointCandidate(true, artifact.decoder_path, artifact.common_files, artifact.cursor.decoder_updates, artifact.decoder_state, decoder, reason)) return(false); m_decoder_stream_native_loads++; if(!CheckpointCandidate(false, artifact.denoiser_path, artifact.common_files, artifact.cursor.denoiser_updates, artifact.denoiser_state, denoiser, reason)) {delete decoder; return(false);} m_decoder_stream_native_loads++; const int handle = FileOpen(canonical, FILE_WRITE | FILE_BIN | (artifact.common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { delete denoiser; delete decoder; reason = "decoder.stream.open"; return(false); } bool ok = WriteDecoderObservation(handle, "DPCC_DECODER_NATIVE_OBSERVATIONS_V1\n" + artifact.manifest_path + "\n" + artifact.manifest_sha256 + "\n" + (string)expected + "\nUNPUBLISHED_UNTIL_STAGE_RESULT_SHA\n", output_budget, written, reason); double sum = 0; for(int ordinal = 0; ok && ordinal < expected; ordinal++) { if(IsStopped()) { reason = "decoder.stream.cancelled"; ok = false; break; } SDPCCDecoderValidationObservation observation; ok = ObserveDecoderCandidate(decoder, denoiser, artifact, indices[ordinal], observation, reason); if(!ok) break; sum += observation.normalized_loss; ok = MathIsValidNumber(sum) && WriteDecoderObservation(handle, (string)observation.record_index + "\n" + observation.window_id + "\n" + StringFormat("%.17g\n", observation.normalized_loss) + (string)ArraySize(observation.normalized) + "\n", output_budget, written, reason); for(int i = 0; ok && i < ArraySize(observation.normalized); i++) ok = WriteDecoderObservation(handle, StringFormat("%.9g\n", observation.normalized[i]), output_budget, written, reason); for(int i = 0; ok && i < ArraySize(observation.native_ohlc); i++) ok = WriteDecoderObservation(handle, StringFormat("%.9g\n", observation.native_ohlc[i]), output_budget, written, reason); for(int i = 0; ok && i < ArraySize(observation.causal_price_norm); i++) ok = WriteDecoderObservation(handle, StringFormat("%.9g\n", observation.causal_price_norm[i]), output_budget, written, reason); if(ok) records++; } if(ok) ok = WriteDecoderObservation(handle, "END_DPCC_DECODER_NATIVE_OBSERVATIONS_V1\n", output_budget, written, reason); ResetLastError(); FileFlush(handle); const int flush_error = GetLastError(); FileClose(handle); delete denoiser; delete decoder; if(!ok || flush_error != 0) { if(StringLen(reason) == 0) reason = "decoder.stream.native_or_flush"; return(false); } if(!DPCCCheckpointFileSHA(artifact.manifest_path, artifact.common_files, 65536, sha, reason) || sha != artifact.manifest_sha256 || !DPCCCheckpointFileSHA(artifact.decoder_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.decoder_sha256 || !DPCCCheckpointFileSHA(artifact.denoiser_path, artifact.common_files, m_checkpoint_max_file_bytes, sha, reason) || sha != artifact.denoiser_sha256 || !DPCCCheckpointFileSHA(canonical, artifact.common_files, output_budget, output_sha, reason)) {reason = "decoder.stream.changed_or_output_hash"; return(false);} mean = sum / (double)expected; return(MathIsValidNumber(mean) && records == (uint)expected); } uint DecoderValidationNativeLoads(void)const {return(m_decoder_stream_native_loads);} bool RestoreTrainingCheckpointFile(const string path, const bool common_files, const string expected_sha, SDPCCCheckpointCursor &cursor, string &reason) { SDPCCCheckpointArtifact artifact; if(!DPCCReadCheckpointArtifact(path, common_files, expected_sha, artifact, reason)) { ZeroMemory(cursor); return(false); } return(RestoreTrainingCheckpoint(artifact, cursor, reason)); } bool DecoderExample(const int record_index, const ENUM_DPCC_TRAIN_OPERATION operation, SDPCCTrainingExampleResult &result) { if(!Begin(record_index, operation, result)) return(false); const bool gradients = operation != DPCC_TRAIN_VALIDATE, update = operation == DPCC_TRAIN_UPDATE; if(update && !m_decoder_initial_saved) return(Fail(result, "checkpoint", "training.initial_decoder_checkpoint_required")); if(update && m_decoder.UpdateCount() == 0) { string checkpoint_reason = ""; if(!CheckpointFirstUpdateGate(checkpoint_reason)) return(Fail(result, "checkpoint", checkpoint_reason)); } result.updates_before = m_decoder.UpdateCount(); result.updates_after = result.updates_before; if(!DecoderNet().TrainMode(gradients)) return(Fail(result, "mode", "training.decoder_mode")); DecoderNet().SetWeightsUpdate(update); if(!DecoderNet().feedForward(GetPointer(m_z), 1, false, (CBufferFloat*)NULL)) return(Fail(result, "forward", "training.decoder_forward")); if(!m_decoder.BuildNativeBankRhoLoss(GetPointer(m_target), GetPointer(m_norm), GetPointer(m_rho), m_config.probability_tolerance, gradients, result.loss)) { if(gradients && !m_decoder.CancelGeneration()) m_poisoned = true; return(Fail(result, "loss", "training.decoder_native_rho_loss")); } if(gradients) { result.update_attempted = update; if(!DecoderNet().backPropGradient((CBufferFloat*)NULL, (CBufferFloat*)NULL, 1, false)) {m_poisoned = true; result.updates_after = m_decoder.UpdateCount(); return(Fail(result, "backward", "training.decoder_backward_or_update"));} } result.updates_after = m_decoder.UpdateCount(); result.completed = true; result.stage = "complete"; return(true); } bool DenoiserExample(const int record_index, const ENUM_DPCC_TRAIN_OPERATION operation, const SDPCCTrainingExampleKeys &keys, SDPCCTrainingExampleResult &result) { ZeroMemory(result); if(!m_initialized || m_poisoned || record_index < 0 || record_index >= ArraySize(m_records)) return(Fail(result, "admission", "training.instance_or_record")); result.window_id = m_records[record_index].window_id; result.source_id = m_records[record_index].source_id; result.split = m_records[record_index].admission.split; result.decision_time = m_records[record_index].admission.decision_time; result.bank_sha256 = m_integrity.bank_sha256; result.header_sha256 = m_integrity.header_sha256; result.j = keys.j; result.repeat = keys.repeat; result.seed = keys.seed; const SDPCCTrainingFrozenRecordKeys fixed = m_record_keys[record_index]; if(keys.decision_identity != fixed.decision_identity || keys.context_identity != fixed.context_identity || keys.owner_identity != fixed.owner_identity || (operation == DPCC_TRAIN_VALIDATE && keys.j != fixed.validation_j)) return(Fail(result, "keys", "training.frozen_record_keys")); if(keys.decision_time != result.decision_time || keys.j == 0 || keys.j > m_config.shape.J || keys.decision_identity == 0 || keys.context_identity == 0 || keys.owner_identity == 0 || keys.seed != (operation == DPCC_TRAIN_VALIDATE ? m_validation_seed : m_training_seed) || (operation == DPCC_TRAIN_VALIDATE && keys.repeat != m_validation_repeat)) return(Fail(result, "keys", "training.explicit_keys")); if(!Begin(record_index, operation, result)) return(false); result.j = keys.j; result.repeat = keys.repeat; result.seed = keys.seed; float indexed[], epsilon[]; if(!DPCCBuildIndexedNoiseV1(keys.seed, keys.decision_time, keys.repeat, 1, m_config.shape.J, m_config.shape.H, indexed) || ArrayResize(epsilon, m_config.shape.H * 6) != (int)(m_config.shape.H * 6)) return(Fail(result, "noise", "training.indexed_noise")); //--- Raw slice only; the canonical helper defines all noise arithmetic. for(uint i = 0; i < m_config.shape.H * 6; i++) epsilon[i] = indexed[keys.j * m_config.shape.H * 6 + i]; if(!Upload(m_epsilon, epsilon)) return(Fail(result, "noise", "training.noise_upload")); const bool gradients = operation != DPCC_TRAIN_VALIDATE, update = operation == DPCC_TRAIN_UPDATE; if(update && !m_denoiser_initial_saved) return(Fail(result, "checkpoint", "training.initial_denoiser_checkpoint_required")); if(update && m_denoiser.UpdateCount() == 0) { string checkpoint_reason = ""; if(!CheckpointFirstUpdateGate(checkpoint_reason)) return(Fail(result, "checkpoint", checkpoint_reason)); } result.updates_before = m_denoiser.UpdateCount(); result.updates_after = result.updates_before; if(!DenoiserNet().TrainMode(gradients)) return(Fail(result, "mode", "training.denoiser_mode")); DenoiserNet().SetWeightsUpdate(update); if(!m_denoiser.PrepareTrainingExample(GetPointer(m_actions), GetPointer(m_epsilon), GetPointer(m_context), GetPointer(m_account), keys.j, m_alpha[keys.j], GetPointer(m_noisy), GetPointer(m_condition))) return(Fail(result, "prepare", "training.native_example_prepare")); if(!m_denoiser.SetContextIdentity(keys.decision_identity, keys.context_identity, keys.owner_identity) || !DenoiserNet().feedForward(GetPointer(m_noisy), 1, false, GetPointer(m_condition))) return(Fail(result, "forward", "training.denoiser_forward")); if(!m_denoiser.BuildEpsilonMSELoss(GetPointer(m_epsilon), gradients, result.loss)) { if(gradients && !m_denoiser.CancelGeneration()) m_poisoned = true; return(Fail(result, "loss", "training.epsilon_mse")); } if(gradients) { result.update_attempted = update; if(!DenoiserNet().backPropGradient(GetPointer(m_condition), (CBufferFloat*)NULL, 1, false)) {m_poisoned = true; result.updates_after = m_denoiser.UpdateCount(); return(Fail(result, "backward", "training.denoiser_backward_or_update"));} } result.updates_after = m_denoiser.UpdateCount(); result.completed = true; result.stage = "complete"; return(true); } }; #endif struct SDPCCTrainingRoleConfig { long max_epochs, validation_every_epochs, patience; double min_delta; string training_seed, validation_seed, loss; }; struct SDPCCTrainingConfigAdmission { bool example_only; string training_id, dataset_id, bank_id, normalization_id, source_sha256, dataset_sha256, model_sha256, bank_manifest_sha256, time_index_sha256, normalization_section_sha256; SDPCCDatasetTimeIndex time_index; SDPCCBankMetadataResult bank_receipt; SDPCCTrainingModelBinding model_binding; SDPCCArchitectureContract architecture; SDPCCTrainingRoleConfig decoder, denoiser; }; struct SDPCCTrainingPlanEntry { uint epoch, ordinal, j, repeat; int record_index; string window_id; }; //--- Exact native optimizer admission independently of caller JSON schema. bool DPCCRunOptimizer(CDPCCJsonDocument &d, const int role, string &reason) { const int o = d.FindMember(role, "optimizer"); string names[] = {"name", "profile", "bias_correction", "gradient_convention", "clip_location", "zero_variance_update"}; string expected[] = {"ADAM", "NN_NATIVE_CONV_ADAM_C6A8_V1", "HOST_LT_FROM_T", "NEGATIVE_DERIVATIVE", "PER_WEIGHT_AFTER_TOKEN_SUM", "ZERO_DELTA"}; for(int i = 0; i < ArraySize(names); i++) {string v; if(!DPCCResourceReadString(d, o, names[i], v, reason, "training.optimizer") || v != expected[i]) {reason = "training.optimizer." + names[i]; return(false);}} string numeric[] = {"learning_rate", "beta1", "beta2", "epsilon", "weight_decay", "gradient_clip"}; double values[] = {0.0003, 0.9, 0.999, 0, 0, 0.01}; for(int i = 0; i < ArraySize(numeric); i++) {double v; if(!DPCCResourceReadDouble(d, o, numeric[i], v, reason, "training.optimizer") || v != values[i]) {reason = "training.optimizer." + numeric[i]; return(false);}} long step; if(!DPCCResourceReadLong(d, o, "initial_step", step, reason, "training.optimizer.initial_step") || step != 1) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCRunRole | //+------------------------------------------------------------------+ bool DPCCRunRole(CDPCCJsonDocument &d, const int root, const string role, const string loss, SDPCCTrainingRoleConfig &c, string &reason) { int n = d.FindMember(root, role); long batch; if(!DPCCRunOptimizer(d, n, reason) || !DPCCResourceReadLong(d, n, "max_epochs", c.max_epochs, reason, role) || !DPCCResourceReadLong(d, n, "validation_every_epochs", c.validation_every_epochs, reason, role) || !DPCCResourceReadLong(d, n, "patience", c.patience, reason, role) || !DPCCResourceReadLong(d, n, "batch_examples", batch, reason, role) || !DPCCResourceReadDouble(d, n, "min_delta", c.min_delta, reason, role) || !DPCCResourceReadString(d, n, "training_seed", c.training_seed, reason, role) || !DPCCResourceReadString(d, n, "validation_seed", c.validation_seed, reason, role) || !DPCCResourceReadString(d, n, "loss", c.loss, reason, role)) return(false); if(batch != 1 || c.max_epochs < 1 || c.validation_every_epochs < 1 || c.patience < 1 || c.validation_every_epochs > c.max_epochs || c.min_delta < 0 || !MathIsValidNumber(c.min_delta) || c.loss != loss) {reason = "training.role_config"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCRunNormalizationSection | //+------------------------------------------------------------------+ bool DPCCRunNormalizationSection(CDPCCJsonDocument &d, const SDPCCTrainingModelBinding &binding, const SDPCCDatasetManifest &dataset, string &digest, string &reason) { int n = d.FindMember(d.Root(), "normalization"); string wire = "DPCC_MODEL_NORMALIZATION_V1\n"; string names[] = {"id", "mode", "statistics_source", "inverse_implementation", "loss_coordinates", "account_feature_contract", "account_builder_sha256"}; for(int i = 0; i < ArraySize(names); i++) { string v; if(!DPCCResourceReadString(d, n, names[i], v, reason, "normalization")) return(false); if(i == 0 && v != dataset.normalization_id) { reason = "training.model_normalization_id"; return(false); } wire += (string)StringLen(v) + ":" + v + "\n"; } string channels[] = {"Open", "High", "Low", "Close"}; int a = d.FindMember(n, "channels"); if(d.ChildCount(a) != 4) { reason = "training.normalization_channels"; return(false); } for(int i = 0; i < 4; i++) if(d.Type(d.ChildAt(a, i)) != DPCC_JSON_STRING || d.Text(d.ChildAt(a, i)) != channels[i]) {reason = "training.normalization_channels"; return(false);} wire += "OHLC_GAIN1_BIAS0\n"; SDPCCDatasetInterval fit; if(!DPCCDatasetDecodeInterval(d, d.FindMember(n, "fit_interval"), fit, reason, "model.normalization.fit_interval")) return(false); if(fit.start_utc != dataset.normalization_fit.start_utc || fit.end_utc != dataset.normalization_fit.end_utc || fit.bar_start != dataset.normalization_fit.bar_start || fit.bar_end != dataset.normalization_fit.bar_end) {reason = "training.normalization_fit_interval"; return(false);} wire += (string)fit.start_utc + ":" + (string)fit.end_utc + ":" + (string)fit.bar_start + ":" + (string)fit.bar_end + "\n"; for(int i = 0; i < 6; i++) wire += StringFormat("%.17g:%.17g\n", (double)binding.action_bias[i], (double)binding.action_scale[i]); return(DPCCBankSourceHash(wire, digest, reason, "normalization.section")); } //+------------------------------------------------------------------+ //| Function DPCCRunSameInterval | //+------------------------------------------------------------------+ bool DPCCRunSameInterval(const SDPCCDatasetInterval &left, const SDPCCDatasetInterval &right) { return(left.start_utc == right.start_utc && left.end_utc == right.end_utc && left.bar_start == right.bar_start && left.bar_end == right.bar_end); } //--- The staged source is already the owner’s exact data contract. IDs and hashes //--- authenticate artifacts; this check also refuses a semantically different //--- decoded DatasetManifest that reuses those identities. bool DPCCRunSameDatasetContract(const SDPCCDatasetManifest &actual, const SDPCCDatasetManifest &source, string &reason) { if(actual.dataset_id != source.dataset_id || actual.normalization_id != source.normalization_id || actual.embargo_bars != source.embargo_bars || actual.time_index.id != source.time_index.id || actual.time_index.sha256 != source.time_index.sha256 || !DPCCRunSameInterval(actual.fit, source.fit) || !DPCCRunSameInterval(actual.validation, source.validation) || !DPCCRunSameInterval(actual.test, source.test) || !DPCCRunSameInterval(actual.normalization_fit, source.normalization_fit)) {reason = "training.actual_dataset_contract"; return(false);} return(true); } //+------------------------------------------------------------------+ //| Function DPCCRunContainingArtifact | //+------------------------------------------------------------------+ bool DPCCRunContainingArtifact(const string package_root, const string canonical_path, string &relative, string &reason) { string prepared; string segments[]; if(!DPCCResolverPreparePath(package_root, prepared, reason, "run.package_root") || !DPCCResolverAppendSegments(prepared, segments, 0, reason, "run.package_root")) return(false); const string prefix = DPCCResolverJoinSegments(segments) + "/"; if(StringFind(canonical_path, prefix) != 0) { reason = "run.artifact_outside_package"; return(false); } relative = StringSubstr(canonical_path, StringLen(prefix)); return(relative != ""); } //--- Whole JSON schema admission plus source hash; component copies compare actual JSON objects. //--- linked_model must already be resolved/admitted by the factory's model metadata path. bool DPCCDecodeTrainingConfigDocuments(CDPCCJsonDocument &d, CDPCCJsonDocument &m, const string source, const SDPCCTrainingBankSource &bank, SDPCCTrainingConfigAdmission &result, string &reason) { ZeroMemory(result); const int root = d.Root(), mr = m.Root(); SDPCCTrainingConfigAdmission c; if(!DPCCResourceReadBool(d, root, "example_only", c.example_only, reason, "training") || !DPCCResourceReadString(d, root, "training_id", c.training_id, reason, "training") || !DPCCResourceReadString(d, root, "dataset_id", c.dataset_id, reason, "training") || !DPCCResourceReadString(d, root, "bank_id", c.bank_id, reason, "training") || !DPCCResourceReadString(d, root, "normalization_id", c.normalization_id, reason, "training") || !DPCCRunRole(d, root, "decoder", "RHO_WEIGHTED_NORMALIZED_MSE", c.decoder, reason) || !DPCCRunRole(d, root, "denoiser", "EPSILON_MSE", c.denoiser, reason)) return(false); if(c.dataset_id != bank.dataset.dataset_id || c.normalization_id != bank.dataset.normalization_id || c.bank_id != bank.bank.bank_id) {reason = "training.dataset_binding"; return(false);} int graph = m.FindMember(m.FindMember(mr, "composition"), "denoiser"); string graph_sha; if(!DPCCResourceReadString(m, graph, "graph_sha256", graph_sha, reason, "training.graph") || !DPCCDecodeArchitectureContract(m, mr, d, root, graph_sha, c.architecture, reason)) return(false); if(!DPCCBankSourceHash(source, c.source_sha256, reason, "training")) return(false); c.dataset_sha256 = bank.links.dataset_sha256; result = c; return(true); } //+------------------------------------------------------------------+ //| Function DPCCDecodeTrainingConfigFields | //+------------------------------------------------------------------+ bool DPCCDecodeTrainingConfigFields(const string source, const string schema, const string linked_model, const SDPCCTrainingBankSource &bank, SDPCCTrainingConfigAdmission &result, string &reason) { string digest; CDPCCJsonDocument d, m; if(!DPCCBankSourceHash(schema, digest, reason, "training.schema") || digest != "7c630093d92684a80dede93bd51c0fa339521d8586660b019e4107703a338a2a") {reason = "training.schema_sha256"; return(false);} if(!DPCCCentralSchemaValidate(d, source, schema, reason, "training") || !m.Parse(linked_model, reason)) return(false); return(DPCCDecodeTrainingConfigDocuments(d, m, source, bank, result, reason)); } enum ENUM_DPCC_TRAIN_ROLE { DPCC_ROLE_DECODER = 0, DPCC_ROLE_DENOISER = 1 }; //+------------------------------------------------------------------+ //| Declaration CDPCCAdmittedTrainingConfig | //+------------------------------------------------------------------+ class CDPCCAdmittedTrainingConfig { private: bool m_admitted; SDPCCTrainingConfigAdmission m_config; string m_model_source; public: CDPCCAdmittedTrainingConfig(void) {m_admitted = false;} bool Admit(const string package_root, const string containing_json, const SDPCCResourceReference &training_reference, const SDPCCResourceReference &model_reference, const bool common_files, const string schema, const string model_schema, const string bank_schema, const string dataset_schema, const string time_index_schema, const SDPCCTrainingBankSource &bank, string &reason) { if(m_admitted) {reason = "training.config_already_admitted"; return(false); } SDPCCArtifactResolveResult t, m; if(!DPCCResolveArtifact(package_root, containing_json, training_reference, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, t, reason) || !DPCCResolveArtifact(package_root, containing_json, model_reference, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, m, reason)) return(false); string ts = CharArrayToString(t.bytes, 0, ArraySize(t.bytes), CP_UTF8), ms = CharArrayToString(m.bytes, 0, ArraySize(m.bytes), CP_UTF8); string schema_digest; if(!DPCCBankSourceHash(model_schema, schema_digest, reason, "model.schema") || schema_digest != "10498a359f69fb249441f3bdbe6c3f52f0823a6b854205e42d2da6b9ae482248") {reason = "training.model_schema_sha256"; return(false);} CDPCCJsonDocument model_document; SDPCCCentralModel model_typed; if(!DPCCCentralSchemaValidate(model_document, ms, model_schema, reason, "model") || !DPCCDecodeCentralModel(model_document, model_typed, reason) || !DPCCDatasetShapeEqual(bank.dataset.shapes, model_typed.shapes, reason)) return(false); string training_schema_hash; CDPCCJsonDocument training_document; if(!DPCCBankSourceHash(schema, training_schema_hash, reason, "training.schema") || training_schema_hash != "7c630093d92684a80dede93bd51c0fa339521d8586660b019e4107703a338a2a") {reason = "training.schema_sha256"; return(false);} if(!DPCCCentralSchemaValidate(training_document, ts, schema, reason, "training")) return(false); if(model_typed.type_ids.decoder != defNeuronScenarioDecoderOCL || model_typed.type_ids.rollout != defNeuronScenarioRolloutOCL || model_typed.type_ids.projection != defNeuronPlanProjectionOCL || model_typed.type_ids.denoiser != defNeuronPlanDenoiserOCL || model_typed.type_ids.dpcc != defNeuronDPCCOCL) { reason = "training.native_type_ids"; return(false); } SDPCCTrainingConfigAdmission value; if(!DPCCDecodeTrainingConfigDocuments(training_document, model_document, ts, bank, value, reason)) return(false); if(value.source_sha256 != t.observed_sha256 || value.training_id != training_reference.id) { reason = "training.resolved_identity"; return(false); } string model_id; if(!DPCCResourceReadString(model_document, model_document.Root(), "model_id", model_id, reason, "training.model_id") || model_id != model_reference.id) {reason = "training.model_id"; return(false);} SDPCCResourceReference linked_training; if(!DPCCBankReadReference(model_document, model_document.FindMember(model_document.Root(), "training"), "training_config", linked_training, reason, "model.training_config")) return(false); if(linked_training.id != training_reference.id || linked_training.sha256 != t.observed_sha256) {reason = "training.model_training_link"; return(false);} SDPCCResourceReference linked_dataset, linked_bank; const int training_node = model_document.FindMember(model_document.Root(), "training"); if(!DPCCBankReadReference(model_document, training_node, "dataset_manifest", linked_dataset, reason, "model.training.dataset_manifest") || !DPCCBankReadReference(model_document, training_node, "bank_manifest", linked_bank, reason, "model.training.bank_manifest")) return(false); if(linked_dataset.id != bank.dataset.dataset_id || linked_dataset.sha256 != bank.links.dataset_sha256 || linked_bank.id != bank.bank.bank_id) {reason = "training.model_data_link"; return(false);} string bank_schema_hash, dataset_schema_hash; if(!DPCCBankSourceHash(bank_schema, bank_schema_hash, reason, "bank.schema") || bank_schema_hash != "13780a0801286e89c276e20fe892bfcc08ee5fc38aac710c0c044b698ce9cb06" || !DPCCBankSourceHash(dataset_schema, dataset_schema_hash, reason, "dataset.schema") || dataset_schema_hash != "a52ba580e67dbf507622c85fe336b45d9f54286c8d18a0839f99bdccf812417c") {reason = "training.linked_schema_sha256"; return(false);} string model_containing; if(!DPCCRunContainingArtifact(package_root, m.canonical_path, model_containing, reason)) return(false); SDPCCArtifactResolveResult actual_training, actual_bank, actual_dataset; if(!DPCCResolveArtifact(package_root, model_containing, linked_training, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, actual_training, reason) || !DPCCResolveArtifact(package_root, model_containing, linked_bank, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, actual_bank, reason) || !DPCCResolveArtifact(package_root, model_containing, linked_dataset, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, actual_dataset, reason)) return(false); if(actual_training.observed_sha256 != t.observed_sha256) { reason = "training.backlink_actual_hash"; return(false); } string bs, ds; if(!DPCCCentralBytesSource(actual_bank.bytes, bs, reason, "bank") || !DPCCCentralBytesSource(actual_dataset.bytes, ds, reason, "dataset")) return(false); uchar header_bytes[]; int header_count = StringToCharArray(bank.header, header_bytes, 0, WHOLE_ARRAY, CP_UTF8); if(header_count <= 0 || ArrayResize(header_bytes, header_count - 1) != header_count - 1) { reason = "training.header_bytes"; return(false); } if(!DPCCValidateBankMetadata(bs, header_bytes, ds, bank_schema, bank.header_schema, dataset_schema, bank.dataset, bank.links, bank.profile, bank.bank.bank_id, bank.bank.bank_sha256, bank.bank.record_count, !bank.dataset.example_only, value.bank_receipt, reason)) return(false); if(value.bank_receipt.header_sha256 != bank.bank.header_sha256 || value.bank_receipt.time_index_id != bank.bank.time_index_id || value.bank_receipt.dataset_id != bank.bank.dataset_id) { reason = "training.linked_bank_receipt"; return(false); } if(!DPCCRunModelValues(model_document, model_typed.shapes, value.model_binding, reason)) return(false); if(!DPCCRunNormalizationSection(model_document, value.model_binding, bank.dataset, value.normalization_section_sha256, reason)) return(false); //--- Authenticate the actual declared TimeIndex and BankManifest dataset backlink. CDPCCJsonDocument dataset_document, bank_document; SDPCCDatasetManifest actual_dataset_metadata; if(!dataset_document.Parse(ds, reason) || !DPCCDecodeDatasetManifest(dataset_document, dataset_document.Root(), actual_dataset_metadata, reason) || !DPCCRunSameDatasetContract(actual_dataset_metadata, bank.dataset, reason) || !bank_document.Parse(bs, reason)) return(false); const int model_price = model_document.FindMember(model_document.Root(), "price_contract"), dataset_price = dataset_document.FindMember(dataset_document.Root(), "price_contract"); if(!DPCCBankJsonEqual(model_document, model_price, dataset_document, dataset_price, 0)) { reason = "training.model_dataset_price_contract"; return(false); } SDPCCResourceReference bank_dataset; if(!DPCCBankReadReference(bank_document, bank_document.Root(), "dataset_manifest", bank_dataset, reason, "bank.dataset_manifest")) return(false); string bank_containing, dataset_containing; if(!DPCCRunContainingArtifact(package_root, actual_bank.canonical_path, bank_containing, reason) || !DPCCRunContainingArtifact(package_root, actual_dataset.canonical_path, dataset_containing, reason)) return(false); SDPCCArtifactResolveResult bank_dataset_artifact, index_artifact; if(!DPCCResolveArtifact(package_root, bank_containing, bank_dataset, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, bank_dataset_artifact, reason) || bank_dataset_artifact.observed_sha256 != actual_dataset.observed_sha256 || !DPCCResolveArtifact(package_root, dataset_containing, actual_dataset_metadata.time_index, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, index_artifact, reason)) return(false); string index_source, index_schema_hash; if(!DPCCCentralBytesSource(index_artifact.bytes, index_source, reason, "time_index") || !DPCCBankSourceHash(time_index_schema, index_schema_hash, reason, "time_index.schema") || index_schema_hash != "040554860c0a1c1ff9f823c70710f5d7c51934c8c18c97e8c06e709f217ffb57") {reason = "training.time_index_schema"; return(false);} CDPCCJsonDocument index_document; if(!DPCCCentralSchemaValidate(index_document, index_source, time_index_schema, reason, "time_index") || !DPCCDecodeDatasetTimeIndex(index_document, index_document.Root(), bank.dataset.dataset_id, value.time_index, reason) || value.time_index.time_index_id != bank.links.time_index_id || index_artifact.observed_sha256 != bank.links.time_index_sha256) {reason = "training.actual_time_index"; return(false);} value.time_index_sha256 = index_artifact.observed_sha256; value.model_sha256 = m.observed_sha256; value.bank_manifest_sha256 = actual_bank.observed_sha256; m_config = value; m_model_source = ms; m_admitted = true; return(true); } bool ModelSource(string &source) {source = ""; if(!m_admitted)return(false); source = m_model_source; return(true);} bool Get(const ENUM_DPCC_TRAIN_ROLE role, SDPCCTrainingConfigAdmission &config, SDPCCTrainingRoleConfig &value) { if(!m_admitted || (role != DPCC_ROLE_DECODER && role != DPCC_ROLE_DENOISER)) return(false); config = m_config; if(role == DPCC_ROLE_DECODER) value = m_config.decoder; else value = m_config.denoiser; return(true); } }; //+------------------------------------------------------------------+ //| Function DPCCRunPlanHash | //+------------------------------------------------------------------+ bool DPCCRunPlanHash(const SDPCCTrainingConfigAdmission &config, const SDPCCTrainingRoleConfig &role, const SDPCCTrainingRecordIdentity &records[], const SDPCCTrainingPlanEntry &fit[], const SDPCCTrainingPlanEntry &validation[], string &digest, string &reason) { string wire = "DPCC_EXPLICIT_RUN_PLAN_V1\n" + config.source_sha256 + "\n" + config.model_sha256 + "\n" + config.bank_manifest_sha256 + "\n" + config.dataset_sha256 + "\n" + config.time_index_sha256 + "\n" + config.normalization_section_sha256 + "\n" + (string)role.max_epochs + "\n" + role.loss + "\n" + role.training_seed + "\n" + role.validation_seed + "\n"; for(int k = 0; k < ArraySize(fit); k++) {SDPCCTrainingPlanEntry p = fit[k]; wire += (string)p.epoch + ":" + (string)p.ordinal + ":" + ( string)p.record_index + ":" + p.window_id + ":" + (string)p.j + ":" + (string)p.repeat + "\n";} for(int k = 0; k < ArraySize(validation); k++) {SDPCCTrainingPlanEntry p = validation[k]; wire += "V:" + (string)p.ordinal + ":" + (string)p.record_index + ":" + p.window_id + ":" + (string)p.j + ":" + (string)p.repeat + "\n";} for(int i = 0; i < ArraySize(records); i++) wire += "R:" + (string)i + ":" + records[i].window_id + ":" + records[i].source_id + ":" + records[i].split + ":" + records[i].bank_sha256 + ":" + records[i].header_sha256 + ":" + (string)records[i].decision_time + ":" + ( string)records[i].decision_bar_index + ":" + (string)records[i].keys.decision_identity + ":" + (string)records[i].keys.context_identity + ":" + ( string)records[i].keys.owner_identity + "\n"; return(DPCCBankSourceHash(wire, digest, reason, "plan")); } //--- Immutable owned plans; no model math, optimizer calls, READY or invented PRNG. class CDPCCFrozenRolePlan { private: bool m_admitted; SDPCCTrainingPlanEntry m_fit[], m_validation[]; string m_sha; public: CDPCCFrozenRolePlan(void) {m_admitted = false;} bool Admit(CDPCCAdmittedTrainingConfig &admitted_config, const ENUM_DPCC_TRAIN_ROLE role_id, const uint J, CDPCCTrainingExamples &owner, const uint validation_repeat, const SDPCCTrainingPlanEntry &fit[], const SDPCCTrainingPlanEntry &validation[], const string expected_sha, string &reason) { SDPCCTrainingRoleConfig role; SDPCCTrainingConfigAdmission config; if(!admitted_config.Get(role_id, config, role)) { reason = "plan.config_not_admitted"; return(false); } reason = ""; if(m_admitted) { reason = "plan.already_admitted"; return(false); } if(J == 0 || role.max_epochs < 1 || !DPCCResolverLowerSha256(expected_sha)) { reason = "plan.config"; return(false); } if(!owner.RunBinding(config.dataset_id, config.bank_id, config.normalization_id, config.dataset_sha256, config.bank_receipt, config.model_binding, config.time_index_sha256, config.time_index, config.architecture, role.training_seed, role.validation_seed, validation_repeat, J, role_id == DPCC_ROLE_DENOISER, reason)) return(false); SDPCCTrainingRecordIdentity records[]; int size = owner.RecordCount(); if(size <= 0 || ArrayResize(records, size) != size) { reason = "plan.owner_records"; return(false); } for(int i = 0; i < size; i++) if(!owner.RecordIdentity(i, records[i])) { reason = "plan.owner_record"; return(false); } int nf = 0, nv = 0; for(int i = 0; i < ArraySize(records); i++) { if(records[i].record_index != i || records[i].window_id == "" || records[i].keys.window_id != records[i].window_id) {reason = "plan.record_identity"; return(false);} if(records[i].split == "fit") nf++; if(records[i].split == "validation") nv++; } if(nf == 0 || nv == 0 || (nf > 0 && (ArraySize(fit) % nf != 0 || (long)(ArraySize(fit) / nf) != role.max_epochs)) || ArraySize(validation) != nv) {reason = "plan.count"; return(false);} for(int k = 0; k < ArraySize(fit); k++) { SDPCCTrainingPlanEntry p = fit[k]; if(p.epoch != (uint)(k / nf + 1) || p.ordinal != (uint)(k % nf) || p.record_index < 0 || p.record_index >= ArraySize(records) || records[p.record_index].split != "fit" || p.window_id != records[p.record_index].window_id || (role_id == DPCC_ROLE_DENOISER ? (p.j == 0 || p.j > J) : (p.j != 0 || p.repeat != 0))) {reason = "plan.fit_binding"; return(false);} for(int q = k - k % nf; q < k; q++) if(fit[q].record_index == p.record_index) { reason = "plan.fit_duplicate"; return(false); } } int cursor = 0; for(int i = 0; i < ArraySize(records); i++) if(records[i].split == "validation") { SDPCCTrainingPlanEntry p = validation[cursor]; if(p.epoch != 0 || p.ordinal != (uint)cursor || p.record_index != i || p.window_id != records[i].window_id || (role_id == DPCC_ROLE_DENOISER ? (p.repeat != validation_repeat || p.j != records[i].keys.validation_j || p.j == 0 || p.j > J) : (p.j != 0 || p.repeat != 0))) {reason = "plan.validation_binding"; return(false);} cursor++; } string digest; if(!DPCCRunPlanHash(config, role, records, fit, validation, digest, reason) || digest != expected_sha) { reason = "plan.sha256"; return(false); } if(ArrayResize(m_fit, ArraySize(fit)) != ArraySize(fit) || ArrayResize(m_validation, ArraySize(validation)) != ArraySize(validation)) {ArrayResize(m_fit, 0); ArrayResize(m_validation, 0); reason = "plan.copy"; return(false);} for(int i = 0; i < ArraySize(fit); i++) m_fit[i] = fit[i]; for(int i = 0; i < ArraySize(validation); i++) m_validation[i] = validation[i]; m_sha = digest; m_admitted = true; return(true); } int FitCount(void) {return(m_admitted ? ArraySize(m_fit) : 0);} int ValidationCount(void) {return(m_admitted ? ArraySize(m_validation) : 0);} string Digest(void) {return(m_admitted ? m_sha : "");} bool Fit(const int index, SDPCCTrainingPlanEntry &entry) {if(!m_admitted || index < 0 || index >= ArraySize(m_fit))return(false); entry = m_fit[index]; return(true);} bool Validation(const int index, SDPCCTrainingPlanEntry &entry) {if(!m_admitted || index < 0 || index >= ArraySize(m_validation))return(false); entry = m_validation[index]; return(true);} }; #endif //--- WORK stage C1: native execution and private progress, not exact run resume. struct SDPCCEpochProgress { uint role, completed_epoch, fit_cursor, validations, stale_validations, decoder_updates, denoiser_updates; bool complete, failed, has_best; double best_loss, best_patience_loss; }; #ifndef DPCC_EPOCH_STATE_MQH #define DPCC_EPOCH_STATE_MQH bool DPCCEpochValidationDue(const uint epoch, const SDPCCTrainingRoleConfig &role) {return(epoch > 0 && role.validation_every_epochs > 0 && (long)epoch % role.validation_every_epochs == 0);} //+------------------------------------------------------------------+ //| Function DPCCEpochFinished | //+------------------------------------------------------------------+ bool DPCCEpochFinished(const uint epoch, const bool validation, const SDPCCTrainingRoleConfig &role, const SDPCCEpochProgress &progress) {return((long)epoch >= role.max_epochs || (validation && (long)progress.stale_validations >= role.patience));} //--- Production transition reused by native runner and persisted history replay. bool DPCCEpochObserve(const double loss, const SDPCCTrainingRoleConfig &role, SDPCCEpochProgress &progress, bool &improved) { improved = false; if(!MathIsValidNumber(loss) || loss < 0 || role.min_delta < 0 || !MathIsValidNumber(role.min_delta) || progress.validations == UINT_MAX || progress.stale_validations == UINT_MAX) return(false); improved = !progress.has_best || loss < progress.best_loss; if(!progress.has_best || loss < progress.best_patience_loss - role.min_delta) {progress.best_patience_loss = loss; progress.stale_validations = 0;} else progress.stale_validations++; if(improved) progress.best_loss = loss; progress.has_best = true; progress.validations++; return(true); } struct SDPCCEpochHistory { uint role, epoch; bool validation; double loss; string path, sha; }; //+------------------------------------------------------------------+ //| Function DPCCEpochProgressWire | //+------------------------------------------------------------------+ string DPCCEpochProgressWire(const SDPCCEpochProgress &p) { return((string)p.role + "\n" + (string)(uint)p.completed_epoch + "\n" + (string)p.fit_cursor + "\n" + (string)p.validations + "\n" + (string)p.stale_validations + "\n" + (string)(uint)p.complete + "\n" + (string)(uint)p.failed + "\n" + (string)(uint)p.has_best + "\n" + StringFormat("%.17g\n%.17g\n", p.best_loss, p.best_patience_loss)); } //+------------------------------------------------------------------+ //| Function DPCCEpochNumber | //+------------------------------------------------------------------+ bool DPCCEpochNumber(const string text, double &value, string &reason) { CDPCCJsonDocument number; return(number.Parse(text, reason) && number.TryNumber(number.Root(), value) && MathIsValidNumber(value) && value >= 0 && StringFormat("%.17g", value) == text); } #endif #ifndef DPCC_STAGE_FIT_SOURCE_MQH #define DPCC_STAGE_FIT_SOURCE_MQH #ifndef DPCC_FIXED_NORMALIZATION_FULL_WORK_MQH #define DPCC_FIXED_NORMALIZATION_FULL_WORK_MQH #ifndef DPCC_FIT_STATISTICS_MQH #define DPCC_FIT_STATISTICS_MQH //--- DPCC_FIT_FIXED_POPULATION_WELFORD_V1: deterministic host double population statistics. struct SDPCCFitPriceNorm16 { float mean[4], std[4], gain[4], bias[4]; }; //+------------------------------------------------------------------+ //| Function DPCCFitPriceNorm16 | //+------------------------------------------------------------------+ bool DPCCFitPriceNorm16(const float &feature_major[], const uint rows, const ulong record_cap, SDPCCFitPriceNorm16 &out, string &reason) { ZeroMemory(out); reason = ""; if(rows < 2 || rows > record_cap || rows > (uint)(INT_MAX / 4) || ArraySize(feature_major) != (int)(rows * 4)) { reason = "fit.statistics.capacity"; return(false); } double mean[4], m2[4]; ZeroMemory(mean); ZeroMemory(m2); for(uint r = 0; r < rows; r++) { const double open = feature_major[r], high = feature_major[rows + r]; const double low = feature_major[2 * rows + r], close = feature_major[3 * rows + r]; if(!MathIsValidNumber(open) || !MathIsValidNumber(high) || !MathIsValidNumber(low) || !MathIsValidNumber(close) || open <= 0 || high <= 0 || low <= 0 || close <= 0 || high < low || high < open || high < close || low > open || low > close) { reason = "fit.statistics.ohlc"; return(false); } const double values[] = {open, high, low, close}; for(int c = 0; c < 4; c++) { const double delta = values[c] - mean[c]; mean[c] += delta / (double)(r + 1); m2[c] += delta * (values[c] - mean[c]); } } SDPCCFitPriceNorm16 prepared; ZeroMemory(prepared); for(int c = 0; c < 4; c++) { const double variance = m2[c] / (double)rows; const double sd = MathSqrt(variance); const float fm = (float)mean[c], fs = (float)sd, fv = fs * fs; if(!MathIsValidNumber(mean[c]) || !MathIsValidNumber(variance) || variance <= 0 || !MathIsValidNumber(sd) || sd <= 0 || !MathIsValidNumber(fm) || !MathIsValidNumber(fs) || !MathIsValidNumber(fv) || fs <= 0 || fv <= 0) { reason = "fit.statistics.variance"; return(false); } prepared.mean[c] = fm; prepared.std[c] = fs; prepared.gain[c] = 1; prepared.bias[c] = 0; } out = prepared; return(true); } #endif //--- Internal materialization binding, not a new Model component or readiness receipt. //--- Caller obtains fit_rows_sha256 from the admitted immutable OHLC producer, before this call. //--- The digest authenticates the exact UTC-indexed float rows consumed below; it does not //--- independently prove their market origin. No API guesses that missing origin binding. struct SDPCCFullFitSource { string model_sha256, dataset_sha256, time_index_sha256, normalization_section_sha256; string fit_rows_sha256, actor_sha256; long data_cutoff; ulong record_cap; string calculation_profile, calculation_source_sha256; ulong actor_contract, actor_weights, actor_trainable; }; //+------------------------------------------------------------------+ //| Declaration CDPCCFullFitNormalization | //+------------------------------------------------------------------+ class CDPCCFullFitNormalization { private: CNeuronDPCCOCL *m_actor; SDPCCFullFitSource m_binding; float m_profile[16], m_snapshot[16]; string m_statistics_sha256, m_model_source; SDPCCResourceShape m_shapes; SDPCCTrainingModelBinding m_values; bool m_calculated; bool m_valid; bool Refuse(const string text, string &reason) { m_valid = false; reason = text; return(false); } public: CDPCCFullFitNormalization(void) { m_actor = NULL; m_valid = false; m_calculated = false; } void Reset(void) { m_actor = NULL; m_valid = false; m_calculated = false; ZeroMemory(m_profile); ZeroMemory(m_snapshot); m_statistics_sha256 = ""; m_model_source = ""; } //--- Four feature-major OHLC channels, one value per actual fit bar, ascending UTC. //--- All records in the half-open normalization_fit interval are required exactly once. bool PrepareSource(const string actual_model_source, const SDPCCTrainingConfigAdmission &admission, const string actual_dataset_source, const string actual_time_index_source, const SDPCCFullFitSource &binding, const float &fit_ohlc[], const long &start_utc[], const long &end_utc[], string &reason) { Reset(); string hash; SDPCCDatasetManifest dataset; SDPCCDatasetTimeIndex index; CDPCCJsonDocument dataset_document, index_document; if(!DPCCBankSourceHash(actual_dataset_source, hash, reason, "fixed_norm.dataset") || hash != binding.dataset_sha256 || hash != admission.dataset_sha256 || !dataset_document.Parse(actual_dataset_source, reason) || !DPCCDecodeDatasetManifest(dataset_document, dataset_document.Root(), dataset, reason) || !DPCCBankSourceHash(actual_time_index_source, hash, reason, "fixed_norm.time_index") || hash != binding.time_index_sha256 || hash != admission.time_index_sha256 || !index_document.Parse(actual_time_index_source, reason) || !DPCCDecodeDatasetTimeIndex(index_document, index_document.Root(), dataset.dataset_id, index, reason)) return(Refuse("BLOCKED.fixed_norm.actual_Dataset_TimeIndex_bytes", reason)); if(!DPCCResolverLowerSha256(binding.fit_rows_sha256) || !DPCCResolverLowerSha256(binding.actor_sha256)) return(Refuse("BLOCKED.fixed_norm.actual_fit_OHLC_source_binding", reason)); if(!DPCCBankSourceHash(actual_model_source, hash, reason, "fixed_norm.model") || hash != binding.model_sha256 || hash != admission.model_sha256 || binding.dataset_sha256 != admission.dataset_sha256 || binding.time_index_sha256 != admission.time_index_sha256 || binding.normalization_section_sha256 != admission.normalization_section_sha256 || dataset.dataset_id != admission.dataset_id || dataset.normalization_id != admission.normalization_id) return(Refuse("BLOCKED.fixed_norm.admitted_source_identity", reason)); CDPCCJsonDocument document; if(!document.Parse(actual_model_source, reason)) return(false); if(!DPCCDecodeResourceShape(document, document.FindMember(document.Root(), "shapes"), m_shapes, reason, "fixed_norm.Model.shapes") || !DPCCRunModelValues(document, m_shapes, m_values, reason)) return(false); const int norm = document.FindMember(document.Root(), "normalization"); string mode; if(!DPCCResourceReadString(document, norm, "statistics_source", mode, reason, "normalization") || mode != "FIT_FIXED" || !DPCCRunNormalizationSection(document, admission.model_binding, dataset, hash, reason) || hash != binding.normalization_section_sha256) return(Refuse("BLOCKED.fixed_norm.actual_Model_FIT_FIXED_section", reason)); SDPCCResourceReference component, linked_dataset; const int components = document.FindMember(document.Root(), "components"); const int training = document.FindMember(document.Root(), "training"); if(!DPCCBankReadReference(document, document.FindMember(components, "actor"), "file", component, reason, "model.components.actor.file") || !DPCCBankReadReference(document, training, "dataset_manifest", linked_dataset, reason, "model.dataset") || component.sha256 != binding.actor_sha256 || linked_dataset.sha256 != binding.dataset_sha256 || linked_dataset.id != dataset.dataset_id) return(Refuse("BLOCKED.fixed_norm.Model_component_data_binding", reason)); string index_reason; const SDPCCDatasetInterval fit = dataset.normalization_fit; if(!DPCCValidateDatasetTimeIndex(index, dataset.dataset_id, index_reason) || dataset.time_index.sha256 != binding.time_index_sha256 || dataset.time_index.id != index.time_index_id || dataset.data_source != index.data_source || dataset.timezone_mapping.sha256 != index.timezone_mapping.sha256 || !DPCCDatasetValidateInterval(fit, index, reason, "fixed_norm.fit") || fit.bar_start < dataset.fit.bar_start || fit.bar_end > dataset.fit.bar_end || fit.start_utc < dataset.fit.start_utc || fit.end_utc > dataset.fit.end_utc || binding.data_cutoff < fit.end_utc || binding.data_cutoff > dataset.fit.end_utc) return(Refuse("BLOCKED.fixed_norm.fit_interval_or_calculation_cutoff", reason)); const long total = fit.bar_end - fit.bar_start; if(total < 2 || total > INT_MAX / 4 || binding.record_cap < 2 || (ulong)total > binding.record_cap || ArraySize(fit_ohlc) != 4 * total || ArraySize(start_utc) != total || ArraySize(end_utc) != total) return(Refuse("BLOCKED.fixed_norm.complete_fit_rows", reason)); const int count = (int)total; string wire = "DPCC_INTERNAL_FIT_OHLC_FLOAT_ROWS_V1\n" + binding.dataset_sha256 + "\n" + binding.time_index_sha256 + "\n" + binding.normalization_section_sha256 + "\n"; for(int row = 0; row < count; row++) { const int position = (int)fit.bar_start + row; if(start_utc[row] != index.start_utc[position] || end_utc[row] != index.end_utc[position] || end_utc[row] > binding.data_cutoff) return(Refuse("BLOCKED.fixed_norm.fit_row_times", reason)); wire += (string)position + ":" + (string)start_utc[row] + ":" + (string)end_utc[row]; for(int channel = 0; channel < 4; channel++) { const float value = fit_ohlc[channel * count + row]; if(!MathIsValidNumber((double)value) || value <= 0) return(Refuse("BLOCKED.fixed_norm.fit_OHLC_values", reason)); URAGFloatBits raw; raw.value = value; wire += ":" + (string)raw.bits; } if(fit_ohlc[count + row] < MathMax(fit_ohlc[row], fit_ohlc[3 * count + row]) || fit_ohlc[2 * count + row] > MathMin(fit_ohlc[row], fit_ohlc[3 * count + row])) return(Refuse("BLOCKED.fixed_norm.fit_OHLC_geometry", reason)); wire += "\n"; } if(!DPCCBankSourceHash(wire, hash, reason, "fixed_norm.fit_rows") || hash != binding.fit_rows_sha256) return(Refuse("BLOCKED.fixed_norm.fit_rows_hash", reason)); //--- This is the exact frozen whole-fit helper, not native PeriodNorm arithmetic. const string profile = "DPCC_FIT_FIXED_POPULATION_WELFORD_V1"; const string implementation = "dc36436fc0ddacab261a3fc7e249fd3491e1b28eb4a21af2991af4fcce0109c0"; if(binding.calculation_profile != profile || binding.calculation_source_sha256 != implementation || binding.record_cap == 0) return(Refuse("BLOCKED.fixed_norm.frozen_whole_fit_calculation_profile", reason)); SDPCCFitPriceNorm16 observed; if(!DPCCFitPriceNorm16(fit_ohlc, (uint)count, binding.record_cap, observed, reason)) return(false); string statistics_wire = "DPCC_INTERNAL_FIT_STATISTICS_V1\n" + profile + "\n" + implementation + "\n" + binding.fit_rows_sha256 + "\n" + (string)binding.data_cutoff + "\n"; for(int channel = 0; channel < 4; channel++) { const int target = 4 * channel; m_profile[target] = observed.mean[channel]; m_profile[target + 1] = observed.std[channel]; m_profile[target + 2] = observed.gain[channel]; m_profile[target + 3] = observed.bias[channel]; const float variance = observed.std[channel] * observed.std[channel]; m_snapshot[target] = observed.mean[channel]; m_snapshot[target + 1] = (float)MathSqrt((double)variance); m_snapshot[target + 2] = observed.gain[channel]; m_snapshot[target + 3] = observed.bias[channel]; for(int field = 0; field < 4; field++) { URAGFloatBits raw; raw.value = m_profile[target + field]; statistics_wire += (string)raw.bits + ":"; } URAGFloatBits raw_variance; raw_variance.value = variance; statistics_wire += (string)raw_variance.bits + "\n"; } if(!DPCCBankSourceHash(statistics_wire, m_statistics_sha256, reason, "fixed_norm.statistics")) return(false); m_binding = binding; m_model_source = actual_model_source; m_calculated = true; reason = ""; return(true); } //--- Exact effective native PriceNorm profile, owned copy only; not an Actor admission. //--- Native holder stores float variance=raw_std*raw_std; std is sqrt(variance). bool PreparedTrainingPriceNorm(float &profile[], string &reason) { ArrayResize(profile, 0); if(!m_calculated || ArrayResize(profile, 16) != 16) {reason = "BLOCKED.fixed_norm.calculated_training_profile_required"; return(false);} for(int i = 0; i < 16; i++) profile[i] = m_snapshot[i]; reason = ""; return(true); } bool PreparedMetadata(SDPCCFullFitSource &binding, string &statistics_sha256) { ZeroMemory(binding); statistics_sha256 = ""; if(!m_calculated) return(false); binding = m_binding; statistics_sha256 = m_statistics_sha256; return(true); } //--- This API prepares parameters only. It does not adopt a saved/native-loaded artifact. //--- Selected checkpoint verification and import are delegated to the canonical owner. bool AssembleSelected(CNet &destination, const SDPCCConfig &training, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder, const SDPCCDenoiserComposition &denoiser, const double &beta[], const double &alpha[], const double &sigma[], const SDPCCCheckpointArtifact &selected_decoder, const SDPCCCheckpointArtifact &selected_denoiser, const string expected_checkpoint_binding, const ulong maximum_bytes, CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; if(!m_calculated) return(Refuse("BLOCKED.fixed_norm.actual_fit_source_not_calculated", reason)); if(training.shape.K != m_shapes.K || training.shape.V != m_shapes.V || training.shape.H != m_shapes.H || training.shape.D != m_shapes.D || training.shape.M != m_shapes.M || training.shape.J != m_shapes.J || training.shape.E != m_shapes.E || training.shape.NActions != m_shapes.A || training.shape.StateSize != m_shapes.account_features || training.price_basis != m_values.price_basis || training.volume_scale != m_values.volume_scale || training.price_scale != m_values.price_scale || training.money_scale != m_values.money_scale || ArraySize(beta) != ArraySize(m_values.beta) || ArraySize(alpha) != ArraySize(m_values.alpha_bar) || ArraySize(sigma) != ArraySize(m_values.sigma)) return(Refuse("BLOCKED.fixed_norm.actual_Model_shape_scales", reason)); for(int i = 0; i < 6; i++) if(training.action_bias[i] != m_values.action_bias[i] || training.action_scale[i] != m_values.action_scale[i]) return(Refuse("BLOCKED.fixed_norm.actual_Model_action_affine", reason)); for(int i = 0; i < ArraySize(beta); i++) if(beta[i] != m_values.beta[i] || alpha[i] != m_values.alpha_bar[i] || sigma[i] != m_values.sigma[i]) return(Refuse("BLOCKED.fixed_norm.actual_Model_schedule", reason)); SDPCCConfig derived; float limits[]; if(!DPCCDecodePreparedActorConfig(m_model_source, training, derived, limits, reason) || DPCCFixedSignature(derived, defNeuronDPCCOCL) != DPCCFixedSignature(config, defNeuronDPCCOCL)) return(Refuse("BLOCKED.fixed_norm.actual_Model_Actor_config", reason)); CNeuronDPCCOCL *candidate = NULL; if(!DPCCAssembleSelectedCheckpoints(destination, training, derived, decoder, denoiser, beta, alpha, sigma, selected_decoder, selected_denoiser, expected_checkpoint_binding, m_profile, maximum_bytes, candidate, reason)) return(false); if(!candidate || !ProfileMatches(candidate, reason)) return(false); //--- Successful preparation is not a native artifact/model-hash adoption. actor = candidate; reason = ""; return(true); } bool BindLoaded(CNet &actor, const SDPCCFullFitSource &binding, const string actual_model_source, string &reason) { m_valid = false; m_actor = NULL; if(!m_calculated || !SameSource(binding)) return(Refuse("BLOCKED.fixed_norm.prepared_source_profile_binding", reason)); CDPCCJsonDocument document; string hash, format; bool prepared = false; if(!DPCCBankSourceHash(actual_model_source, hash, reason, "fixed_norm.loaded_Model") || hash != m_binding.model_sha256 || !document.Parse(actual_model_source, reason)) return(false); const int components = document.FindMember(document.Root(), "components"); const int actor_node = document.FindMember(components, "actor"); SDPCCResourceReference reference; if(!DPCCBankReadReference(document, actor_node, "file", reference, reason, "model.actor.file") || !DPCCResourceReadString(document, actor_node, "format", format, reason, "model.actor.format") || !DPCCResourceReadBool(document, actor_node, "prepared", prepared, reason, "model.actor.prepared") || reference.sha256 != binding.actor_sha256 || format != "CNET_BINARY" || !prepared) return(Refuse("BLOCKED.fixed_norm.actual_prepared_Model_Actor", reason)); CNeuronDPCCOCL *candidate = dynamic_cast(actor.Layer(-1)); if(!candidate || !actor.IsOpenCLTransferReady() || !actor.GetOpenCL() || candidate.GetOpenCL() != actor.GetOpenCL() || binding.actor_contract == 0 || binding.actor_weights == 0 || binding.actor_trainable == 0 || candidate.ContractSignature() != binding.actor_contract || candidate.WeightSignature() != binding.actor_weights || candidate.TrainableStateFingerprint() != binding.actor_trainable || !ProfileMatches(candidate, reason)) return(Refuse("BLOCKED.fixed_norm.admitted_loaded_Actor_identity_or_statistics", reason)); m_binding = binding; m_actor = candidate; m_valid = true; reason = ""; return(true); } bool Initialize(CNet &actor, const string actual_model_source, const SDPCCTrainingConfigAdmission &admission, const string actual_dataset_source, const string actual_time_index_source, const SDPCCFullFitSource &binding, const float &fit_ohlc[], const long &start_utc[], const long &end_utc[], string &reason) { return(PrepareSource(actual_model_source, admission, actual_dataset_source, actual_time_index_source, binding, fit_ohlc, start_utc, end_utc, reason) && BindLoaded(actor, binding, actual_model_source, reason)); } bool Matches(COpenCLMy *context, const ulong contract, const ulong weights, const ulong trainable) { return(m_valid && CheckPointer(m_actor) != POINTER_INVALID && m_actor.GetOpenCL() == context && contract == m_binding.actor_contract && weights == m_binding.actor_weights && trainable == m_binding.actor_trainable); } bool Copy(const string model_sha256, const string section_sha256, const long decision_time, float &price_norm[], string &reason) { ArrayResize(price_norm, 0); if(!m_valid || CheckPointer(m_actor) == POINTER_INVALID || model_sha256 != m_binding.model_sha256 || section_sha256 != m_binding.normalization_section_sha256 || decision_time < m_binding.data_cutoff || m_actor.ContractSignature() != m_binding.actor_contract || m_actor.WeightSignature() != m_binding.actor_weights || m_actor.TrainableStateFingerprint() != m_binding.actor_trainable) return(Refuse("BLOCKED.fixed_norm.frozen_profile_identity_or_cutoff", reason)); if(ArrayResize(price_norm, 16) != 16) return(Refuse("fixed_norm.output_capacity", reason)); for(int i = 0; i < 16; i++) price_norm[i] = m_snapshot[i]; reason = ""; return(true); } private: bool SameSource(const SDPCCFullFitSource &value) { return(value.model_sha256 == m_binding.model_sha256 && value.dataset_sha256 == m_binding.dataset_sha256 && value.time_index_sha256 == m_binding.time_index_sha256 && value.normalization_section_sha256 == m_binding.normalization_section_sha256 && value.fit_rows_sha256 == m_binding.fit_rows_sha256 && value.actor_sha256 == m_binding.actor_sha256 && value.data_cutoff == m_binding.data_cutoff && value.record_cap == m_binding.record_cap && value.calculation_profile == m_binding.calculation_profile && value.calculation_source_sha256 == m_binding.calculation_source_sha256); } bool ProfileMatches(CNeuronDPCCOCL *actor, string &reason) { if(!m_calculated || CheckPointer(actor) == POINTER_INVALID) return(Refuse("fixed_norm.native_profile_lifecycle", reason)); const ulong contract = actor.ContractSignature(), weights = actor.WeightSignature(), state = actor.TrainableStateFingerprint(); float saved[]; if(contract == 0 || weights == 0 || state == 0 || !actor.ReadNormalizationSnapshot(saved) || ArraySize(saved) != 16) return(Refuse("fixed_norm.native_profile_read", reason)); for(int i = 0; i < 16; i++) { URAGFloatBits expected, observed; expected.value = m_snapshot[i]; observed.value = saved[i]; if(expected.bits != observed.bits) return(Refuse("BLOCKED.fixed_norm.saved_Actor_vs_WELFORD_V1_fit_statistics", reason)); } if(actor.ContractSignature() != contract || actor.WeightSignature() != weights || actor.TrainableStateFingerprint() != state) return(Refuse("fixed_norm.Actor_changed", reason)); return(true); } }; #endif #ifndef DPCC_STAGE_FIT_SOURCE_SCHEMA_MQH #define DPCC_STAGE_FIT_SOURCE_SCHEMA_MQH string DPCCStageFitSourceSchema(void) { return("{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"title\":\"Internal DPCC stage fit rows, not a Model role or readiness evidence\",\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"schema_version\":{\"const\":1},\"source_kind\":{\"const\":\"DPCC_STAGE_FIT_OHLC_ROWS_V1\"},\"model_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"dataset_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"time_index_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"normalization_section_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"fit_rows_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"calculation_source_sha256\":{\"type\":\"string\",\"pattern\":\"^[0-9a-f]{64}$\"},\"calculation_profile\":{\"const\":\"DPCC_FIT_FIXED_POPULATION_WELFORD_V1\"},\"data_cutoff\":{\"type\":\"integer\",\"minimum\":0},\"record_cap\":{\"type\":\"integer\",\"minimum\":2},\"rows\":{\"type\":\"array\",\"minItems\":2,\"items\":{\"type\":\"object\",\"additionalProperties\":false,\"properties\":{\"bar_index\":{\"type\":\"integer\",\"minimum\":0},\"start_utc\":{\"type\":\"integer\",\"minimum\":0},\"end_utc\":{\"type\":\"integer\",\"minimum\":0},\"open\":{\"type\":\"number\",\"exclusiveMinimum\":0},\"high\":{\"type\":\"number\",\"exclusiveMinimum\":0},\"low\":{\"type\":\"number\",\"exclusiveMinimum\":0},\"close\":{\"type\":\"number\",\"exclusiveMinimum\":0}},\"required\":[\"bar_index\",\"start_utc\",\"end_utc\",\"open\",\"high\",\"low\",\"close\"]}}},\"required\":[\"schema_version\",\"source_kind\",\"model_sha256\",\"dataset_sha256\",\"time_index_sha256\",\"normalization_section_sha256\",\"fit_rows_sha256\",\"calculation_source_sha256\",\"calculation_profile\",\"data_cutoff\",\"record_cap\",\"rows\"]}"); } #endif //--- Internal operational source, never a Model component or a READY receipt. //--- Reference SHA authenticates these exact rows; their physical origin remains //--- part of the caller's admitted source policy, not inferred from a nonempty ID. struct SDPCCStageFitSource { SDPCCResourceReference reference; string package_root, containing_json; ulong maximum_bytes; bool common_files; }; struct SDPCCStageFitReceipt { string payload_sha256, fit_rows_sha256, normalization_section_sha256; string calculation_profile, calculation_source_sha256, statistics_sha256; long data_cutoff; ulong record_cap; }; //+------------------------------------------------------------------+ //| Function DPCCStageFitBlocked | //+------------------------------------------------------------------+ bool DPCCStageFitBlocked(const string prerequisite, string &reason) { reason = "BLOCKED:study_fit." + prerequisite; return(false); } //+------------------------------------------------------------------+ //| Function DPCCStageFitPrepare | //+------------------------------------------------------------------+ bool DPCCStageFitPrepare(const SDPCCStageFitSource &source, const string actual_model, const SDPCCTrainingConfigAdmission &admission, const string actual_dataset, const string actual_index, CDPCCFullFitNormalization &prepared, SDPCCStageFitReceipt &receipt, string &reason) { prepared.Reset(); ZeroMemory(receipt); reason = ""; if(source.maximum_bytes == 0 || source.maximum_bytes > DPCC_RESOLVER_MAX_JSON_BYTES || !DPCCResolverLowerSha256(source.reference.sha256)) return(DPCCStageFitBlocked("immutable_source_reference_and_capacity", reason)); SDPCCArtifactResolveResult artifact; string raw = ""; if(!DPCCResolveArtifact(source.package_root, source.containing_json, source.reference, source.common_files, (long)source.maximum_bytes, artifact, reason) || !DPCCCentralBytesSource(artifact.bytes, raw, reason, "study_fit")) return(false); CDPCCJsonDocument document; if(!DPCCCentralSchemaValidate(document, raw, DPCCStageFitSourceSchema(), reason, "study_fit")) return(false); SDPCCFullFitSource binding; ZeroMemory(binding); const int root = document.Root(); string kind = ""; ulong cap = 0; if(!DPCCResourceReadString(document, root, "source_kind", kind, reason, "study_fit.source_kind") || kind != "DPCC_STAGE_FIT_OHLC_ROWS_V1" || !DPCCResourceReadString(document, root, "model_sha256", binding.model_sha256, reason, "study_fit.model") || !DPCCResourceReadString(document, root, "dataset_sha256", binding.dataset_sha256, reason, "study_fit.dataset") || !DPCCResourceReadString(document, root, "time_index_sha256", binding.time_index_sha256, reason, "study_fit.index") || !DPCCResourceReadString(document, root, "normalization_section_sha256", binding.normalization_section_sha256, reason, "study_fit.normalization") || !DPCCResourceReadString(document, root, "fit_rows_sha256", binding.fit_rows_sha256, reason, "study_fit.rows_sha") || !DPCCResourceReadString(document, root, "calculation_profile", binding.calculation_profile, reason, "study_fit.profile") || !DPCCResourceReadString(document, root, "calculation_source_sha256", binding.calculation_source_sha256, reason, "study_fit.helper") || !DPCCResourceReadLong(document, root, "data_cutoff", binding.data_cutoff, reason, "study_fit.cutoff") || document.Type(document.FindMember(root, "record_cap")) != DPCC_JSON_NUMBER || !DPCCCheckpointReadUnsigned(document.Text(document.FindMember(root, "record_cap")), ULONG_MAX, cap)) return(DPCCStageFitBlocked("typed_source_fields", reason)); binding.record_cap = cap; //--- Actor reference comes from actual admitted Model bytes, never a guessed future hash. CDPCCJsonDocument model, data; SDPCCDatasetManifest dataset; SDPCCResourceReference actor_ref; if(!model.Parse(actual_model, reason) || !data.Parse(actual_dataset, reason) || !DPCCDecodeDatasetManifest(data, data.Root(), dataset, reason) || !DPCCBankReadReference(model, model.FindMember(model.FindMember(model.Root(), "components"), "actor"), "file", actor_ref, reason, "study_fit.actual_Model.actor")) return(false); binding.actor_sha256 = actor_ref.sha256; const long total = dataset.normalization_fit.bar_end - dataset.normalization_fit.bar_start; const int rows_node = document.FindMember(root, "rows"); if(total < 2 || total > INT_MAX / 4 || cap < 2 || (ulong)total > cap || document.ChildCount(rows_node) != total) return(DPCCStageFitBlocked("complete_fit_row_count_and_cap", reason)); const int rows = (int)total; float ohlc[]; long starts[], ends[]; if(ArrayResize(ohlc, 4 * rows) != 4 * rows || ArrayResize(starts, rows) != rows || ArrayResize(ends, rows) != rows) return(DPCCStageFitBlocked("fit_materialization_capacity", reason)); const string channels[] = {"open", "high", "low", "close"}; for(int row = 0; row < rows; row++) { const int node = document.ChildAt(rows_node, row); long ordinal = 0; if(!DPCCResourceReadLong(document, node, "bar_index", ordinal, reason, "study_fit.ordinal") || ordinal != dataset.normalization_fit.bar_start + row || !DPCCResourceReadLong(document, node, "start_utc", starts[row], reason, "study_fit.start") || !DPCCResourceReadLong(document, node, "end_utc", ends[row], reason, "study_fit.end")) return(DPCCStageFitBlocked("fit_row_ordinals_and_times", reason)); for(int channel = 0; channel < 4; channel++) { double value = 0; if(!DPCCResourceReadDouble(document, node, channels[channel], value, reason, "study_fit.OHLC") || value <= 0) return(DPCCStageFitBlocked("fit_OHLC_numeric_domain", reason)); ohlc[channel * rows + row] = (float)value; if(!MathIsValidNumber((double)ohlc[channel * rows + row]) || ohlc[channel * rows + row] <= 0) return(DPCCStageFitBlocked("fit_OHLC_float_domain", reason)); } } //--- Existing observer enforces exact actual source closure, all UTC rows, fit //--- interval/cutoff, float-row digest, frozen helper/profile, population stats. if(!prepared.PrepareSource(actual_model, admission, actual_dataset, actual_index, binding, ohlc, starts, ends, reason)) return(false); string statistics = ""; SDPCCFullFitSource calculated; if(!prepared.PreparedMetadata(calculated, statistics)) return(DPCCStageFitBlocked("private_calculated_statistics", reason)); receipt.payload_sha256 = artifact.observed_sha256; receipt.fit_rows_sha256 = calculated.fit_rows_sha256; receipt.normalization_section_sha256 = calculated.normalization_section_sha256; receipt.calculation_profile = calculated.calculation_profile; receipt.calculation_source_sha256 = calculated.calculation_source_sha256; receipt.statistics_sha256 = statistics; receipt.data_cutoff = calculated.data_cutoff; receipt.record_cap = calculated.record_cap; return(true); } #endif class CDPCCEpochOwner { private: CDPCCTrainingExamples m_examples; CDPCCAdmittedTrainingConfig m_admitted; CDPCCFrozenRolePlan m_decoder_plan, m_denoiser_plan; SDPCCTrainingRoleConfig m_decoder_role, m_denoiser_role; SDPCCEpochProgress m_progress, m_decoder_progress, m_denoiser_progress; SDPCCEpochHistory m_history[]; string m_run_binding, m_confirmed_path, m_confirmed_sha; bool m_restore_required; bool m_fixed_required, m_fixed_admitted; float m_fixed_profile[16]; SDPCCStageFitReceipt m_fixed_receipt; SDPCCCheckpointArtifact m_initial, m_latest, m_decoder_best, m_denoiser_best; string m_stem; bool m_common, m_initialized; ulong m_training_seed, m_validation_seed, m_actor_maximum_bytes; bool Failure(const string cause, string &reason) {m_progress.failed = true; reason = cause; return(false);} bool DecoderSelectionBinding(string &reason) { if(!m_initialized || m_restore_required || m_progress.failed || !m_decoder_progress.complete || !m_decoder_progress.has_best || StringLen(m_decoder_best.manifest_sha256) == 0) {reason = "epoch.decoder_not_selected"; return(false);} if(m_decoder_best.cursor.stage != 1 || m_decoder_best.binding_sha256 != m_initial.binding_sha256 || m_decoder_best.cursor.decoder_updates == 0 || m_decoder_best.cursor.denoiser_updates != 0) {reason = "epoch.decoder_selected_binding"; return(false);} return(true); } bool StateCheckpoint(const uint stage, const uint epoch, SDPCCCheckpointArtifact &artifact, string &reason) { SDPCCCheckpointCursor cursor; ZeroMemory(cursor); ulong decoder_state = 0, denoiser_state = 0; if(!m_examples.NativeState(cursor.decoder_updates, cursor.denoiser_updates, decoder_state, denoiser_state)) return(Failure("epoch.native_state", reason)); cursor.stage = stage; cursor.completed_epoch = epoch; cursor.next_epoch = epoch + 1; cursor.next_record = 0; const string path = m_stem + ".stage" + (string)stage + ".epoch" + (string)epoch; if(!m_examples.SaveVerifiedCheckpointArtifact(path, m_common, cursor, artifact, reason)) return(Failure(reason, reason)); return(true); } bool ValidateRole(CDPCCFrozenRolePlan &plan, const bool denoiser, double &loss, string &reason) { const int count = plan.ValidationCount(); if(count <= 0) return(Failure("epoch.validation_empty", reason)); uint db = 0, nb = 0, da = 0, na = 0; ulong dsb = 0, nsb = 0, dsa = 0, nsa = 0; if(!m_examples.NativeState(db, nb, dsb, nsb)) return(Failure("epoch.validation_state_before", reason)); double sum = 0; for(int i = 0; i < count; i++) { SDPCCTrainingPlanEntry entry; SDPCCTrainingRecordIdentity identity; SDPCCTrainingExampleResult result; if(!plan.Validation(i, entry) || !m_examples.RecordIdentity(entry.record_index, identity)) return(Failure("epoch.validation_identity", reason)); bool ok = false; if(denoiser) { SDPCCTrainingExampleKeys keys; keys.j = entry.j; keys.repeat = entry.repeat; keys.seed = m_validation_seed; keys.decision_time = identity.decision_time; keys.decision_identity = identity.keys.decision_identity; keys.context_identity = identity.keys.context_identity; keys.owner_identity = identity.keys.owner_identity; ok = m_examples.DenoiserExample(entry.record_index, DPCC_TRAIN_VALIDATE, keys, result); } else ok = m_examples.DecoderExample(entry.record_index, DPCC_TRAIN_VALIDATE, result); if(!ok || !result.completed || result.update_attempted || !MathIsValidNumber(result.loss)) return(Failure("epoch.validation:" + result.reason, reason)); sum += result.loss; if(!MathIsValidNumber(sum)) return(Failure("epoch.validation_sum", reason)); } if(!m_examples.NativeState(da, na, dsa, nsa) || db != da || nb != na || dsb != dsa || nsb != nsa) return(Failure("epoch.validation_mutation", reason)); //--- Explicit C1 aggregation: equal-example arithmetic mean of native losses. loss = sum / (double)count; return(MathIsValidNumber(loss) || Failure("epoch.validation_mean", reason)); } string RunStateText(const SDPCCCheckpointArtifact &initial, const SDPCCCheckpointArtifact &latest, const SDPCCCheckpointArtifact &decoder_best, const SDPCCCheckpointArtifact &denoiser_best, const SDPCCEpochProgress &decoder, const SDPCCEpochProgress &denoiser, const uint current_role, const bool complete, const SDPCCEpochHistory &history[]) { string text = "DPCC_EPOCH_BOUNDARY_STATE_V1\n" + m_run_binding + "\n" + initial.manifest_path + "\n" + initial.manifest_sha256 + "\n" + latest.manifest_path + "\n" + latest.manifest_sha256 + "\n" + decoder_best.manifest_path + "\n" + decoder_best.manifest_sha256 + "\n" + denoiser_best.manifest_path + "\n" + denoiser_best.manifest_sha256 + "\n" + (string)current_role + "\n" + (string)(uint)complete + "\n" + (string)ArraySize(history) + "\n" + DPCCEpochProgressWire(decoder) + DPCCEpochProgressWire(denoiser); for(int i = 0; i < ArraySize(history); i++) text += (string)history[i].role + "\n" + (string)history[i].epoch + "\n" + (string)( uint)history[i].validation + "\n" + StringFormat("%.17g\n", history[i].loss) + history[i].path + "\n" + history[i].sha + "\n"; return(text + "END_DPCC_EPOCH_BOUNDARY_STATE_V1\n"); } bool PersistRunState(string &reason) { const string text = RunStateText(m_initial, m_latest, m_decoder_best, m_denoiser_best, m_decoder_progress, m_denoiser_progress, m_progress.role, m_progress.complete, m_history); const string path = m_stem + ".boundary" + (string)ArraySize(m_history) + ".dpccrun"; string canonical = "", expected = "", actual = ""; if(!DPCCCheckpointCanonicalPath(path, canonical, reason) || !DPCCCheckpointTextSHA(text, expected, reason) || !DPCCCheckpointWriteText(canonical, m_common, text, reason) || !DPCCCheckpointFileSHA(canonical, m_common, 4194304, actual, reason) || actual != expected) return(Failure("epoch.runstate.publish:" + reason, reason)); m_confirmed_path = canonical; m_confirmed_sha = actual; return(true); } bool ReplayEpoch(const SDPCCEpochHistory &h, const SDPCCCheckpointArtifact &artifact, CDPCCFrozenRolePlan &plan, const SDPCCTrainingRoleConfig &role, SDPCCEpochProgress &progress, SDPCCCheckpointArtifact &best, const uint other_updates, string &reason) { const int fit_count = plan.FitCount(); if(role.max_epochs <= 0 || fit_count <= 0 || fit_count % role.max_epochs != 0 || progress.complete || h.role != progress.role || h.epoch != progress.completed_epoch + 1 || (long)h.epoch > role.max_epochs || artifact.cursor.stage != h.role + 1 || artifact.cursor.completed_epoch != h.epoch || h.validation != DPCCEpochValidationDue(h.epoch, role)) {reason = "epoch.resume.history_order"; return(false);} const uint expected_updates = h.epoch * (uint)(fit_count / role.max_epochs); if((h.role == 0 && (artifact.cursor.decoder_updates != expected_updates || artifact.cursor.denoiser_updates != other_updates)) || (h.role == 1 && (artifact.cursor.denoiser_updates != expected_updates || artifact.cursor.decoder_updates != other_updates))) {reason = "epoch.resume.plan_update_count"; return(false);} double dl = 0, nl = 0; if(!m_examples.VerifyRunnerArtifact(artifact, dl, nl, reason)) return(false); if(h.validation) { const double actual = (h.role == 0 ? dl : nl); if(actual != h.loss) { reason = "epoch.resume.actual_validation_loss"; return(false); } bool improved = false; if(!DPCCEpochObserve(actual, role, progress, improved)) { reason = "epoch.resume.validation_transition"; return(false); } if(improved) best = artifact; } else if(h.loss != 0) { reason = "epoch.resume.unperformed_validation"; return(false); } progress.completed_epoch = h.epoch; progress.fit_cursor = expected_updates; progress.complete = DPCCEpochFinished(h.epoch, h.validation, role, progress); if(progress.complete && !progress.has_best) { reason = "epoch.resume.no_best"; return(false); } return(true); } bool RestoreRunStateImpl(const string path, const string expected_sha, string &reason) { if(!m_initialized || !m_restore_required || m_progress.failed) {reason = "epoch.resume.destination"; return(false);} string canonical = "", observed = "", source = ""; uchar bytes[]; if(!DPCCResolverLowerSha256(expected_sha) || !DPCCCheckpointCanonicalPath(path, canonical, reason) || !DPCCReadArtifactBytes(canonical, m_common, 4194304, bytes, reason) || !DPCCSHA256Lower(bytes, observed, reason) || observed != expected_sha || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 4194304, source, reason)) {reason = "epoch.resume.state_bytes"; return(false);} string fields[]; const int size = StringSplit(source, 10, fields); ulong count = 0; if(size < 35 || fields[0] != "DPCC_EPOCH_BOUNDARY_STATE_V1" || fields[1] != m_run_binding || !DPCCCheckpointReadUnsigned(fields[12], UINT_MAX, count) || count > (ulong)m_decoder_role.max_epochs + (ulong)m_denoiser_role.max_epochs || (ulong)size != 35 + count * 6 || fields[size - 2] != "END_DPCC_EPOCH_BOUNDARY_STATE_V1" || fields[size - 1] != "") {reason = "epoch.resume.state_contract"; return(false);} SDPCCCheckpointArtifact initial, latest, decoder_best, denoiser_best; if(!DPCCReadCheckpointArtifact(fields[2], m_common, fields[3], initial, reason) || initial.cursor.stage != 0) return(false); double initial_dl = 0, initial_nl = 0; if(!m_examples.VerifyRunnerArtifact(initial, initial_dl, initial_nl, reason)) return(false); latest = initial; ZeroMemory(decoder_best); ZeroMemory(denoiser_best); SDPCCEpochProgress decoder, denoiser; ZeroMemory(decoder); ZeroMemory(denoiser); denoiser.role = 1; SDPCCEpochHistory history[]; if(ArrayResize(history, (int)count) != (int)count) { reason = "epoch.resume.capacity"; return(false); } for(int i = 0; i < (int)count; i++) { const int offset = 33 + i * 6; ulong role = 0, epoch = 0, validation = 0; if(!DPCCCheckpointReadUnsigned(fields[offset], 1, role) || !DPCCCheckpointReadUnsigned(fields[offset + 1], UINT_MAX, epoch) || !DPCCCheckpointReadUnsigned(fields[offset + 2], 1, validation) || !DPCCEpochNumber(fields[offset + 3], history[i].loss, reason)) {reason = "epoch.resume.history_fields"; return(false);} history[i].role = (uint)role; history[i].epoch = (uint)epoch; history[i].validation = (bool)validation; history[i].path = fields[offset + 4]; history[i].sha = fields[offset + 5]; SDPCCCheckpointArtifact artifact; if(!DPCCReadCheckpointArtifact(history[i].path, m_common, history[i].sha, artifact, reason) || artifact.binding_sha256 != initial.binding_sha256 || artifact.initial_manifest_path != initial.manifest_path || artifact.initial_manifest_sha256 != initial.manifest_sha256) {reason = "epoch.resume.history_native_binding"; return(false);} if(role == 0) { if(denoiser.completed_epoch != 0 || !ReplayEpoch(history[i], artifact, m_decoder_plan, m_decoder_role, decoder, decoder_best, 0, reason)) return(false); } else { if(!decoder.complete || !ReplayEpoch(history[i], artifact, m_denoiser_plan, m_denoiser_role, denoiser, denoiser_best, decoder.fit_cursor, reason)) return(false); } latest = artifact; } const uint current_role = (decoder.complete ? 1 : 0); const bool complete = denoiser.complete; if(RunStateText(initial, latest, decoder_best, denoiser_best, decoder, denoiser, current_role, complete, history) != source) {reason = "epoch.resume.replayed_state"; return(false);} //--- Rehash durable run state after all linked candidate probes, before adoption. if(!DPCCCheckpointFileSHA(canonical, m_common, 4194304, observed, reason) || observed != expected_sha) {reason = "epoch.resume.state_changed"; return(false);} if(ArrayResize(m_history, (int)count) != (int)count) { reason = "epoch.resume.adoption_capacity"; return(false); } SDPCCCheckpointCursor restored; if(!m_examples.RestoreTrainingCheckpoint(latest, restored, reason)) return(Failure("epoch.resume.native_adoption:" + reason, reason)); //--- No fallible allocation or external checks after native pair adoption. m_initial = initial; m_latest = latest; m_decoder_best = decoder_best; m_denoiser_best = denoiser_best; m_decoder_progress = decoder; m_denoiser_progress = denoiser; if(current_role == 0) m_progress = decoder; else m_progress = denoiser; for(int i = 0; i < (int)count; i++) m_history[i] = history[i]; m_confirmed_path = canonical; m_confirmed_sha = expected_sha; m_restore_required = false; return(true); } bool AdvanceRole(CDPCCFrozenRolePlan &plan, const SDPCCTrainingRoleConfig &role, const bool denoiser, string &reason) { if(m_fixed_required && !m_fixed_admitted) return(Failure("BLOCKED.epoch.fixed_profile_before_UPDATE", reason)); SDPCCTrainingPlanEntry entry; if(!plan.Fit((int)m_progress.fit_cursor, entry)) return(Failure("epoch.plan_cursor", reason)); SDPCCTrainingRecordIdentity identity; if(!m_examples.RecordIdentity(entry.record_index, identity) || identity.window_id != entry.window_id) return(Failure("epoch.fit_identity", reason)); SDPCCTrainingExampleResult result; bool ok = false; if(denoiser) { SDPCCTrainingExampleKeys keys; keys.j = entry.j; keys.repeat = entry.repeat; keys.seed = m_training_seed; keys.decision_time = identity.decision_time; keys.decision_identity = identity.keys.decision_identity; keys.context_identity = identity.keys.context_identity; keys.owner_identity = identity.keys.owner_identity; ok = m_examples.DenoiserExample(entry.record_index, DPCC_TRAIN_UPDATE, keys, result); } else ok = m_examples.DecoderExample(entry.record_index, DPCC_TRAIN_UPDATE, result); if(!ok || !result.completed || !result.update_attempted || result.updates_before == UINT_MAX || result.updates_after != result.updates_before + 1 || !MathIsValidNumber(result.loss)) return(Failure("epoch.update:" + result.reason, reason)); //--- Only this executed native update grants advancement. m_progress.fit_cursor++; SDPCCTrainingPlanEntry next; const bool boundary = !plan.Fit((int)m_progress.fit_cursor, next) || next.epoch != entry.epoch; if(!boundary) return(true); m_progress.completed_epoch = entry.epoch; const bool validation = DPCCEpochValidationDue(entry.epoch, role); bool improved = false; double loss = 0; if(validation) { if(!ValidateRole(plan, denoiser, loss, reason)) return(false); if(!DPCCEpochObserve(loss, role, m_progress, improved)) return(Failure("epoch.validation_transition", reason)); } SDPCCCheckpointArtifact artifact; if(!StateCheckpoint(denoiser ? 2 : 1, entry.epoch, artifact, reason)) return(false); m_latest = artifact; if(validation && improved) { m_progress.best_loss = loss; m_progress.has_best = true; if(denoiser) m_denoiser_best = artifact; else m_decoder_best = artifact; } const bool finished = DPCCEpochFinished(entry.epoch, validation, role, m_progress); m_progress.complete = finished; if(denoiser) m_denoiser_progress = m_progress; else m_decoder_progress = m_progress; const int history_size = ArraySize(m_history); if(ArrayResize(m_history, history_size + 1) != history_size + 1) return(Failure("epoch.history_capacity", reason)); m_history[history_size].role = (denoiser ? 1 : 0); m_history[history_size].epoch = entry.epoch; m_history[history_size].validation = validation; m_history[history_size].loss = (validation ? loss : 0); m_history[history_size].path = artifact.manifest_path; m_history[history_size].sha = artifact.manifest_sha256; if(finished) { if(!m_progress.has_best) return(Failure("epoch.no_selected_checkpoint", reason)); if(denoiser) m_progress.complete = true; else { ZeroMemory(m_progress); m_progress.role = 1; } } return(PersistRunState(reason)); } public: CDPCCEpochOwner(void) {m_fixed_required = false; m_fixed_admitted = false; ZeroMemory( m_fixed_profile); ZeroMemory(m_fixed_receipt); m_initialized = false; m_common = false; m_training_seed = 0; m_validation_seed = 0; m_restore_required = false; ZeroMemory(m_progress); ZeroMemory(m_decoder_progress); ZeroMemory(m_denoiser_progress); m_denoiser_progress.role = 1;} bool Initialize(const SDPCCTrainingBankSource &source, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder_composition, const SDPCCDenoiserComposition &composition, const double &alpha[], const SDPCCTrainingFrozenRecordKeys &keys[], const ulong training_seed, const ulong validation_seed, const uint validation_repeat, const string package_root, const string containing_json, const SDPCCResourceReference &config_reference, const SDPCCResourceReference &model_reference, const string schema, const string model_schema, const string bank_schema, const string dataset_schema, const string time_schema, const SDPCCTrainingPlanEntry &decoder_fit[], const SDPCCTrainingPlanEntry &decoder_validation[], const string decoder_plan_sha, const SDPCCTrainingPlanEntry &denoiser_fit[], const SDPCCTrainingPlanEntry &denoiser_validation[], const string denoiser_plan_sha, const string checkpoint_stem, const ulong maximum_checkpoint_bytes, string &reason, const bool for_resume = false) { SDPCCStageFitSource absent; ZeroMemory(absent); return(InitializeWithFitSource(source, config, decoder_composition, composition, alpha, keys, training_seed, validation_seed, validation_repeat, package_root, containing_json, config_reference, model_reference, schema, model_schema, bank_schema, dataset_schema, time_schema, decoder_fit, decoder_validation, decoder_plan_sha, denoiser_fit, denoiser_validation, denoiser_plan_sha, checkpoint_stem, absent, maximum_checkpoint_bytes, reason, for_resume)); } bool InitializeWithFitSource(const SDPCCTrainingBankSource &source, const SDPCCConfig &config, const SDPCCDecoderComposition &decoder_composition, const SDPCCDenoiserComposition &composition, const double &alpha[], const SDPCCTrainingFrozenRecordKeys &keys[], const ulong training_seed, const ulong validation_seed, const uint validation_repeat, const string package_root, const string containing_json, const SDPCCResourceReference &config_reference, const SDPCCResourceReference &model_reference, const string schema, const string model_schema, const string bank_schema, const string dataset_schema, const string time_schema, const SDPCCTrainingPlanEntry &decoder_fit[], const SDPCCTrainingPlanEntry &decoder_validation[], const string decoder_plan_sha, const SDPCCTrainingPlanEntry &denoiser_fit[], const SDPCCTrainingPlanEntry &denoiser_validation[], const string denoiser_plan_sha, const string checkpoint_stem, const SDPCCStageFitSource &fit_source, const ulong maximum_checkpoint_bytes, string &reason, const bool for_resume = false) { reason = ""; if(m_initialized || m_progress.failed) return(Failure("epoch.destination_not_empty", reason)); if(!m_examples.Initialize(source, config, decoder_composition, composition, composition, .008, .999, alpha, keys, training_seed, validation_seed, validation_repeat, reason)) return(Failure(reason, reason)); if(!m_admitted.Admit(package_root, containing_json, config_reference, model_reference, source.common_files, schema, model_schema, bank_schema, dataset_schema, time_schema, source, reason)) return(Failure(reason, reason)); //--- Actual retained Model selects the policy, not a caller config or flag. string actual_model = "", dataset_source = "", index_source = "", observed = "", mode = ""; CDPCCJsonDocument model_document; SDPCCTrainingConfigAdmission fixed_admission; SDPCCTrainingRoleConfig fixed_role; if(!m_admitted.ModelSource(actual_model) || !model_document.Parse(actual_model, reason) || !DPCCResourceReadString(model_document, model_document.FindMember(model_document.Root(), "normalization"), "statistics_source", mode, reason, "epoch.actual_normalization_source")) return(Failure(reason, reason)); m_fixed_required = (mode == "FIT_FIXED"); if(mode != "FIT_FIXED" && mode != "INPUT_WINDOW") return(Failure("epoch.normalization_source", reason)); if(m_fixed_required) { if(!DPCCResolverLowerSha256(fit_source.reference.sha256) || fit_source.maximum_bytes == 0) return(Failure("BLOCKED.epoch.immutable_FIT_FIXED_source_before_UPDATE", reason)); if(fit_source.common_files != source.common_files) return(Failure("BLOCKED.epoch.fixed_source_namespace", reason)); if(!m_admitted.Get(DPCC_ROLE_DECODER, fixed_admission, fixed_role) || !DPCCDatasetReadSource(source.links.dataset_path, source.common_files, fixed_admission.dataset_sha256, dataset_source, observed, reason, "epoch.fit.dataset") || !DPCCDatasetReadSource(source.links.time_index_path, source.common_files, fixed_admission.time_index_sha256, index_source, observed, reason, "epoch.fit.index")) return(Failure(reason, reason)); uint db = 0, nb = 0, da = 0, na = 0; ulong ds = 0, ns = 0, dsa = 0, nsa = 0; if(!m_examples.NativeState(db, nb, ds, ns)) return(Failure("epoch.fit.initial_native_state", reason)); CDPCCFullFitNormalization prepared; SDPCCStageFitReceipt fit_receipt; float profile[]; if(!DPCCStageFitPrepare(fit_source, actual_model, fixed_admission, dataset_source, index_source, prepared, fit_receipt, reason) || !prepared.PreparedTrainingPriceNorm(profile, reason) || !m_examples.CheckFixedPriceNorm(profile, fit_receipt.data_cutoff, reason)) return(Failure(reason, reason)); if(!m_examples.NativeState(da, na, dsa, nsa) || db != da || nb != na || ds != dsa || ns != nsa) return(Failure("epoch.fit.admission_mutated_native_state", reason)); for(int i = 0; i < 16; i++) m_fixed_profile[i] = profile[i]; m_fixed_receipt = fit_receipt; m_fixed_admitted = true; } if(!m_decoder_plan.Admit(m_admitted, DPCC_ROLE_DECODER, config.shape.J, m_examples, 0, decoder_fit, decoder_validation, decoder_plan_sha, reason) || !m_denoiser_plan.Admit(m_admitted, DPCC_ROLE_DENOISER, config.shape.J, m_examples, validation_repeat, denoiser_fit, denoiser_validation, denoiser_plan_sha, reason)) return(Failure(reason, reason)); SDPCCTrainingConfigAdmission receipts, denoiser_receipts; if(!m_admitted.Get(DPCC_ROLE_DECODER, receipts, m_decoder_role) || !m_admitted.Get(DPCC_ROLE_DENOISER, denoiser_receipts, m_denoiser_role)) return(Failure("epoch.config_copy", reason)); SDPCCCheckpointBinding binding; binding.training_config_sha256 = receipts.source_sha256; binding.model_manifest_sha256 = receipts.model_sha256; binding.bank_manifest_sha256 = receipts.bank_manifest_sha256; binding.dataset_manifest_sha256 = receipts.dataset_sha256; binding.time_index_sha256 = receipts.time_index_sha256; binding.normalization_section_sha256 = receipts.normalization_section_sha256; binding.decoder_plan_sha256 = m_decoder_plan.Digest(); binding.denoiser_plan_sha256 = m_denoiser_plan.Digest(); if(!m_examples.BindCheckpointMetadata(binding, maximum_checkpoint_bytes, reason)) return(Failure(reason, reason)); m_stem = checkpoint_stem; m_common = source.common_files; m_training_seed = training_seed; m_validation_seed = validation_seed; m_actor_maximum_bytes = maximum_checkpoint_bytes; string binding_wire = "DPCC_EPOCH_BINDING_V1\n" + binding.training_config_sha256 + "\n" + binding.model_manifest_sha256 + "\n" + binding.bank_manifest_sha256 + "\n" + binding.dataset_manifest_sha256 + "\n" + binding.time_index_sha256 + "\n" + binding.normalization_section_sha256 + "\n" + binding.decoder_plan_sha256 + "\n" + binding.denoiser_plan_sha256 + "\n"; if(m_fixed_required) binding_wire += "DPCC_EPOCH_FIXED_PROFILE_V1\n" + m_fixed_receipt.payload_sha256 + "\n" + m_fixed_receipt.fit_rows_sha256 + "\n" + m_fixed_receipt.calculation_profile + "\n" + m_fixed_receipt.calculation_source_sha256 + "\n" + m_fixed_receipt.statistics_sha256 + "\n" + (string)m_fixed_receipt.data_cutoff + "\n"; if(!DPCCCheckpointTextSHA(binding_wire, m_run_binding, reason)) return(Failure(reason, reason)); m_restore_required = for_resume; if(for_resume) { m_initialized = true; return(true); } if(!StateCheckpoint(0, 0, m_initial, reason)) return(false); m_latest = m_initial; m_initialized = true; return(PersistRunState(reason)); } bool ConfirmedBoundary(string &path, string &sha) {path = m_confirmed_path; sha = m_confirmed_sha; return(m_initialized && !m_restore_required && !m_progress.failed && StringLen(sha) > 0);} bool RestoreConfirmedBoundary(const string path, const string sha, string &reason) {reason = ""; return(RestoreRunStateImpl(path, sha, reason));} bool RejectedInitializationNativeState(uint &decoder_updates, uint &denoiser_updates, ulong &decoder_state, ulong &denoiser_state) {return(m_progress.failed && m_examples.NativeState(decoder_updates, denoiser_updates, decoder_state, denoiser_state));} bool NativeFingerprint(uint &decoder_updates, uint &denoiser_updates, ulong &decoder_state, ulong &denoiser_state) {return(m_initialized && !m_restore_required && !m_progress.failed && m_examples.NativeState( decoder_updates, denoiser_updates, decoder_state, denoiser_state));} bool NormalizationStatisticsSource(string &mode, string &reason) { mode = ""; string source = ""; CDPCCJsonDocument model; if(!m_initialized || m_restore_required || m_progress.failed || !m_admitted.ModelSource(source) || !model.Parse( source, reason)) {reason = "study.normalization_actual_Model_required"; return(false);} if(!DPCCResourceReadString(model, model.FindMember(model.Root(), "normalization"), "statistics_source", mode, reason, "study.normalization_source")) return(false); if(mode != "INPUT_WINDOW" && mode != "FIT_FIXED") {reason = "study.normalization_source_unsupported"; return(false);} return(true); } bool PrepareSelectedActorFullFit(CNet &net, const SDPCCStageFitSource &fit_source, const string dataset_path, const string time_index_path, SDPCCConfig &config, float &limits[], CNeuronDPCCOCL *&actor, SDPCCStageFitReceipt &fit_receipt, string &reason) { actor = NULL; ZeroMemory(fit_receipt); reason = ""; if(!m_initialized || m_restore_required || m_progress.failed || !m_progress.complete) {reason = "actor.owner_run_not_selected"; return(false);} if(fit_source.common_files != m_common) return(DPCCStageFitBlocked("owner_namespace", reason)); SDPCCConfig training; SDPCCDecoderComposition decoder; SDPCCDenoiserComposition denoiser; SDPCCTrainingConfigAdmission receipt; SDPCCTrainingRoleConfig role; string model_source = "", dataset_source = "", index_source = "", observed = ""; double beta[], alpha[], sigma[], dl = 0, nl = 0; if(!m_examples.PreparedActorContract(training, decoder, denoiser, reason) || !m_admitted.ModelSource(model_source) || !m_admitted.Get(DPCC_ROLE_DECODER, receipt, role) || !DPCCDatasetReadSource(dataset_path, m_common, receipt.dataset_sha256, dataset_source, observed, reason, "study_fit.dataset") || !DPCCDatasetReadSource(time_index_path, m_common, receipt.time_index_sha256, index_source, observed, reason, "study_fit.index") || !DPCCDecodePreparedActorConfig(model_source, training, config, limits, reason) || !DPCCBuildCosineDiffusionSchedule(config.shape.J, receipt.model_binding.schedule_s, receipt.model_binding.beta_cap, beta, alpha, sigma, reason) || !m_examples.VerifyRunnerArtifact(m_decoder_best, dl, nl, reason) || !m_examples.VerifyRunnerArtifact(m_denoiser_best, dl, nl, reason)) return(false); CDPCCFullFitNormalization prepared; if(!DPCCStageFitPrepare(fit_source, model_source, receipt, dataset_source, index_source, prepared, fit_receipt, reason)) return(false); if(!m_fixed_admitted || fit_receipt.payload_sha256 != m_fixed_receipt.payload_sha256 || fit_receipt.statistics_sha256 != m_fixed_receipt.statistics_sha256 || fit_receipt.data_cutoff != m_fixed_receipt.data_cutoff) return(DPCCStageFitBlocked("original_preUPDATE_profile_binding", reason)); float assembly_profile[]; if(!prepared.PreparedTrainingPriceNorm(assembly_profile, reason)) return(false); for(int i = 0; i < 16; i++) { URAGFloatBits a, b; a.value = assembly_profile[i]; b.value = m_fixed_profile[i]; if(a.bits != b.bits) return(DPCCStageFitBlocked("original_preUPDATE_profile_bits", reason)); } return(prepared.AssembleSelected(net, training, config, decoder, denoiser, beta, alpha, sigma, m_decoder_best, m_denoiser_best, m_initial.binding_sha256, m_actor_maximum_bytes, actor, reason)); } bool PrepareSelectedActor(CNet &net, const int normalization_record, SDPCCConfig &config, float &limits[], float &causal_price_norm[], CNeuronDPCCOCL *&actor, string &reason) { actor = NULL; reason = ""; if(!m_initialized || m_restore_required || m_progress.failed || !m_progress.complete) {reason = "actor.owner_run_not_selected"; return(false);} string statistics_source = ""; if(!NormalizationStatisticsSource(statistics_source, reason)) return(false); if(statistics_source != "INPUT_WINDOW") return(DPCCStageFitBlocked("FIT_FIXED_requires_whole_fit_source_not_record", reason)); SDPCCConfig training; SDPCCDecoderComposition decoder; SDPCCDenoiserComposition denoiser; string model_source = ""; double beta[], alpha[], sigma[], dl = 0, nl = 0; SDPCCTrainingConfigAdmission receipt; SDPCCTrainingRoleConfig role; if(!m_examples.PreparedActorProfile(normalization_record, training, decoder, denoiser, causal_price_norm, reason) || !m_admitted.ModelSource(model_source) || !m_admitted.Get(DPCC_ROLE_DECODER, receipt, role) || !DPCCDecodePreparedActorConfig(model_source, training, config, limits, reason) || !DPCCBuildCosineDiffusionSchedule(config.shape.J, receipt.model_binding.schedule_s, receipt.model_binding.beta_cap, beta, alpha, sigma, reason) || !m_examples.VerifyRunnerArtifact(m_decoder_best, dl, nl, reason) || !m_examples.VerifyRunnerArtifact(m_denoiser_best, dl, nl, reason)) return(false); return(DPCCAssembleSelectedCheckpoints(net, training, config, decoder, denoiser, beta, alpha, sigma, m_decoder_best, m_denoiser_best, m_initial.binding_sha256, causal_price_norm, m_actor_maximum_bytes, actor, reason)); } //--- Stage authority derives only from the executed private role progress. bool AdvanceDecoder(string &reason) { reason = ""; if(!m_initialized || m_restore_required || m_progress.failed || m_progress.complete || m_progress.role != 0) {reason = "epoch.decoder_not_running"; return(false);} return(AdvanceRole(m_decoder_plan, m_decoder_role, false, reason)); } bool AdvanceDenoiser(string &reason) { reason = ""; if(!m_initialized || m_restore_required || m_progress.failed || m_progress.complete || m_progress.role != 1 || !m_decoder_progress.complete || !m_decoder_progress.has_best) {reason = "epoch.denoiser_not_running"; return(false);} return(AdvanceRole(m_denoiser_plan, m_denoiser_role, true, reason)); } bool DecoderSelection(SDPCCCheckpointArtifact &artifact, double &normalized_loss, string &reason) { ZeroMemory(artifact); normalized_loss = 0; reason = ""; if(!DecoderSelectionBinding(reason)) return(false); double denoiser_loss = 0; if(!m_examples.VerifyRunnerArtifact(m_decoder_best, normalized_loss, denoiser_loss, reason)) return(false); if(normalized_loss != m_decoder_progress.best_loss) {reason = "epoch.decoder_selected_loss"; return(false);} artifact = m_decoder_best; return(true); } int DecoderValidationCount(void) {return(m_initialized && !m_restore_required && !m_progress.failed && m_decoder_progress.complete ? m_decoder_plan.ValidationCount() : 0);} bool BeginDecoderInference(string &reason) { if(!DecoderSelectionBinding(reason)) return(false); return(m_examples.BeginSelectedDecoderInference(m_decoder_best, reason)); } bool DecoderInferenceObservation(const int record_index, SDPCCDecoderInferenceObservation &out, string &reason) { out.record_index = -1; out.window_id = ""; out.selected_manifest_sha = ""; out.decision_time = 0; ArrayResize(out.normalized, 0); return(m_examples.InferSelectedDecoderRow(record_index, out, reason)); } bool EndDecoderInference(string &reason) { return(m_examples.EndSelectedDecoderInference(reason)); } bool DecoderValidationObservation(const int ordinal, SDPCCDecoderValidationObservation &out, string &reason) { out.record_index = -1; out.window_id = ""; out.selected_manifest_sha = ""; out.normalized_loss = 0; ArrayResize(out.normalized, 0); ArrayResize(out.native_ohlc, 0); ArrayResize(out.causal_price_norm, 0); SDPCCTrainingPlanEntry entry; if(!DecoderSelectionBinding(reason) || !m_decoder_plan.Validation(ordinal, entry)) {if(StringLen(reason) == 0)reason = "epoch.decoder_validation_ordinal"; return(false);} return(m_examples.ObserveSelectedDecoder(m_decoder_best, entry.record_index, out, reason)); } bool DecoderValidationStream(const string path, SDPCCCheckpointArtifact &selected, double &mean, uint &records, ulong &bytes, string &sha, string &reason) { ZeroMemory(selected); mean = 0; records = 0; bytes = 0; sha = ""; if(!DecoderSelectionBinding(reason)) return(false); const int count = m_decoder_plan.ValidationCount(); int indices[]; if(count <= 0 || ArrayResize(indices, count) != count) { reason = "epoch.decoder_stream.plan_capacity"; return(false); } for(int i = 0; i < count; i++) { SDPCCTrainingPlanEntry entry; if(!m_decoder_plan.Validation(i, entry)) { reason = "epoch.decoder_stream.private_ordinal"; return(false); } indices[i] = entry.record_index; } if(!m_examples.StreamSelectedDecoder(m_decoder_best, indices, path, m_actor_maximum_bytes, mean, records, bytes, sha, reason)) return(false); if(mean != m_decoder_progress.best_loss) { reason = "epoch.decoder_stream.selected_mean"; return(false); } selected = m_decoder_best; return(true); } uint DecoderValidationNativeLoads(void) {return(m_examples.DecoderValidationNativeLoads());} bool RoleProgress(const ENUM_DPCC_TRAIN_ROLE role, SDPCCEpochProgress &progress) { ZeroMemory(progress); if(!m_initialized || m_restore_required || m_progress.failed) return(false); if(role == DPCC_ROLE_DECODER) progress = m_decoder_progress; else if(role == DPCC_ROLE_DENOISER) progress = m_denoiser_progress; else return(false); return(true); } //--- Read-only access to the rows admitted by this epoch owner. bool AdmittedRecordCount(int &count, string &reason) const { count = 0; reason = ""; if(!m_initialized) {reason = "epoch.not_initialized"; return(false);} if(m_restore_required) {reason = "epoch.restore_required"; return(false);} if(m_progress.failed) {reason = "epoch.failed"; return(false);} return(m_examples.AdmittedRecordCount(count, reason)); } bool CopyAdmittedRecord(const int index, SDPCCMaterializedRecord &record, string &reason) const { DPCCClearMaterializedRecord(record); reason = ""; if(!m_initialized) {reason = "epoch.not_initialized"; return(false);} if(m_restore_required) {reason = "epoch.restore_required"; return(false);} if(m_progress.failed) {reason = "epoch.failed"; return(false);} return(m_examples.CopyAdmittedRecord(index, record, reason)); } bool Advance(string &reason) { reason = ""; if(!m_initialized || m_restore_required || m_progress.failed || m_progress.complete) { reason = "epoch.not_running"; return(false); } if(m_progress.role == 0) return(AdvanceRole(m_decoder_plan, m_decoder_role, false, reason)); return(AdvanceRole(m_denoiser_plan, m_denoiser_role, true, reason)); } bool Progress(SDPCCEpochProgress &progress) { progress = m_progress; ulong ds = 0, ns = 0; return(m_initialized && m_examples.NativeState(progress.decoder_updates, progress.denoiser_updates, ds, ns)); } bool Selected(const ENUM_DPCC_TRAIN_ROLE role, SDPCCCheckpointArtifact &artifact) { ZeroMemory(artifact); if(!m_initialized || m_restore_required || m_progress.failed || !m_progress.complete) return(false); if(role == DPCC_ROLE_DECODER) artifact = m_decoder_best; else if(role == DPCC_ROLE_DENOISER) artifact = m_denoiser_best; else return(false); return(StringLen(artifact.manifest_sha256) > 0); } }; #endif //--- This is a typed call input, not a readiness receipt or an alternative manifest schema. //--- Loader fills actual resolved data, explicit immutable keys/tables and pinned schema bytes. struct SDPCCStudyStageInput { SDPCCTrainingBankSource source; SDPCCConfig config; SDPCCDecoderComposition decoder; SDPCCDenoiserComposition denoiser; double alpha[]; SDPCCTrainingFrozenRecordKeys keys[]; ulong training_seed, validation_seed; uint validation_repeat; string package_root, containing_json; SDPCCResourceReference training_reference, model_reference; string training_schema, model_schema, bank_schema, dataset_schema, time_index_schema; SDPCCTrainingPlanEntry decoder_fit[], decoder_validation[], denoiser_fit[], denoiser_validation[]; string decoder_plan_sha, denoiser_plan_sha, checkpoint_stem; ulong maximum_checkpoint_bytes; string boundary_path, boundary_sha, selected_decoder_path, selected_decoder_sha; string result_path, actor_path; int normalization_record; SDPCCStageFitSource fit_source; SDPCCOfflinePolicyProfile offline_policy; SDPCCOfflineFitKey offline_fit[]; SDPCCOfflinePolicyAdmission offline_admission; SDPCCClockIdentity offline_clock_identity; bool offline_clock_historical_policy; bool offline_clock_server_calendar; SDPCCResourceReference offline_clock_map, offline_clock_index, offline_reviewed_origin; bool offline_resume; SDPCCResourceReference offline_resume_manifest; string offline_checkpoint_stem; //--- Optional copy from the same authoritative loader admission; no second bank parser. SDPCCMaterializedRecord online_records[]; SDPCCOfflineFitWitness online_witnesses[]; }; //--- Direct-history rows contain financial context only; no behavior actions or rewards. struct SDPCCOracleHistoryRow { long decision_bar; float context[], native_ohlc[]; long timeline_opens[], timeline_closes[]; string time_domain; }; struct SDPCCOracleHistoryInput { SDPCCConfig config; ENUM_TIMEFRAMES timeframe; double actor_beta[], actor_alpha_bar[], actor_sigma[]; SDPCCStudyStageInput clock; //--- Raw source plus decision indices; one financial row is built per step. SDPCCStudyForecastBar raw_bars[]; int fit_indices[]; long decision_bars[]; SDPCCHistoricalFinancialProfile financial_profile; float deltas[]; string model_sha, model_text, rows_sha, raw_sha, raw_path, raw_indicator_params, output_stem, execution_sha, source_server; long raw_successor_open_server; string source_labels[], source_paths[], source_hashes[]; string config_sha, deltas_sha; }; //+------------------------------------------------------------------+ //| Canonical history config includes every native fixed-profile | //| field | //+------------------------------------------------------------------+ bool DPCCOracleHistoryConfigSHA(const SDPCCConfig &config, string &sha, string &reason) { string wire = "DPCC_ORACLE_HISTORY_CONFIG_V1\n"; const long integers[] = {config.shape.K, config.shape.V, config.shape.H, config.shape.D, config.shape.M, config.shape.J, config.shape.E, config.shape.PlanBatch, config.shape.NActions, config.shape.StateSize, config.shape.Embedding, config.shape.Heads, config.shape.KeyDimension, config.shape.Depth, config.contract_version, config.binary_layer_version, (long)config.mode, config.projection_max_iterations, config.projection_max_evaluations, config.projection_lookback_steps, config.price_basis, config.quote_adapter, config.event_model, config.risk_type, config.normalization_mode, config.loss_coordinates, config.randomness_version, config.margin_fx_profile, config.account_currency_digits }; for(int i = 0; i < ArraySize(integers); i++) DPCCOfflineField(wire, (string)integers[i]); const double reals[] = {config.projection_improvement_atol, config.projection_distance_tie_atol, config.volume_scale, config.price_scale, config.money_scale, config.max_quote_age_seconds, config.max_last_age_seconds }; for(int i = 0; i < ArraySize(reals); i++) { if(!MathIsValidNumber(reals[i])) { reason = "study.history.config_nonfinite"; return(false); } DPCCOfflineField(wire, StringFormat("%.17g", reals[i])); } const float scalars[] = {config.risk_parameter, config.mean_weight, config.gamma, config.probability_tolerance}; if(!DPCCOfflineFloats(wire, scalars, reason) || !DPCCOfflineFloats(wire, config.action_bias, reason) || !DPCCOfflineFloats(wire, config.action_scale, reason)) return(false); return(DPCCCheckpointTextSHA(wire, sha, reason)); } //+------------------------------------------------------------------+ //| Function DPCCOracleHistoryDeltasSHA | //+------------------------------------------------------------------+ bool DPCCOracleHistoryDeltasSHA(const float &deltas[], string &sha, string &reason) { string wire = "DPCC_ORACLE_HISTORY_ORACLE_DELTAS_V1\n"; return(ArraySize(deltas) == 6 && DPCCOfflineFloats(wire, deltas, reason) && DPCCCheckpointTextSHA(wire, sha, reason)); } #endif //--- Internal call envelope only. Normative artifact validation stays in DPCC. //--- A trusted expected envelope SHA pins external review claims, keys and tables. bool DPCCStageBlocked(string &reason, const string prerequisite) { reason = "BLOCKED:study_input." + prerequisite; return(false); } //+------------------------------------------------------------------+ //| Function DPCCStageGate | //+------------------------------------------------------------------+ bool DPCCStageGate(string &reason, const string name, const ENUM_DPCC_GATE_VERDICT verdict) { reason = (verdict == DPCC_GATE_FAIL ? "FAIL:" : "BLOCKED:") + "study_input." + name + "." + reason; return(false); } //+------------------------------------------------------------------+ //| Function DPCCStageResolve | //+------------------------------------------------------------------+ bool DPCCStageResolve(CDPCCJsonDocument &d, const int object, const string name, const string root, const string containing, const bool common, SDPCCArtifactResolveResult &artifact, string &text, string &reason) { SDPCCResourceReference reference; text = ""; if(!DPCCBankReadReference(d, object, name, reference, reason, "study_input." + name) || !DPCCResolveArtifact(root, containing, reference, common, DPCC_RESOLVER_MAX_JSON_BYTES, artifact, reason) || !DPCCCentralBytesSource(artifact.bytes, text, reason, name)) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageSchema | //+------------------------------------------------------------------+ bool DPCCStageSchema(CDPCCJsonDocument &d, const int schemas, const string name, const string pin, const string root, const string containing, const bool common, string &text, string &reason) { SDPCCArtifactResolveResult artifact; if(!DPCCStageResolve(d, schemas, name, root, containing, common, artifact, text, reason)) return(false); if(artifact.observed_sha256 != pin) return(DPCCStageBlocked(reason, "schema_pin." + name)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadHex | //+------------------------------------------------------------------+ bool DPCCStageReadHex(CDPCCJsonDocument &d, const int object, const string name, ulong &value, string &reason) { string text = ""; value = 0; if(!DPCCResourceReadString(d, object, name, text, reason, "study_input." + name) || StringLen(text) != 16) return(DPCCStageBlocked(reason, "hex16." + name)); for(int i = 0; i < 16; i++) { const ushort ch = StringGetCharacter(text, i); if(!((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'))) return(DPCCStageBlocked(reason, "hex16." + name)); value = (value << 4) | (ulong)(ch <= '9' ? ch - '0' : ch - 'a' + 10); } return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadUnsigned | //+------------------------------------------------------------------+ bool DPCCStageReadUnsigned(CDPCCJsonDocument &d, const int object, const string name, const ulong maximum, ulong &value, string &reason) { const int node = d.FindMember(object, name); if(d.Type(node) != DPCC_JSON_NUMBER || !DPCCCheckpointReadUnsigned(d.Text(node), maximum, value)) return(DPCCStageBlocked(reason, "unsigned." + name)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageOutputPath | //+------------------------------------------------------------------+ bool DPCCStageOutputPath(CDPCCJsonDocument &d, const int object, const string name, const string root, const string containing, string &path, string &reason) { string supplied = ""; if(!DPCCResourceReadString(d, object, name, supplied, reason, "study_input." + name) || !DPCCNormalizePackagePath(root, containing, supplied, path, reason)) return(false); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageSamePath | //+------------------------------------------------------------------+ bool DPCCStageSamePath(const string path_a, const string path_b) { return(StringCompare(path_a, path_b, false) == 0); } //+------------------------------------------------------------------+ //| Function DPCCStageNormalizeReviewedOrigin | //+------------------------------------------------------------------+ bool DPCCStageNormalizeReviewedOrigin(const string package_root, const string containing_json, SDPCCResourceReference &origin, string &reason) { string canonical = "", relative = ""; if(!DPCCNormalizePackagePath(package_root, containing_json, origin.path, canonical, reason) || !DPCCRunContainingArtifact(package_root, canonical, relative, reason)) return(false); origin.path = relative; return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadOfflineClock | //+------------------------------------------------------------------+ bool DPCCStageReadOfflineClock(CDPCCJsonDocument &d, const int object, const string package_root, const string containing_json, SDPCCStudyStageInput &out, string &reason) { ZeroMemory(out.offline_clock_identity); ZeroMemory(out.offline_reviewed_origin); out.offline_clock_historical_policy = false; const int node = d.FindMember(object, "offline_clock"); if(d.Type(node) != DPCC_JSON_OBJECT) return(DPCCStageBlocked(reason, "offline_clock_identity_and_origin")); const int scope_node = d.FindMember(node, "mapping_scope"); string mapping_scope = "ACCOUNT_OBSERVED_V1"; if(scope_node >= 0 && !DPCCResourceReadString(d, node, "mapping_scope", mapping_scope, reason, "offline_clock.mapping_scope")) return(DPCCStageBlocked(reason, "offline_clock_mapping_scope")); if(mapping_scope != "ACCOUNT_OBSERVED_V1" && mapping_scope != "HISTORICAL_POLICY_V1") return(DPCCStageBlocked(reason, "offline_clock_mapping_scope")); const bool historical_policy = (mapping_scope == "HISTORICAL_POLICY_V1"); const int login_node = d.FindMember(node, "account_login"); const int expected_children = (historical_policy ? 4 : (scope_node >= 0 ? 5 : 4)); if(d.ChildCount(node) != expected_children || !DPCCResourceReadString(d, node, "broker_server", out.offline_clock_identity.broker_server, reason, "offline_clock") || !DPCCResourceReadString(d, node, "broker_domain", out.offline_clock_identity.broker_domain, reason, "offline_clock") || !DPCCBankReadReference(d, node, "reviewed_origin", out.offline_reviewed_origin, reason, "offline_clock.reviewed_origin")) return(DPCCStageBlocked(reason, "offline_clock_identity_and_origin")); if(historical_policy) { if(login_node >= 0) return(DPCCStageBlocked(reason, "offline_clock_policy_must_not_bind_account")); } else if(login_node < 0 || !DPCCResourceReadLong(d, node, "account_login", out.offline_clock_identity.account_login, reason, "offline_clock.account_login")) return(DPCCStageBlocked(reason, "offline_clock_identity_and_origin")); if(StringLen(out.offline_clock_identity.broker_server) == 0 || out.offline_clock_identity.broker_domain != "BROKER_SERVER_SECONDS_V1" || (historical_policy ? out.offline_clock_identity.account_login != 0 : out.offline_clock_identity.account_login <= 0) || StringLen(out.offline_reviewed_origin.path) == 0 || StringLen(out.offline_reviewed_origin.id) == 0 || !DPCCResolverLowerSha256(out.offline_reviewed_origin.sha256)) return(DPCCStageBlocked(reason, "offline_clock_identity_and_origin")); out.offline_clock_historical_policy = historical_policy; if(!DPCCStageNormalizeReviewedOrigin(package_root, containing_json, out.offline_reviewed_origin, reason)) return(DPCCStageBlocked(reason, "offline_reviewed_origin_path")); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadOfflineOutputs | //+------------------------------------------------------------------+ bool DPCCStageReadOfflineOutputs(CDPCCJsonDocument &d, const int outputs, const string root, const string containing, const bool common, SDPCCStudyStageInput &out, string &reason) { out.offline_resume = false; ZeroMemory(out.offline_resume_manifest); out.offline_checkpoint_stem = ""; const int node = d.FindMember(outputs, "offline"); if(d.Type(node) != DPCC_JSON_OBJECT || d.ChildCount(node) != 2 || !DPCCStageOutputPath(d, node, "checkpoint_stem", root, containing, out.offline_checkpoint_stem, reason)) return(DPCCStageBlocked(reason, "offline_checkpoint_output")); if(DPCCStageSamePath(out.offline_checkpoint_stem, out.checkpoint_stem) || DPCCStageSamePath( out.offline_checkpoint_stem, out.result_path)) return(DPCCStageBlocked(reason, "offline_checkpoint_distinct_from_epoch_outputs")); const int resume = d.FindMember(node, "resume_manifest"); if(resume < 0) return(DPCCStageBlocked(reason, "offline_resume_manifest_explicit_null_or_reference")); if(d.Type(resume) == DPCC_JSON_NULL) return(true); SDPCCArtifactResolveResult manifest; string ignored = ""; if(d.Type(resume) != DPCC_JSON_OBJECT || d.ChildCount(resume) != 3 || !DPCCStageResolve(d, node, "resume_manifest", root, containing, common, manifest, ignored, reason)) return(DPCCStageBlocked(reason, "offline_trusted_resume_manifest")); if(DPCCStageSamePath(manifest.canonical_path, out.offline_checkpoint_stem) || DPCCStageSamePath( manifest.canonical_path, out.result_path)) return(DPCCStageBlocked(reason, "offline_resume_output_collision")); out.offline_resume = true; if(!DPCCBankReadReference(d, node, "resume_manifest", out.offline_resume_manifest, reason, "outputs.offline.resume_manifest")) return(DPCCStageBlocked(reason, "offline_trusted_resume_manifest")); out.offline_resume_manifest.path = manifest.canonical_path; out.offline_resume_manifest.sha256 = manifest.observed_sha256; return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageAdmitOfflineClock | //+------------------------------------------------------------------+ bool DPCCStageAdmitOfflineClock(const string root, const bool common, SDPCCStudyStageInput &out, string &reason) { string dataset_container = "", map_relative = "", index_relative = ""; if(!DPCCRunContainingArtifact(root, out.source.links.dataset_path, dataset_container, reason)) return(DPCCStageBlocked(reason, "offline_dataset_container")); SDPCCArtifactResolveResult map_artifact, index_artifact; if(!DPCCResolveArtifact(root, dataset_container, out.source.dataset.timezone_mapping, common, DPCC_RESOLVER_MAX_JSON_BYTES, map_artifact, reason) || !DPCCResolveArtifact(root, dataset_container, out.source.dataset.time_index, common, DPCC_RESOLVER_MAX_JSON_BYTES, index_artifact, reason)) return(DPCCStageBlocked(reason, "offline_dataset_clock_artifacts")); if(map_artifact.canonical_path == "" || index_artifact.canonical_path != out.source.links.time_index_path || index_artifact.observed_sha256 != out.source.links.time_index_sha256 || out.source.time_index.time_index_id != out.source.links.time_index_id || out.source.time_index.dataset_id != out.source.links.dataset_id || out.source.time_index.timezone_mapping.id != out.source.dataset.timezone_mapping.id || out.source.time_index.timezone_mapping.sha256 != out.source.dataset.timezone_mapping.sha256 || !DPCCRunContainingArtifact(root, map_artifact.canonical_path, map_relative, reason) || !DPCCRunContainingArtifact(root, index_artifact.canonical_path, index_relative, reason)) return(DPCCStageBlocked(reason, "offline_clock_dataset_index_binding")); out.offline_clock_map = out.source.dataset.timezone_mapping; out.offline_clock_map.path = map_relative; out.offline_clock_index = out.source.dataset.time_index; out.offline_clock_index.path = index_relative; CDPCCMarketClock clock; bool clock_ready = false; if(out.offline_clock_historical_policy) clock_ready = clock.InitializeHistoricalPolicy(root, "root.json", out.offline_clock_map, out.offline_clock_index, out.offline_reviewed_origin, out.offline_clock_identity, common, reason); else clock_ready = clock.Initialize(root, "root.json", out.offline_clock_map, out.offline_clock_index, out.offline_reviewed_origin, out.offline_clock_identity, common, false, reason); if(!clock_ready) return(DPCCStageBlocked(reason, "offline_reviewed_clock_identity_origin_and_index")); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageExternalReview | //+------------------------------------------------------------------+ bool DPCCStageExternalReview(CDPCCJsonDocument &envelope, CDPCCJsonDocument &report, const string root, const string containing, const bool common, string &reason) { string policy = ""; if(!DPCCResourceReadString(envelope, envelope.Root(), "external_review_policy", policy, reason, "study_input.external_review_policy") || policy != "TRUST_PINNED_REVIEWED_BASELINE_EXECUTION_V1") return(DPCCStageBlocked(reason, "externally_reviewed_baseline_execution_policy")); const string names[] = {"baseline", "execution"}; for(int gate = 0; gate < 2; gate++) { const int node = report.FindMember(report.Root(), names[gate]); string verdict = "", explanation = ""; if(!DPCCResourceReadString(report, node, "verdict", verdict, reason, names[gate]) || verdict != "PASS" || !DPCCResourceReadString(report, node, "reason", explanation, reason, names[gate]) || !DPCCDataNonemptyText( explanation)) return(DPCCStageBlocked(reason, "external_claim." + names[gate])); SDPCCResourceReference evidence[]; if(!DPCCDecodeDataReferenceArray(report, report.FindMember(node, "evidence"), evidence, reason, names[gate]) || ArraySize(evidence) == 0) return(DPCCStageBlocked(reason, "external_evidence." + names[gate])); for(int i = 0; i < ArraySize(evidence); i++) { SDPCCArtifactResolveResult actual; if(!DPCCResolveArtifact(root, containing, evidence[i], common, DPCC_RESOLVER_MAX_JSON_BYTES, actual, reason)) return(false); } } //--- These are pinned externally reviewed claims, not machine-derived PASS. return(true); } //+------------------------------------------------------------------+ //| Function DPCCStagePretrainResources | //+------------------------------------------------------------------+ bool DPCCStagePretrainResources(CDPCCJsonDocument &report, const string schema, const SDPCCResourceProfile &profile, const string profile_sha, const string root, const string containing, const bool common, string &reason) { SDPCCResourceReport decoded; if(!DPCCDecodeResourceReport(report, report.Root(), decoded, reason) || !DPCCBindResourceReportProfile(profile, decoded, profile_sha, reason)) return(false); SDPCCResourceMeasurement rows[]; ENUM_DPCC_GATE_VERDICT verdicts[]; int count = 0; for(int i = 0; i < ArraySize(decoded.rows); i++) { if(decoded.rows[i].measurement.stage != "PRETRAIN_ESTIMATE") continue; if(ArraySize(decoded.rows[i].evidence) == 0) return(DPCCStageBlocked(reason, "pretrain.raw_evidence")); SDPCCResourceMeasurement derived; ZeroMemory(derived); ENUM_DPCC_GATE_VERDICT verdict = DPCC_GATE_BLOCKED; for(int e = 0; e < ArraySize(decoded.rows[i].evidence); e++) { SDPCCArtifactResolveResult actual; string text = ""; CDPCCJsonDocument raw; SDPCCResourceEvidence evidence; if(!DPCCResolveArtifact(root, containing, decoded.rows[i].evidence[e], common, DPCC_RESOLVER_MAX_JSON_BYTES, actual, reason) || !DPCCCentralBytesSource(actual.bytes, text, reason, "resource_evidence") || !DPCCCentralSchemaValidate(raw, text, schema, reason, "resource_evidence") || !DPCCDecodeResourceEvidence(raw, raw.Root(), evidence, reason) || !DPCCBindResourceReportRow(profile, decoded.rows[i], evidence, actual.observed_sha256, profile_sha, reason)) return(false); string evidence_relative = ""; SDPCCArtifactResolveResult linked_profile; string ignored = ""; if(!DPCCRunContainingArtifact(root, actual.canonical_path, evidence_relative, reason) || !DPCCStageResolve(raw, raw.Root(), "readiness_profile", root, evidence_relative, common, linked_profile, ignored, reason) || linked_profile.observed_sha256 != profile_sha) return(DPCCStageBlocked(reason, "pretrain.actual_profile")); bool example = false; if(!DPCCResourceReadBool(raw, raw.Root(), "example_only", example, reason, "resource_evidence.example_only") || example) return(DPCCStageBlocked(reason, "pretrain.example_only")); verdict = DPCCRecomputeResourceEvidence(profile, evidence, derived, reason); if(verdict != DPCC_GATE_PASS) return(DPCCStageGate(reason, "pretrain.raw", verdict)); if(!DPCCResourceMeasurementEqual(derived, decoded.rows[i].measurement)) return(DPCCStageBlocked(reason, "pretrain.derived_measurement")); } if(ArrayResize(rows, count + 1) != count + 1 || ArrayResize(verdicts, count + 1) != count + 1) return(DPCCStageBlocked(reason, "pretrain.capacity")); rows[count] = derived; verdicts[count] = verdict; count++; } const ENUM_DPCC_GATE_VERDICT derived_gate = DPCCEvaluateResourceGate(profile, "PRETRAIN_ESTIMATE", decoded.claims.evaluated_at_utc, rows, verdicts, reason); if(derived_gate != DPCC_GATE_PASS || decoded.claims.pretrain_resources != derived_gate) return(DPCCStageGate(reason, "pretrain.matrix", derived_gate)); return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadPlan | //+------------------------------------------------------------------+ bool DPCCStageReadPlan(CDPCCJsonDocument &d, const int plans, const string name, SDPCCTrainingPlanEntry &rows[], string &reason) { const int node = d.FindMember(plans, name); if(d.Type(node) != DPCC_JSON_ARRAY || d.ChildCount(node) <= 0 || ArrayResize(rows, d.ChildCount(node)) != d.ChildCount(node)) return(DPCCStageBlocked(reason, "plan_array." + name)); for(int i = 0; i < ArraySize(rows); i++) { const int item = d.ChildAt(node, i); ulong epoch = 0, ordinal = 0, index = 0, j = 0, repeat = 0; if(!DPCCStageReadUnsigned(d, item, "epoch", UINT_MAX, epoch, reason) || !DPCCStageReadUnsigned(d, item, "ordinal", UINT_MAX, ordinal, reason) || !DPCCStageReadUnsigned(d, item, "record_index", INT_MAX, index, reason) || !DPCCStageReadUnsigned(d, item, "j", UINT_MAX, j, reason) || !DPCCStageReadUnsigned(d, item, "repeat", UINT_MAX, repeat, reason) || !DPCCResourceReadString(d, item, "window_id", rows[i].window_id, reason, "study_input.plan.window_id")) return(false); rows[i].epoch = (uint)epoch; rows[i].ordinal = (uint)ordinal; rows[i].record_index = (int)index; rows[i].j = (uint)j; rows[i].repeat = (uint)repeat; } return(true); } //+------------------------------------------------------------------+ //| Function DPCCStageReadKeys | //+------------------------------------------------------------------+ bool DPCCStageReadKeys(CDPCCJsonDocument &d, const SDPCCMaterializedRecord &records[], const uint J, SDPCCTrainingFrozenRecordKeys &keys[], string &reason) { const int node = d.FindMember(d.Root(), "keys"); const int count = ArraySize(records); if(d.Type(node) != DPCC_JSON_ARRAY || d.ChildCount(node) != count || count <= 0 || ArrayResize(keys, count) != count) return(DPCCStageBlocked(reason, "exact_materialized_record_keys")); for(int i = 0; i < count; i++) { const int item = d.ChildAt(node, i); ulong j = 0; if(!DPCCResourceReadString(d, item, "window_id", keys[i].window_id, reason, "study_input.key.window_id") || !DPCCResourceReadLong(d, item, "decision_time", keys[i].decision_time, reason, "study_input.key.decision_time") || !DPCCStageReadHex(d, item, "decision_identity", keys[i].decision_identity, reason) || !DPCCStageReadHex(d, item, "context_identity", keys[i].context_identity, reason) || !DPCCStageReadHex(d, item, "owner_identity", keys[i].owner_identity, reason) || !DPCCStageReadUnsigned(d, item, "validation_j", J, j, reason)) return(false); keys[i].validation_j = (uint)j; if(keys[i].window_id != records[i].window_id || keys[i].decision_time != records[i].admission.decision_time || keys[i].decision_identity == 0 || keys[i].context_identity == 0 || keys[i].owner_identity == 0 || (records[i].admission.split == "validation" ? j == 0 : j != 0)) return(DPCCStageBlocked(reason, "record_key_binding")); for(int before = 0; before < i; before++) if(keys[before].decision_identity == keys[i].decision_identity) return(DPCCStageBlocked(reason, "duplicate_decision_identity")); } return(true); } //+------------------------------------------------------------------+ //| Project the complete materialized record into a pinned witness | //+------------------------------------------------------------------+ bool DPCCStageOfflineWitness(const SDPCCMaterializedRecord &record, const SDPCCTrainingFrozenRecordKeys &key, SDPCCOfflineFitWitness &out, string &reason) { ZeroMemory(out); string wire = "DPCC_OFFLINE_MATERIALIZED_RECORD_V1\n"; const string text[] = {record.admission.split, record.window_id, record.dataset_id, record.forecast_id, record.normalization_id, record.source_id, record.noise_seed, record.execution_label }; for(int i = 0; i < ArraySize(text); i++) DPCCOfflineField(wire, text[i]); DPCCOfflineField(wire, (string)record.admission.admitted); DPCCOfflineField(wire, (string)record.admission.decision_bar_index); DPCCOfflineField(wire, (string)record.admission.decision_time); DPCCOfflineField(wire, (string)record.admission.horizon); DPCCOfflineField(wire, (string)record.episode_id); DPCCOfflineField(wire, (string)record.forecast_asof); DPCCOfflineField(wire, (string)record.normalization_asof); DPCCOfflineField(wire, StringFormat("%016I64x", record.noise_seed_u64)); if(!DPCCOfflineFloats(wire, record.account_features, reason) || !DPCCOfflineFloats(wire, record.context_without_noise, reason) || !DPCCOfflineFloats(wire, record.target_native_ohlc, reason) || !DPCCOfflineFloats(wire, record.rho, reason) || !DPCCOfflineFloats(wire, record.actions, reason) || !DPCCOfflineFloats(wire, record.executed_actions, reason) || !DPCCOfflineFloats(wire, record.rewards, reason)) return(false); DPCCOfflineField(wire, (string)ArraySize(record.terminal_flags)); for(int i = 0; i < ArraySize(record.terminal_flags); i++) DPCCOfflineField(wire, (string)record.terminal_flags[i]); DPCCOfflineLongs(wire, record.bar_open_times); DPCCOfflineLongs(wire, record.bar_close_times); DPCCOfflineLongs(wire, record.bar_indices); DPCCOfflineLongs(wire, record.source_action_times); if(!DPCCCheckpointTextSHA(wire, out.record_sha256, reason)) return(false); out.split = record.admission.split; out.window_id = record.window_id; out.episode_id = record.episode_id; out.decision_bar = record.admission.decision_bar_index; out.decision_time = record.admission.decision_time; out.horizon = record.admission.horizon; out.decision_identity = key.decision_identity; out.context_identity = key.context_identity; out.owner_identity = key.owner_identity; if(ArrayCopy(out.bar_indices, record.bar_indices) != ArraySize(record.bar_indices) || ArrayCopy(out.bar_open_times, record.bar_open_times) != ArraySize(record.bar_open_times) || ArrayCopy(out.bar_close_times, record.bar_close_times) != ArraySize(record.bar_close_times)) return(DPCCStageBlocked(reason, "offline_witness_capacity")); out.admitted = record.admission.admitted; return(true); } //+------------------------------------------------------------------+ //| Parse required Oracle inputs into the shared typed admission | //+------------------------------------------------------------------+ bool DPCCStageReadOfflinePolicy(CDPCCJsonDocument &d, const SDPCCMaterializedRecord &records[], SDPCCStudyStageInput &out, const float &model_limits[], const string actual_bank_sha, string &reason) { ZeroMemory(out.offline_policy); DPCCClearOfflinePolicyAdmission(out.offline_admission); ArrayResize(out.offline_fit, 0); const int root = d.Root(), node = d.FindMember(root, "offline_policy"), plan = d.FindMember(root, "offline_fit"); SDPCCOfflinePolicyProfile p; ZeroMemory(p); const int optimizer = d.FindMember(node, "optimizer"), seed = d.FindMember(node, "seed"); ulong version = 0, horizon = 0; if(d.Type(node) != DPCC_JSON_OBJECT || d.ChildCount(node) != 19 || d.Type(seed) != DPCC_JSON_OBJECT || d.ChildCount(seed) != 9 || d.Type(optimizer) != DPCC_JSON_OBJECT || d.ChildCount(optimizer) != 13 || !DPCCRunOptimizer(d, node, reason) || !DPCCStageReadUnsigned(d, node, "version", 1, version, reason) || version != 1 || !DPCCStageReadUnsigned(d, node, "horizon", INT_MAX, horizon, reason) || !DPCCStageReadUnsigned(d, node, "maximum_candidates", INT_MAX, p.maximum_candidates, reason) || !DPCCStageReadUnsigned(d, node, "maximum_work", ULONG_MAX, p.maximum_work, reason) || !DPCCResourceReadLong(d, node, "episode_id", p.episode_id, reason, "offline_policy") || !DPCCResourceReadLong(d, node, "first_decision_bar", p.first_decision_bar, reason, "offline_policy") || !DPCCResourceReadLong(d, node, "last_decision_bar", p.last_decision_bar, reason, "offline_policy")) return(DPCCStageBlocked(reason, "required_offline_policy." + reason)); p.version = (uint)version; p.horizon = (uint)horizon; if(!DPCCResourceReadString(d, node, "model_sha256", p.model_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "training_sha256", p.training_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "bank_sha256", p.bank_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "time_index_sha256", p.time_index_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "decoder_sha256", p.decoder_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "seed_sha256", p.seed_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "profile_sha256", p.profile_sha256, reason, "offline_policy") || !DPCCResourceReadString(d, node, "fit_plan_sha256", p.fit_plan_sha256, reason, "offline_policy")) return(false); double gamma = 0; if(!DPCCResourceReadDouble(d, node, "gamma", gamma, reason, "offline_policy") || !MathIsValidNumber(gamma) || gamma <= 0 || gamma > 1 || (float)gamma != out.config.gamma || !DPCCRecordFloatArray(d, d.FindMember(node, "coordinate_deltas"), 6, p.coordinate_deltas, "offline_policy.coordinate_deltas", reason)) return(DPCCStageBlocked(reason, "explicit_delta_model_discount." + reason)); p.gamma = (float)gamma; if(!DPCCResourceReadString(d, optimizer, "name", p.optimizer_name, reason, "offline_policy.optimizer") || !DPCCResourceReadString(d, optimizer, "profile", p.optimizer_profile, reason, "offline_policy.optimizer") || !DPCCResourceReadString(d, optimizer, "bias_correction", p.bias_correction, reason, "offline_policy.optimizer") || !DPCCResourceReadString(d, optimizer, "gradient_convention", p.gradient_convention, reason, "offline_policy.optimizer") || !DPCCResourceReadString(d, optimizer, "clip_location", p.clip_location, reason, "offline_policy.optimizer") || !DPCCResourceReadString(d, optimizer, "zero_variance_update", p.zero_variance_update, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "learning_rate", p.learning_rate, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "beta1", p.beta1, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "beta2", p.beta2, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "epsilon", p.epsilon, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "weight_decay", p.weight_decay, reason, "offline_policy.optimizer") || !DPCCResourceReadDouble(d, optimizer, "gradient_clip", p.gradient_clip, reason, "offline_policy.optimizer") || !DPCCResourceReadLong(d, optimizer, "initial_step", p.initial_step, reason, "offline_policy.optimizer")) return(false); if(!DPCCResourceReadString(d, seed, "schema", p.seed.schema, reason, "offline_policy.seed") || !DPCCResourceReadString(d, seed, "account_currency", p.seed.account_currency, reason, "offline_policy.seed") || !DPCCResourceReadString(d, seed, "source_sha256", p.seed.source_sha256, reason, "offline_policy.seed") || !DPCCResourceReadLong(d, seed, "snapshot_bar", p.seed.snapshot_bar, reason, "offline_policy.seed") || !DPCCResourceReadLong(d, seed, "snapshot_utc", p.seed.snapshot_utc, reason, "offline_policy.seed") || !DPCCResourceReadLong(d, seed, "snapshot_server", p.seed.snapshot_server, reason, "offline_policy.seed") || !DPCCRecordFloatArray(d, d.FindMember(seed, "state"), 33, p.seed.state, "offline_policy.seed.state", reason) || !DPCCRecordFloatArray(d, d.FindMember(seed, "risk_limits"), 24, p.seed.risk_limits, "offline_policy.seed.risk_limits", reason)) return(DPCCStageBlocked(reason, "required_starting_seed." + reason)); p.seed.position.time_domain = "UTC_SECONDS_V1"; const int position = d.FindMember(seed, "position"); if(d.Type(position) != DPCC_JSON_OBJECT || d.ChildCount(position) != 12 || !DPCCResourceReadBool(d, position, "initialized", p.seed.position.initialized, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "snapshot_bar", p.seed.position.snapshot_bar, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "snapshot_server", p.seed.position.snapshot_server, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "snapshot_utc", p.seed.position.snapshot_time, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "bar_server_open", p.seed.position.bar_server_open, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "bar_server_close", p.seed.position.bar_server_close, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "bar_open_utc", p.seed.position.bar_open_time, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "bar_close_utc", p.seed.position.bar_close_time, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "first_open_bar", p.seed.position.first_open_bar, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "first_open_server", p.seed.position.first_open_server, reason, "offline_policy.position") || !DPCCResourceReadLong(d, position, "first_open_utc", p.seed.position.first_open_time, reason, "offline_policy.position")) return(false); double lots = 0; if(!DPCCResourceReadDouble(d, position, "signed_lots", lots, reason, "offline_policy.position") || !MathIsValidNumber(lots) || !MathIsValidNumber((float)lots) || (lots != 0 && (float)lots == 0)) return(DPCCStageBlocked(reason, "exact_starting_position_lots")); p.seed.position.signed_lots = (float)lots; if(p.model_sha256 != out.model_reference.sha256 || p.training_sha256 != out.training_reference.sha256 || p.bank_sha256 != actual_bank_sha || p.time_index_sha256 != out.source.links.time_index_sha256) return(DPCCStageBlocked(reason, "offline_actual_artifact_binding")); const int count = d.ChildCount(plan); if(d.Type(plan) != DPCC_JSON_ARRAY || count <= 0 || (ulong)count > p.maximum_work || ArrayResize(out.offline_fit, count) != count) return(DPCCStageBlocked(reason, "required_offline_fit")); for(int i = 0; i < count; i++) { const int item = d.ChildAt(plan, i); ulong record = 0; if(d.Type(item) != DPCC_JSON_OBJECT || d.ChildCount(item) != 9 || !DPCCStageReadUnsigned(d, item, "record_index", INT_MAX, record, reason) || !DPCCResourceReadString(d, item, "window_id", out.offline_fit[i].window_id, reason, "offline_fit") || !DPCCResourceReadString(d, item, "record_sha256", out.offline_fit[i].record_sha256, reason, "offline_fit") || !DPCCResourceReadLong(d, item, "episode_id", out.offline_fit[i].episode_id, reason, "offline_fit") || !DPCCResourceReadLong(d, item, "decision_bar", out.offline_fit[i].decision_bar, reason, "offline_fit") || !DPCCResourceReadLong(d, item, "decision_time", out.offline_fit[i].decision_time, reason, "offline_fit") || !DPCCStageReadHex(d, item, "decision_identity", out.offline_fit[i].decision_identity, reason) || !DPCCStageReadHex(d, item, "context_identity", out.offline_fit[i].context_identity, reason) || !DPCCStageReadHex(d, item, "owner_identity", out.offline_fit[i].owner_identity, reason)) return(false); out.offline_fit[i].record_index = (int)record; } SDPCCOfflineFitWitness witnesses[]; if(ArraySize(records) != ArraySize(out.keys) || ArrayResize(witnesses, ArraySize(records)) != ArraySize(records)) return(DPCCStageBlocked(reason, "offline_witness_array")); for(int i = 0; i < ArraySize(records); i++) if(!DPCCStageOfflineWitness(records[i], out.keys[i], witnesses[i], reason)) return(false); SDPCCOfflineFitPartition fit; fit.dataset_id = out.source.dataset.dataset_id; fit.bar_start = out.source.dataset.fit.bar_start; fit.bar_end = out.source.dataset.fit.bar_end; fit.start_utc = out.source.dataset.fit.start_utc; fit.end_utc = out.source.dataset.fit.end_utc; fit.validation_bar_start = out.source.dataset.validation.bar_start; fit.validation_start_utc = out.source.dataset.validation.start_utc; fit.embargo_bars = out.source.dataset.embargo_bars; if(!DPCCAdmitOfflinePolicy(p, out.offline_fit, witnesses, fit, out.source.time_index, out.config, model_limits, out.offline_admission, reason)) return(false); out.offline_policy = p; return(true); } //+------------------------------------------------------------------+ //| Function DPCCLoadStudyStageInputImpl | //+------------------------------------------------------------------+ bool DPCCLoadStudyStageInputImpl(const string path, const string expected_sha, const bool common_files, SDPCCStudyStageInput &out, string &loaded_phase, string &reason, const bool retain_online_records = false) { ZeroMemory(out); loaded_phase = ""; reason = ""; //--- Candidate publishes only after real artifacts, readiness and owner admission. SDPCCStudyStageInput candidate; ZeroMemory(candidate); uchar bytes[]; string source = "", observed = "", normalized = ""; if(!DPCCResolverLowerSha256(expected_sha) || !DPCCResolverPreparePath(path, normalized, reason, "study_input.path")) return(DPCCStageBlocked(reason, "trusted_envelope_path_sha")); string segments[]; if(!DPCCResolverAppendSegments(normalized, segments, 0, reason, "study_input.path")) return(false); normalized = DPCCResolverJoinSegments(segments); if(!DPCCReadArtifactBytes(normalized, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, bytes, reason) || !DPCCSHA256Lower(bytes, observed, reason) || observed != expected_sha || !DPCCCentralBytesSource(bytes, source, reason, "study_input")) return(DPCCStageBlocked(reason, "envelope_bytes_sha." + reason)); CDPCCJsonDocument envelope; if(!envelope.Parse(source, reason)) return(false); const int root = envelope.Root(); ulong version = 0; string phase = ""; if(!DPCCStageReadUnsigned(envelope, root, "version", 1, version, reason) || version != 1 || !DPCCResourceReadString(envelope, root, "phase", phase, reason, "study_input.phase") || (phase != "DECODER_TRAIN" && phase != "DECODER_VALIDATE" && phase != "DENOISER_TRAIN" && phase != "ASSEMBLE" && phase != "OFFLINE_ORACLE") || !DPCCResourceReadString(envelope, root, "package_root", candidate.package_root, reason, "study_input.package_root") || !DPCCRunContainingArtifact(candidate.package_root, normalized, candidate.containing_json, reason)) return(DPCCStageBlocked(reason, "version_phase_package." + reason)); candidate.source.common_files = common_files; const int schemas = envelope.FindMember(root, "schemas"); string integration_schema = "", profile_schema = "", report_schema = "", data_schema = "", resource_schema = ""; if(!DPCCStageSchema(envelope, schemas, "model", "10498a359f69fb249441f3bdbe6c3f52f0823a6b854205e42d2da6b9ae482248", candidate.package_root, candidate.containing_json, common_files, candidate.model_schema, reason) || !DPCCStageSchema(envelope, schemas, "training", "7c630093d92684a80dede93bd51c0fa339521d8586660b019e4107703a338a2a", candidate.package_root, candidate.containing_json, common_files, candidate.training_schema, reason) || !DPCCStageSchema(envelope, schemas, "bank", "13780a0801286e89c276e20fe892bfcc08ee5fc38aac710c0c044b698ce9cb06", candidate.package_root, candidate.containing_json, common_files, candidate.bank_schema, reason) || !DPCCStageSchema(envelope, schemas, "dataset", "a52ba580e67dbf507622c85fe336b45d9f54286c8d18a0839f99bdccf812417c", candidate.package_root, candidate.containing_json, common_files, candidate.dataset_schema, reason) || !DPCCStageSchema(envelope, schemas, "time_index", "040554860c0a1c1ff9f823c70710f5d7c51934c8c18c97e8c06e709f217ffb57", candidate.package_root, candidate.containing_json, common_files, candidate.time_index_schema, reason) || !DPCCStageSchema(envelope, schemas, "header", "d2baae000248d04a059541ed899265e2ea44e2c4318e740d8eb043d9d97de156", candidate.package_root, candidate.containing_json, common_files, candidate.source.header_schema, reason) || !DPCCStageSchema(envelope, schemas, "record", "4f4f472f4750416f88b5ad0d2ff2a0da5517fa23d7f24c8a32bf4badeed43b00", candidate.package_root, candidate.containing_json, common_files, candidate.source.record_schema, reason) || !DPCCStageSchema(envelope, schemas, "integration", "57746ce808197a39fad36279d98614b9e5ebc94d68e11901ce8318df90d11203", candidate.package_root, candidate.containing_json, common_files, integration_schema, reason) || !DPCCStageSchema(envelope, schemas, "profile", "1ecc318194a74c4c8af907e7a77278b3569d086c7c9a252cf838f098c4180379", candidate.package_root, candidate.containing_json, common_files, profile_schema, reason) || !DPCCStageSchema(envelope, schemas, "report", "a7206d77e68fc8d30ce426caea1c731d3f440aba5c3b50a753d1e9fbad3eec9e", candidate.package_root, candidate.containing_json, common_files, report_schema, reason) || !DPCCStageSchema(envelope, schemas, "data_evidence", "2d4e4d40986ee7c2947bc6503982f76862a5e130b660a631f13bf989d9fcb8cc", candidate.package_root, candidate.containing_json, common_files, data_schema, reason) || !DPCCStageSchema(envelope, schemas, "resource_evidence", "f31702b71f26848af66c89783faf39e58989cd5e6d09bdc88f2cfcc9ecaecb7d", candidate.package_root, candidate.containing_json, common_files, resource_schema, reason)) return(false); if(!DPCCBankReadReference(envelope, root, "model", candidate.model_reference, reason, "study_input.model") || !DPCCBankReadReference(envelope, root, "training", candidate.training_reference, reason, "study_input.training")) return(false); SDPCCArtifactResolveResult model_artifact, training_artifact, report_artifact; string model_source = "", training_source = "", report_source = ""; if(!DPCCStageResolve(envelope, root, "model", candidate.package_root, candidate.containing_json, common_files, model_artifact, model_source, reason) || !DPCCStageResolve(envelope, root, "training", candidate.package_root, candidate.containing_json, common_files, training_artifact, training_source, reason) || !DPCCStageResolve(envelope, root, "readiness_report", candidate.package_root, candidate.containing_json, common_files, report_artifact, report_source, reason)) return(false); string model_relative = "", report_relative = ""; if(!DPCCRunContainingArtifact(candidate.package_root, model_artifact.canonical_path, model_relative, reason) || !DPCCRunContainingArtifact(candidate.package_root, report_artifact.canonical_path, report_relative, reason)) return(false); SDPCCCentralLinkResult central; if(!DPCCResolveCentralMetadata(candidate.package_root, model_relative, common_files, true, model_artifact.observed_sha256, candidate.model_schema, integration_schema, candidate.training_schema, profile_schema, central, reason)) return(false); if(central.training_path != training_artifact.canonical_path || central.training_sha256 != training_artifact.observed_sha256 || central.training_id != candidate.training_reference.id || central.model_id != candidate.model_reference.id) return(DPCCStageBlocked(reason, "actual_model_training_link")); CDPCCJsonDocument model_document, profile_document, report_document; string profile_source = "", profile_observed = ""; if(!model_document.Parse(model_source, reason) || !DPCCDatasetReadSource(central.profile_path, common_files, central.profile_sha256, profile_source, profile_observed, reason, "profile") || !profile_document.Parse(profile_source, reason) || !DPCCCentralSchemaValidate(report_document, report_source, report_schema, reason, "readiness_report")) return(false); bool report_example = false; if(!DPCCResourceReadBool(report_document, report_document.Root(), "example_only", report_example, reason, "report.example_only") || report_example) return(DPCCStageBlocked(reason, "report.example_only")); SDPCCResourceProfile resource_profile; SDPCCDataReport data_report; if(!DPCCDecodeResourceProfile(profile_document, profile_document.Root(), resource_profile, reason) || !DPCCDecodeDataProfile(profile_document, profile_document.Root(), candidate.source.profile, reason) || !DPCCDecodeDataReport(report_document, report_document.Root(), data_report, reason) || !DPCCStageExternalReview(envelope, report_document, candidate.package_root, report_relative, common_files, reason) || !DPCCStagePretrainResources(report_document, resource_schema, resource_profile, central.profile_sha256, candidate.package_root, report_relative, common_files, reason)) return(false); SDPCCArtifactResolveResult actual_report_profile; string ignored_profile = ""; if(!DPCCStageResolve(report_document, report_document.Root(), "readiness_profile", candidate.package_root, report_relative, common_files, actual_report_profile, ignored_profile, reason) || actual_report_profile.canonical_path != central.profile_path) return(DPCCStageBlocked(reason, "report.actual_profile")); const int training_node = model_document.FindMember(model_document.Root(), "training"); SDPCCArtifactResolveResult bank_artifact; string bank_source = "", bank_relative = ""; if(!DPCCStageResolve(model_document, training_node, "bank_manifest", candidate.package_root, model_relative, common_files, bank_artifact, bank_source, reason) || !DPCCRunContainingArtifact(candidate.package_root, bank_artifact.canonical_path, bank_relative, reason)) return(false); CDPCCJsonDocument bank_document; if(!DPCCCentralSchemaValidate(bank_document, bank_source, candidate.bank_schema, reason, "bank")) return(false); SDPCCResourceReference bank_file; string bank_id = ""; if(!DPCCBankReadReference(bank_document, bank_document.Root(), "file", bank_file, reason, "bank.file") || !DPCCResourceReadString(bank_document, bank_document.Root(), "bank_id", bank_id, reason, "bank.bank_id") || !DPCCNormalizePackagePath(candidate.package_root, bank_relative, bank_file.path, candidate.source.path, reason)) return(false); SDPCCBankIntegrityResult integrity; if(!DPCCScanPublishedBankIntegrity(candidate.source.path, common_files, bank_file.sha256, integrity, reason)) return(false); if(integrity.record_count > INT_MAX || !DPCCCentralBytesSource(integrity.header_bytes, candidate.source.header, reason, "bank.header")) return(DPCCStageBlocked(reason, "bank.capacity_header")); if(!data_report.source_distribution_present) return(DPCCStageBlocked(reason, "source_distribution_report")); SDPCCArtifactResolveResult data_artifact; if(!DPCCResolveArtifact(candidate.package_root, report_relative, data_report.source_distribution_report, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, data_artifact, reason)) return(false); string data_relative = ""; if(!DPCCRunContainingArtifact(candidate.package_root, data_artifact.canonical_path, data_relative, reason)) return(false); ENUM_DPCC_GATE_VERDICT data_gate = DPCC_GATE_BLOCKED; if(!DPCCResolveDatasetClosure(candidate.package_root, common_files, true, central, candidate.model_schema, candidate.training_schema, profile_schema, candidate.dataset_schema, candidate.time_index_schema, data_schema, data_relative, data_artifact.observed_sha256, bank_id, bank_artifact.observed_sha256, candidate.source.links, data_gate, reason) || data_gate != DPCC_GATE_PASS) return(DPCCStageGate(reason, "data.closure", data_gate)); string dataset_source = "", dataset_observed = "", index_source = "", index_observed = ""; CDPCCJsonDocument dataset_document, index_document, data_document; SDPCCDataEvidence data_evidence; if(!DPCCDatasetReadSource(candidate.source.links.dataset_path, common_files, candidate.source.links.dataset_sha256, dataset_source, dataset_observed, reason, "dataset") || !DPCCDatasetReadSource(candidate.source.links.time_index_path, common_files, candidate.source.links.time_index_sha256, index_source, index_observed, reason, "time_index") || !dataset_document.Parse(dataset_source, reason) || !index_document.Parse(index_source, reason) || !DPCCDecodeDatasetManifest(dataset_document, dataset_document.Root(), candidate.source.dataset, reason) || !DPCCDecodeDatasetTimeIndex(index_document, index_document.Root(), candidate.source.dataset.dataset_id, candidate.source.time_index, reason) || !DPCCValidateBankMetadata(bank_source, integrity.header_bytes, dataset_source, candidate.bank_schema, candidate.source.header_schema, candidate.dataset_schema, candidate.source.dataset, candidate.source.links, candidate.source.profile, bank_id, integrity.bank_sha256, (long)integrity.record_count, true, candidate.source.bank, reason)) return(false); string data_source = ""; if(!DPCCCentralBytesSource(data_artifact.bytes, data_source, reason, "data") || !data_document.Parse(data_source, reason) || !DPCCDecodeDataEvidence(data_document, data_document.Root(), data_evidence, reason)) return(false); SDPCCArtifactResolveResult evidence_bank; string ignored_bank = ""; if(!DPCCStageResolve(data_document, data_document.Root(), "bank_manifest", candidate.package_root, data_relative, common_files, evidence_bank, ignored_bank, reason) || evidence_bank.canonical_path != bank_artifact.canonical_path) return(DPCCStageBlocked(reason, "data.actual_bank_manifest")); SDPCCDataBindings bindings; bindings.readiness_profile_id = central.profile_id; bindings.readiness_profile_sha256 = central.profile_sha256; bindings.dataset_id = candidate.source.dataset.dataset_id; bindings.dataset_sha256 = candidate.source.links.dataset_sha256; bindings.bank_id = bank_id; bindings.bank_sha256 = bank_artifact.observed_sha256; bindings.training_id = central.training_id; bindings.training_sha256 = central.training_sha256; bindings.rho_id = candidate.source.profile.rho_implementation.id; bindings.rho_sha256 = candidate.source.links.rho_sha256; bindings.forecast_id = candidate.source.profile.forecast_id; bindings.normalization_id = candidate.source.dataset.normalization_id; SDPCCResourceReference observed_data[]; if(ArrayResize(observed_data, ArraySize(data_report.data_evidence)) != ArraySize(data_report.data_evidence)) return(DPCCStageBlocked(reason, "data_evidence_reference_capacity")); for(int i = 0; i < ArraySize(observed_data); i++) { SDPCCArtifactResolveResult resolved; if(!DPCCResolveArtifact(candidate.package_root, report_relative, data_report.data_evidence[i], common_files, DPCC_RESOLVER_MAX_JSON_BYTES, resolved, reason)) return(false); observed_data[i] = data_report.data_evidence[i]; } for(int i = 0; i < ArraySize(data_report.validated_inputs); i++) { SDPCCArtifactResolveResult validated; if(!DPCCResolveArtifact(candidate.package_root, report_relative, data_report.validated_inputs[i], common_files, DPCC_RESOLVER_MAX_JSON_BYTES, validated, reason)) return(false); } SDPCCArtifactResolveResult report_rho; if(!data_report.rho_extraction_present || !DPCCResolveArtifact(candidate.package_root, report_relative, data_report.rho_extraction_report, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, report_rho, reason) || report_rho.canonical_path != candidate.source.links.rho_path) return(DPCCStageBlocked(reason, "report.actual_rho")); if(!DPCCEvaluateDataReportClaim(candidate.source.profile, data_evidence, bindings, data_report, data_artifact.observed_sha256, observed_data, true, data_gate, reason) || data_gate != DPCC_GATE_PASS) return(DPCCStageGate(reason, "data.derived", data_gate)); CDPCCAdmittedTrainingConfig admitted; if(!admitted.Admit(candidate.package_root, candidate.containing_json, candidate.training_reference, candidate.model_reference, common_files, candidate.training_schema, candidate.model_schema, candidate.bank_schema, candidate.dataset_schema, candidate.time_index_schema, candidate.source, reason)) return(false); SDPCCTrainingConfigAdmission admission; SDPCCTrainingRoleConfig decoder_role, denoiser_role; if(!admitted.Get(DPCC_ROLE_DECODER, admission, decoder_role) || !admitted.Get(DPCC_ROLE_DENOISER, admission, denoiser_role) || admission.example_only) return(DPCCStageBlocked(reason, "production_training_admission")); string statistics_source = ""; float fixed_profile[]; SDPCCStageFitReceipt fit_receipt; if(!DPCCResourceReadString(model_document, model_document.FindMember(model_document.Root(), "normalization"), "statistics_source", statistics_source, reason, "study_input.normalization_source")) return(false); if(statistics_source == "FIT_FIXED") { candidate.fit_source.package_root = candidate.package_root; candidate.fit_source.containing_json = candidate.containing_json; candidate.fit_source.common_files = common_files; if(!DPCCBankReadReference(envelope, root, "fit_source", candidate.fit_source.reference, reason, "study_input.fit_source") || !DPCCStageReadUnsigned(envelope, root, "maximum_fit_source_bytes", DPCC_RESOLVER_MAX_JSON_BYTES, candidate.fit_source.maximum_bytes, reason) || candidate.fit_source.maximum_bytes == 0) return(DPCCStageBlocked(reason, "FIT_FIXED.immutable_fit_source_before_UPDATE")); CDPCCFullFitNormalization prepared; if(!DPCCStageFitPrepare(candidate.fit_source, model_source, admission, dataset_source, index_source, prepared, fit_receipt, reason) || !prepared.PreparedTrainingPriceNorm(fixed_profile, reason)) return(false); } else if(statistics_source != "INPUT_WINDOW") return(DPCCStageBlocked(reason, "normalization_source_unsupported")); SDPCCCentralModel model; if(!DPCCDecodeCentralModel(model_document, model, reason)) return(false); const long dimensions[] = {model.shapes.K, model.shapes.V, model.shapes.H, model.shapes.D, model.shapes.M, model.shapes.J, model.shapes.E}; for(int i = 0; i < ArraySize(dimensions); i++) if(dimensions[i] <= 0 || (ulong)dimensions[i] > UINT_MAX) return(DPCCStageBlocked(reason, "native_uint_shape")); if(model.shapes.J > 64) return(DPCCStageBlocked(reason, "native_diffusion_capacity")); candidate.config.shape.K = (uint)model.shapes.K; candidate.config.shape.V = (uint)model.shapes.V; candidate.config.shape.H = (uint)model.shapes.H; candidate.config.shape.D = (uint)model.shapes.D; candidate.config.shape.M = (uint)model.shapes.M; candidate.config.shape.J = (uint)model.shapes.J; candidate.config.shape.E = (uint)model.shapes.E; candidate.config.shape.PlanBatch = 1; candidate.config.shape.NActions = 6; candidate.config.shape.StateSize = 13; candidate.config.shape.Embedding = admission.architecture.model_denoiser.embedding; candidate.config.shape.Heads = admission.architecture.model_denoiser.heads; candidate.config.shape.KeyDimension = admission.architecture.model_denoiser.key_dimension; candidate.config.shape.Depth = admission.architecture.model_denoiser.depth; candidate.config.contract_version = 2; candidate.config.binary_layer_version = 3; candidate.config.normalization_mode = DPCC_NORMALIZATION_CAUSAL_AFFINE_V2; candidate.config.loss_coordinates = DPCC_LOSS_NORMALIZED; candidate.config.price_basis = admission.model_binding.price_basis; candidate.config.volume_scale = admission.model_binding.volume_scale; candidate.config.price_scale = admission.model_binding.price_scale; candidate.config.money_scale = admission.model_binding.money_scale; candidate.config.probability_tolerance = admission.model_binding.probability_tolerance; for(int i = 0; i < 6; i++) { candidate.config.action_bias[i] = admission.model_binding.action_bias[i]; candidate.config.action_scale[i] = admission.model_binding.action_scale[i]; } SDPCCConfig operational; float limits[]; if(!DPCCDecodePreparedActorConfig(model_source, candidate.config, operational, limits, reason)) return(false); operational.shape = candidate.config.shape; candidate.config = operational; candidate.decoder.embedding = admission.architecture.model_decoder.embedding; candidate.decoder.architecture = admission.architecture.model_decoder.architecture; candidate.decoder.activation_profile = admission.architecture.model_decoder.activation; candidate.decoder.backward_profile = admission.architecture.model_decoder.backward_profile; candidate.denoiser.graph = admission.architecture.model_denoiser; candidate.denoiser.parallel = admission.architecture.model_parallel; CDPCCJsonDocument training_document; if(!training_document.Parse(training_source, reason)) return(false); const int decoder_node = training_document.FindMember(training_document.Root(), "decoder"), denoiser_node = training_document.FindMember(training_document.Root(), "denoiser"); if(!DPCCResourceReadString(training_document, training_document.FindMember(decoder_node, "optimizer"), "profile", candidate.decoder.optimizer_profile, reason, "decoder.optimizer") || !DPCCResourceReadString(training_document, training_document.FindMember(denoiser_node, "optimizer"), "profile", candidate.denoiser.optimizer_profile, reason, "denoiser.optimizer")) return(false); //--- Decoder has no standalone graph artifact; bind its actual semantic section. string decoder_wire = "DPCC_DECODER_COMPOSITION_V1\n" + candidate.decoder.architecture + "\n" + ( string)candidate.decoder.embedding + "\n" + candidate.decoder.activation_profile + "\n" + candidate.decoder.backward_profile + "\n" + candidate.decoder.optimizer_profile + "\n"; if(!DPCCBankSourceHash(decoder_wire, candidate.decoder.graph_digest, reason, "decoder.composition")) return(false); SDPCCArtifactResolveResult graph; string graph_source = ""; if(!DPCCStageResolve(envelope, root, "denoiser_graph", candidate.package_root, candidate.containing_json, common_files, graph, graph_source, reason) || graph.observed_sha256 != admission.architecture.model_denoiser.graph_sha256) return(DPCCStageBlocked(reason, "actual_denoiser_graph")); double beta[], sigma[]; if(!DPCCBuildCosineDiffusionSchedule(candidate.config.shape.J, admission.model_binding.schedule_s, admission.model_binding.beta_cap, beta, candidate.alpha, sigma, reason)) return(false); for(int i = 0; i < ArraySize(candidate.alpha); i++) if(candidate.alpha[i] != admission.model_binding.alpha_bar[i] || beta[i] != admission.model_binding.beta[i] || sigma[i] != admission.model_binding.sigma[i]) return(DPCCStageBlocked(reason, "actual_cosine_schedule")); ulong capacity = 0, budget = 0, repeat = 0; if(!DPCCStageReadUnsigned(envelope, root, "maximum_records", INT_MAX, capacity, reason) || capacity == 0 || capacity < integrity.record_count || !DPCCStageReadUnsigned(envelope, root, "maximum_staged_tensor_bytes", ULONG_MAX, budget, reason) || budget == 0 || !DPCCStageReadUnsigned(envelope, root, "validation_repeat", UINT_MAX, repeat, reason) || !DPCCStageReadHex(envelope, root, "training_seed", candidate.training_seed, reason) || !DPCCStageReadHex(envelope, root, "validation_seed", candidate.validation_seed, reason) || StringFormat("%016I64x", candidate.training_seed) != denoiser_role.training_seed || StringFormat("%016I64x", candidate.validation_seed) != denoiser_role.validation_seed) return(DPCCStageBlocked(reason, "frozen_seed_repeat_capacity")); candidate.source.maximum_records = (uint)capacity; candidate.source.maximum_staged_tensor_bytes = budget; candidate.validation_repeat = (uint)repeat; SDPCCLayout c, r; uint tokens = 0; if(!DPCCCompositeLayouts(candidate.config.shape, c, r, tokens)) return(DPCCStageBlocked(reason, "native_layout")); const ulong H = candidate.config.shape.H, K = candidate.config.shape.K; const ulong per_record = ((ulong)c.offsets[10] + 13 + H * 4 + K + H * 12 + H) * sizeof(float) + H * sizeof(uint) + H * 4 * sizeof(long); const ulong copies = (retain_online_records ? 4 : 2); if(per_record > budget / copies / capacity) return(DPCCStageBlocked(reason, "staged_tensor_budget")); CDPCCBankMaterializationObserver observer; SDPCCMaterializedRecord records[]; ulong invalid = 0; if(!observer.Configure(candidate.source.record_schema, candidate.source.header, candidate.source.header_schema, candidate.source.dataset, candidate.source.links, candidate.source.profile, candidate.source.bank, candidate.source.time_index, candidate.config, candidate.source.maximum_records, reason) || !DPCCStageMaterializedBank(observer, candidate.source.path, common_files, integrity.bank_sha256, records, integrity, invalid, reason) || !DPCCStageReadKeys(envelope, records, candidate.config.shape.J, candidate.keys, reason)) return(false); const int materialized_count = ArraySize(records); //--- Parse-only admission; existing Study routing cannot execute this phase. if(phase == "OFFLINE_ORACLE" && (!DPCCStageReadOfflinePolicy(envelope, records, candidate, limits, integrity.bank_sha256, reason) || !DPCCStageReadOfflineClock(envelope, root, candidate.package_root, candidate.containing_json, candidate, reason))) return(false); const int plans = envelope.FindMember(root, "plans"); if(!DPCCStageReadPlan(envelope, plans, "decoder_fit", candidate.decoder_fit, reason) || !DPCCStageReadPlan(envelope, plans, "decoder_validation", candidate.decoder_validation, reason) || !DPCCStageReadPlan(envelope, plans, "denoiser_fit", candidate.denoiser_fit, reason) || !DPCCStageReadPlan(envelope, plans, "denoiser_validation", candidate.denoiser_validation, reason) || !DPCCResourceReadString(envelope, plans, "decoder_sha256", candidate.decoder_plan_sha, reason, "plans.decoder_sha256") || !DPCCResourceReadString(envelope, plans, "denoiser_sha256", candidate.denoiser_plan_sha, reason, "plans.denoiser_sha256")) return(false); //--- Owner Initialize repeats the authoritative two-copy staging transaction. //--- Do not retain a third copy of every tensor during that admission. if(ArrayResize(records, 0) != 0) return(DPCCStageBlocked(reason, "materialized_record_release")); //--- Reuse actual owner admission rather than duplicating its plan validator. //--- No UPDATE/checkpoint operation occurs; these candidate native owners expire. CDPCCTrainingExamples examples; CDPCCFrozenRolePlan decoder_plan, denoiser_plan; if(!examples.Initialize(candidate.source, candidate.config, candidate.decoder, candidate.denoiser, candidate.denoiser, admission.model_binding.schedule_s, admission.model_binding.beta_cap, candidate.alpha, candidate.keys, candidate.training_seed, candidate.validation_seed, candidate.validation_repeat, reason) || (statistics_source == "FIT_FIXED" && !examples.CheckFixedPriceNorm(fixed_profile, fit_receipt.data_cutoff, reason)) || !decoder_plan.Admit(admitted, DPCC_ROLE_DECODER, candidate.config.shape.J, examples, candidate.validation_repeat, candidate.decoder_fit, candidate.decoder_validation, candidate.decoder_plan_sha, reason) || !denoiser_plan.Admit(admitted, DPCC_ROLE_DENOISER, candidate.config.shape.J, examples, candidate.validation_repeat, candidate.denoiser_fit, candidate.denoiser_validation, candidate.denoiser_plan_sha, reason)) return(false); const int outputs = envelope.FindMember(root, "outputs"); if(!DPCCStageOutputPath(envelope, outputs, "checkpoint_stem", candidate.package_root, candidate.containing_json, candidate.checkpoint_stem, reason) || !DPCCStageOutputPath(envelope, outputs, "result_path", candidate.package_root, candidate.containing_json, candidate.result_path, reason) || !DPCCStageReadUnsigned(envelope, root, "maximum_checkpoint_bytes", ULONG_MAX, candidate.maximum_checkpoint_bytes, reason) || candidate.maximum_checkpoint_bytes == 0) return(DPCCStageBlocked(reason, "output_capacity_paths")); if(phase == "OFFLINE_ORACLE" && (!DPCCStageReadOfflineOutputs(envelope, outputs, candidate.package_root, candidate.containing_json, common_files, candidate, reason) || !DPCCStageAdmitOfflineClock(candidate.package_root, common_files, candidate, reason))) return(false); if(phase != "DECODER_TRAIN") { SDPCCArtifactResolveResult boundary, selected; string ignored = ""; if(!DPCCStageResolve(envelope, root, "boundary", candidate.package_root, candidate.containing_json, common_files, boundary, ignored, reason) || !DPCCStageResolve(envelope, root, "selected_decoder", candidate.package_root, candidate.containing_json, common_files, selected, ignored, reason)) return(false); candidate.boundary_path = boundary.canonical_path; candidate.boundary_sha = boundary.observed_sha256; candidate.selected_decoder_path = selected.canonical_path; candidate.selected_decoder_sha = selected.observed_sha256; if(phase == "OFFLINE_ORACLE" && candidate.offline_policy.decoder_sha256 != selected.observed_sha256) return(DPCCStageBlocked(reason, "offline_frozen_decoder_binding")); //--- Consumer's fresh owner replay checks authority/role/full native history. } if(phase == "ASSEMBLE") { if(!DPCCStageOutputPath(envelope, outputs, "actor_path", candidate.package_root, candidate.containing_json, candidate.actor_path, reason)) return(DPCCStageBlocked(reason, "assembly_actor_output")); if(statistics_source == "FIT_FIXED") { candidate.normalization_record = -1; } else if(statistics_source == "INPUT_WINDOW") { ulong index = 0; if(!DPCCStageReadUnsigned(envelope, root, "normalization_record", INT_MAX, index, reason) || index >= ( ulong)materialized_count) return(DPCCStageBlocked(reason, "assembly_causal_record_output")); candidate.normalization_record = (int)index; } else return(DPCCStageBlocked(reason, "normalization_source_unsupported")); } //--- Recheck trusted envelope immediately before publication. Linked immutable //--- artifacts are re-resolved again by the executing owner, never DTO-authorized. if(!DPCCReadArtifactBytes(normalized, common_files, DPCC_RESOLVER_MAX_JSON_BYTES, bytes, reason) || !DPCCSHA256Lower(bytes, observed, reason) || observed != expected_sha) return(DPCCStageBlocked(reason, "envelope_changed")); if(retain_online_records) { if(phase != "OFFLINE_ORACLE") return(DPCCStageBlocked(reason, "online_record_phase")); if(ArrayResize(candidate.online_records, materialized_count) != materialized_count) return(DPCCStageBlocked(reason, "online_record_capacity")); if(ArrayResize(candidate.online_witnesses, materialized_count) != materialized_count) return(DPCCStageBlocked(reason, "online_witness_capacity")); for(int i = 0; i < materialized_count; i++) { if(!examples.CopyAdmittedRecord(i, candidate.online_records[i], reason)) return(false); if(!DPCCStageOfflineWitness(candidate.online_records[i], candidate.keys[i], candidate.online_witnesses[i], reason)) return(false); } } out = candidate; loaded_phase = phase; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Function DPCCLoadStudyStageInput | //+------------------------------------------------------------------+ bool DPCCLoadStudyStageInput(const string path, const string expected_sha, const bool common_files, SDPCCStudyStageInput &out, string &loaded_phase, string &reason) { const bool accepted = DPCCLoadStudyStageInputImpl(path, expected_sha, common_files, out, loaded_phase, reason); if(!accepted && StringFind(reason, "BLOCKED:") != 0 && StringFind(reason, "FAIL:") != 0) reason = "BLOCKED:study_input." + reason; return(accepted); } //+------------------------------------------------------------------+ //| Retain admitted rows from the existing loader transaction | //+------------------------------------------------------------------+ bool DPCCLoadStudyStageInput(const string path, const string expected_sha, const bool common_files, SDPCCStudyStageInput &out, string &loaded_phase, string &reason, const bool retain_online_records) { const bool accepted = DPCCLoadStudyStageInputImpl(path, expected_sha, common_files, out, loaded_phase, reason, retain_online_records); if(!accepted && StringFind(reason, "BLOCKED:") != 0 && StringFind(reason, "FAIL:") != 0) reason = "BLOCKED:study_input." + reason; return(accepted); } //--- Fixed requested API. A dispatch consumer should use the overload above and //--- compare its local loaded_phase against the stage it is about to execute. bool DPCCLoadStudyStageInput(const string path, const string expected_sha, const bool common_files, SDPCCStudyStageInput &out, string &reason) { string loaded_phase = ""; return(DPCCLoadStudyStageInput(path, expected_sha, common_files, out, loaded_phase, reason)); } //+------------------------------------------------------------------+ //| Build one financial row from pinned raw and indexed clock | //+------------------------------------------------------------------+ bool DPCCBuildOracleHistoryRow(const SDPCCOracleHistoryInput &source, const int position, CDPCCMarketClock &clock, SDPCCOracleHistoryRow &out, string &reason) { ZeroMemory(out); if(position < 0 || position >= ArraySize(source.fit_indices) || ArraySize(source.fit_indices) != ArraySize(source.decision_bars)) return(DPCCStageBlocked(reason, "history.row.index_shape")); out.decision_bar = source.decision_bars[position]; if(!DPCCBuildHistoricalFinancialRow(source.config, source.financial_profile, source.raw_bars, source.fit_indices[position], out.decision_bar, position == 0, clock, out.context, out.native_ohlc, out.timeline_opens, out.timeline_closes, reason)) return(false); out.time_domain = clock.TimeDomain(); return(DPCCOfflineValidateFiniteFloats(out.context, reason)); } //+------------------------------------------------------------------+ //| DPCCLoadOracleHistoryInput loads Study inputs and derives | //| provenance | //+------------------------------------------------------------------+ bool DPCCLoadOracleHistoryInput(const string package_root_relative, const datetime training_start, const datetime training_end, const SDPCCRuntimeConfig &runtime, const float &limits[], const string raw_path, const string output_stem_relative, const float &coordinate_deltas[], const bool common_files, const ENUM_TIMEFRAMES timeframe, const string market_path, const string market_profile_text, const SDPCCAPNativeLoadContract &market_contract, SDPCCOracleHistoryInput &out, string &reason) { ZeroMemory(out); reason = ""; string root_path = "", container = "StudyInput.runtime.json"; if(training_start <= 0 || training_end <= training_start || ArraySize(coordinate_deltas) != 6 || ArraySize(limits) != 24 || !DPCCResolverPreparePath(package_root_relative, root_path, reason, "history.package") || !DPCCNormalizePackagePath(root_path, container, output_stem_relative, out.output_stem, reason) || ArrayCopy(out.deltas, coordinate_deltas) != 6) return(DPCCStageBlocked(reason, "history.direct_inputs")); if(timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || !DPCCValidateFixedConfig(runtime.financial.shape, runtime.financial) || ArraySize(runtime.beta) != (int)runtime.financial.shape.J + 1 || ArraySize(runtime.alpha_bar) != (int)runtime.financial.shape.J + 1 || ArraySize(runtime.sigma) != (int)runtime.financial.shape.J + 1 || market_path == "") {reason = "history.direct_runtime_or_market"; return(false);} CDPCCJsonDocument market_profile; string profile_sha = "", market_scope_sha = ""; if(!DPCCCheckpointTextSHA(market_profile_text, profile_sha, reason) || profile_sha != market_contract.profile_sha256 || !market_profile.Parse(market_profile_text, reason) || !DPCCExecutionMarketProfileSHA(market_profile, market_profile.Root(), market_scope_sha, reason) || market_scope_sha != market_contract.market_profile_sha256) {reason = "history.market_profile_binding:" + reason; return(false);} out.clock.package_root = root_path; out.clock.containing_json = container; out.clock.offline_clock_server_calendar = true; out.config = runtime.financial; if(ArrayCopy(out.actor_beta, runtime.beta) != ArraySize(runtime.beta) || ArrayCopy(out.actor_alpha_bar, runtime.alpha_bar) != ArraySize(runtime.alpha_bar) || ArrayCopy(out.actor_sigma, runtime.sigma) != ArraySize(runtime.sigma) || !DPCCEncodeAction6ModelRuntime(runtime, limits, out.model_text, reason) || !DPCCCheckpointTextSHA(out.model_text, out.model_sha, reason)) return(DPCCStageBlocked(reason, "history.runtime_metadata")); //--- Read the complete source once; later stages consume these decoded bars directly. int fit[], holdout[]; string source_indicator_params = ""; long raw_successor = 0; string source_server = "", source_symbol = "", source_timeframe = ""; if(!DPCCReadStudyForecastRawEx(raw_path, "", out.raw_bars, out.raw_sha, raw_successor, reason, source_server, source_symbol, source_timeframe, source_indicator_params, common_files, "", false, false, false)) { if(reason == "") reason = "raw_decode_or_structure"; Print("DPCC_STUDY_RAW_INPUT_FAIL phase=decode reason=", reason); return(DPCCStageBlocked(reason, "history.raw_input")); } out.source_server = source_server; out.raw_sha = ""; out.raw_successor_open_server = raw_successor; out.raw_indicator_params = source_indicator_params; out.timeframe = timeframe; if(!DPCCCheckpointCanonicalPath(raw_path, out.raw_path, reason)) return(DPCCStageBlocked(reason, "history.raw_path")); if(!DPCCBuildStudyForecastSplits(out.raw_bars, fit, holdout, reason, true, training_start, training_end) || !DPCCFilterStudyForecastPriceTargets(out.raw_bars, out.raw_bars[0].chart_quote_mode, out.raw_bars[0].raw_version == 3, fit, holdout, reason, timeframe)) { if(reason == "") reason = "raw_split_or_target_filter"; Print("DPCC_STUDY_RAW_INPUT_FAIL phase=split_filter reason=", reason); return(DPCCStageBlocked(reason, "history.raw_input")); } //--- Financial context uses the selected terminal symbol and current native properties. SDPCCExecutionProfileSnapshot execution; string execution_payload = ""; if(!DPCCCaptureExecutionProfile(_Symbol, timeframe, "", 0, execution, execution_payload, out.execution_sha, reason, true)) return(false); if(!DPCCSupportedHistoricalNativeExecution(execution, out.config.margin_fx_profile)) return(DPCCStageBlocked(reason, "history.execution.symbol_server_currency")); if(!DPCCPrepareHistoricalFinancialProfile(out.config, limits, execution, execution_payload, true, false, out.financial_profile, reason, out.clock.offline_clock_server_calendar, true)) return(false); if(!DPCCPrepareHistoricalFxCloses(out.config, execution, out.raw_bars, timeframe, out.financial_profile, reason)) return(false); out.execution_sha = out.financial_profile.sha256; const int count = ArraySize(fit); if(ArrayCopy(out.fit_indices, fit) != count || ArrayResize(out.decision_bars, count) != count) return(DPCCStageBlocked(reason, "history.stream.capacity")); for(int i = 0; i < count; i++) out.decision_bars[i] = fit[i]; const string recipe_path = out.output_stem + ".financial_profile." + out.financial_profile.sha256 + ".txt"; if(!FileIsExist(recipe_path, common_files ? FILE_COMMON : 0)) if(!DPCCCheckpointWriteText(recipe_path, common_files, out.financial_profile.payload, reason)) return(false); string recipe_sha = ""; if(!DPCCCheckpointFileSHA(recipe_path, common_files, 1048576, recipe_sha, reason) || recipe_sha != out.financial_profile.sha256) return(DPCCStageBlocked(reason, "history.profile.receipt_changed")); out.rows_sha = out.financial_profile.sha256; //--- Only the derived financial recipe is a file-backed source; runtime settings came from EA inputs. const int source_count = 1; if(ArrayResize(out.source_labels, source_count) != source_count || ArrayResize(out.source_paths, source_count) != source_count || ArrayResize(out.source_hashes, source_count) != source_count) return(DPCCStageBlocked(reason, "history.provenance.capacity")); out.source_labels[0] = "derived_financial_profile"; if(!DPCCCheckpointCanonicalPath(recipe_path, out.source_paths[0], reason)) return(false); out.source_hashes[0] = out.financial_profile.sha256; string current = ""; if(!DPCCCheckpointFileSHA(out.source_paths[0], common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, current, reason) || current != out.source_hashes[0]) {reason = "BLOCKED:study.history.provenance.source_changed.derived_financial_profile:" + reason; return(false);} if(!DPCCOracleHistoryConfigSHA(out.config, out.config_sha, reason) || !DPCCOracleHistoryDeltasSHA(out.deltas, out.deltas_sha, reason)) return(false); reason = ""; return(true); } #endif #ifndef DPCC_STUDY_STAGE_CONSUMER_MQH #define DPCC_STUDY_STAGE_CONSUMER_MQH //+------------------------------------------------------------------+ //| StudyForecastCore.mqh | //+------------------------------------------------------------------+ //--- Native Oracle-history training consumer; admission remains the existing loader. class CDPCCStudyStageConsumer { private: bool m_initialized, m_failed; //--- Encoder outlives every native policy/rollout borrowing its context. CDPCCStudyForecastCore m_forecast; //--- Declared after m_forecast so this device buffer is released before its context owner. CBufferFloat m_z_latent; CBufferFloat m_action6_snapshot_work, m_action6_context_work, m_action6_flags_work, m_action6_status_work; CNet m_offline_factual, m_offline_scoring; SDPCCConfig m_offline_config; CNet m_z_actor, m_z_q1, m_z_q2; //--- Action6 owns only the trainable policy graphs; Market/context are borrowed from m_forecast. CNet *m_action6_actor, *m_action6_q1, *m_action6_q2; SDPCCRuntimeConfig m_action6_runtime; bool m_action6_history; string m_action6_market_sha, m_action6_market_data_sha, m_action6_profile_sha; string m_action6_market_path, m_action6_profile_text, m_action6_task_id; ulong m_action6_market_max_bytes; bool m_z_history, m_z_floor; uint m_z_step, m_z_rows, m_z_resume_cursor; long m_z_last_bar, m_z_first_server_bar, m_z_next_server_bar; string m_z_resume_manifest, m_z_resume_sha; float m_z_state[], m_z_deltas[]; SDPCCSimulatedPositionMetadata m_z_position; string m_z_raw_sha, m_z_raw_path, m_z_receipt_sha; string m_z_output_stem; string m_z_source_labels[], m_z_source_paths[], m_z_source_hashes[]; string m_z_model_text, m_z_model_sha; string m_z_config_sha, m_z_deltas_sha, m_z_execution_sha, m_z_time_domain, m_z_account_currency; uint m_z_account_currency_digits; bool m_z_provenance_bound, m_z_common_files; bool Fail(const string cause, string &reason) { m_failed = true; reason = (StringLen(cause) > 0 ? cause : "study.operation_failed"); return(false); } bool SameFinancialProfileExceptSnapshotTimesAndTerminalBuild(const string older, const string current, string &reason) { string payloads[2], normalized[2]; payloads[0] = older; payloads[1] = current; normalized[0] = ""; normalized[1] = ""; bool has_terminal_build[2]; ArrayInitialize(has_terminal_build, false); for(int which = 0; which < 2; which++) { int pos = 0; while(pos < StringLen(payloads[which])) { const int end = StringFind(payloads[which], "\n", pos); if(end < pos) { reason = "study.history.restore.profile_line_termination"; return(false); } const string line = StringSubstr(payloads[which], pos, end - pos); const int first = StringFind(line, "|"), second = (first < 0 ? -1 : StringFind(line, "|", first + 1)); ulong declared = 0; if(first <= 0 || second <= first + 1 || !DPCCCheckpointReadUnsigned(StringSubstr(line, first + 1, second - first - 1), 1048576, declared) || StringLen(StringSubstr(line, second + 1)) != (int)declared) {reason = "study.history.restore.profile_record"; return(false);} const string key = StringSubstr(line, 0, first), value = StringSubstr(line, second + 1); if(key == "clock_mapping_id" || key == "snapshot_time_utc_epoch" || key == "snapshot_time_utc" || key == "snapshot_time_server") { //--- Legacy/volatile clock metadata is provenance, not a model compatibility gate. } else if(key == "terminal_build") { ulong build = 0; if(has_terminal_build[which] || !DPCCCheckpointReadUnsigned(value, (ulong)LONG_MAX, build) || build == 0) {reason = "study.history.restore.profile_terminal_build"; return(false);} has_terminal_build[which] = true; normalized[which] += "terminal_build|\n"; } else normalized[which] += StringSubstr(payloads[which], pos, end - pos + 1); pos = end + 1; } if(!has_terminal_build[which]) {reason = "study.history.restore.profile_terminal_build"; return(false);} } if(normalized[0] != normalized[1]) {reason = "study.history.restore.profile_non_exception_difference"; return(false);} reason = ""; return(true); } bool Action6TrainingPaths(const string base, const bool common_files, string &actor_path, string &q1_path, string &q2_path, string &manifest_path, string &reason) { actor_path = ""; q1_path = ""; q2_path = ""; manifest_path = ""; string canonical = ""; if(base == "" || StringLen(base) > 220 || !DPCCCheckpointCanonicalPath(base, canonical, reason) || canonical != base) { reason = "study.action6.training_state_path"; return(false); } actor_path = base + ".actor.dpct"; q1_path = base + ".q1.dpct"; q2_path = base + ".q2.dpct"; manifest_path = base + ".manifest"; string paths[4]; paths[0] = actor_path; paths[1] = q1_path; paths[2] = q2_path; paths[3] = manifest_path; for(int i = 0; i < ArraySize(paths); i++) if(!DPCCCheckpointCanonicalPath(paths[i], canonical, reason) || canonical != paths[i]) { reason = "study.action6.training_state_path"; return(false); } int slash = StringFind(base, "/"); while(slash > 0) { FolderCreate(StringSubstr(base, 0, slash), common_files ? FILE_COMMON : 0); slash = StringFind(base, "/", slash + 1); } reason = ""; return(true); } bool Action6SavePolicyState(CNet *net, const string path, const bool common_files, string &sha, string &reason) { sha = ""; if(CheckPointer(net) == POINTER_INVALID) { reason = "study.action6.training_state.network"; return(false); } ResetLastError(); const int handle = FileOpen(path, FILE_WRITE | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("study.action6.training_state.open.%d", GetLastError()); return(false); } ResetLastError(); CNeuronDPCCOCL *action6_actor = dynamic_cast(net.Layer(1)); bool saved = false; if(CheckPointer(action6_actor) != POINTER_INVALID && action6_actor.Type() == defNeuronDPCCOCL && action6_actor.ActorABI() == 3 && net.Layer(2) == NULL) saved = action6_actor.Save(handle); else saved = net.Save(handle); FileFlush(handle); saved = saved && GetLastError() == 0; FileClose(handle); if(!saved || !DPCCCheckpointFileSHA(path, common_files, 536870912, sha, reason)) { if(reason == "") reason = "study.action6.training_state.write"; return(false); } return(true); } bool Action6LoadPolicyState(CNet *net, const string path, const bool common_files, string &reason) { if(CheckPointer(net) == POINTER_INVALID) { reason = "study.action6.training_state.network"; return(false); } ResetLastError(); const int handle = FileOpen(path, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("study.action6.training_state.open.%d", GetLastError()); return(false); } CNeuronDPCCOCL *action6_actor = dynamic_cast(net.Layer(1)); const bool action6_actor_graph = (CheckPointer(action6_actor) != POINTER_INVALID && action6_actor.Type() == defNeuronDPCCOCL && action6_actor.ActorABI() == 3 && net.Layer(2) == NULL); bool loaded = false; if(action6_actor_graph) { //--- The direct Actor save includes its layer type; CNeuronDPCCOCL::Load expects CLayer to consume it. if(FileTell(handle) + INT_VALUE > FileSize(handle) || FileReadInteger(handle, INT_VALUE) != (int)action6_actor.Type()) { FileClose(handle); reason = "study.action6.training_state.actor_type_prefix"; return(false); } const bool actor_loaded = action6_actor.Load(handle); const int actor_status = (int)action6_actor.LastStatus(); const ulong actor_offset = FileTell(handle); const bool actor_complete = (actor_loaded && FileTell(handle) == FileSize(handle) && GetLastError() == 0); loaded = actor_loaded && actor_complete; if(loaded && (!net.TrainMode(true) || !net.SetWeightsUpdate(true))) { loaded = false; reason = "study.action6.training_state.trainable_restore"; } FileClose(handle); if(!actor_loaded) reason = StringFormat("study.action6.training_state.actor_layer_load.status_%d.offset_%I64u", actor_status, actor_offset); else if(!actor_complete) reason = "study.action6.training_state.actor_file_end"; } else { //--- CNet::Load owns and closes the file handle after reading the network. if(!net.IsOpenCLTransferReady()) { FileClose(handle); reason = "study.action6.training_state.cnet_transfer_not_ready"; } else { loaded = net.Load(handle); if(!loaded) reason = "study.action6.training_state.cnet_layer_load"; } } if(!loaded) { if(reason == "") reason = "study.action6.training_state.restore"; return(false); } return(true); } bool Action6ManifestNextLine(const string &source, int &position, string &line) { line = ""; const int end = StringFind(source, "\n", position); if(end < position) return(false); line = StringSubstr(source, position, end - position); position = end + 1; return(true); } bool Action6ParseTrainingManifest(const string &source, string &actor_sha, string &q1_sha, string &q2_sha, string &reason) { actor_sha = ""; q1_sha = ""; q2_sha = ""; int position = 0; string lines[5]; for(int i = 0; i < ArraySize(lines); i++) if(!Action6ManifestNextLine(source, position, lines[i])) { reason = "study.action6.training_state.manifest_lines"; return(false); } if(position != StringLen(source) || lines[0] != "DPCC_ACTION6_TRAINING_STATE_V1" || lines[4] != "END_DPCC_ACTION6_TRAINING_STATE_V1" || !DPCCResolverLowerSha256(lines[1]) || !DPCCResolverLowerSha256(lines[2]) || !DPCCResolverLowerSha256(lines[3])) { reason = "study.action6.training_state.manifest_format"; return(false); } actor_sha = lines[1]; q1_sha = lines[2]; q2_sha = lines[3]; reason = ""; return(true); } public: CDPCCStudyStageConsumer(void) { m_initialized = false; m_failed = false; m_z_history = false; m_action6_actor = NULL; m_action6_q1 = NULL; m_action6_q2 = NULL; m_action6_history = false; m_action6_market_sha = ""; m_action6_profile_sha = ""; m_action6_market_path = ""; m_action6_profile_text = ""; m_z_model_text = ""; m_z_model_sha = ""; m_action6_task_id = ""; m_action6_market_max_bytes = 0; m_z_output_stem = ""; m_z_provenance_bound = false; m_z_common_files = false; m_z_floor = false; m_z_step = 0; m_z_rows = 0; m_z_resume_cursor = 0; m_z_resume_manifest = ""; m_z_resume_sha = ""; m_z_last_bar = -1; m_z_first_server_bar = 0; m_z_next_server_bar = 0; m_z_account_currency = ""; m_z_account_currency_digits = 0; ZeroMemory(m_z_position); } ~CDPCCStudyStageConsumer(void) { if(CheckPointer(m_action6_actor) != POINTER_INVALID) delete m_action6_actor; if(CheckPointer(m_action6_q2) != POINTER_INVALID) delete m_action6_q2; if(CheckPointer(m_action6_q1) != POINTER_INVALID) delete m_action6_q1; } //+------------------------------------------------------------------+ //| Reset transient policy/account state after one full history pass | //+------------------------------------------------------------------+ bool BeginAction6HistoryPass(string &reason) { reason = ""; if(!m_initialized || m_failed || !m_action6_history || !m_z_history || !m_z_provenance_bound || m_z_rows == 0 || m_z_step != m_z_rows || m_z_floor || CheckPointer(m_action6_actor) == POINTER_INVALID || CheckPointer(m_action6_q1) == POINTER_INVALID || CheckPointer(m_action6_q2) == POINTER_INVALID) { reason = "study.action6.history_pass.requires_complete_prior_pass"; return(false); } CNeuronDPCCOCL *actor_layer = dynamic_cast(m_action6_actor.Layer(1)); CNeuronBaseOCL *q1_last = m_action6_q1.Layer(-1); CNeuronBaseOCL *q2_last = m_action6_q2.Layer(-1); if(CheckPointer(actor_layer) == POINTER_INVALID || actor_layer.Type() != defNeuronDPCCOCL || actor_layer.ActorABI() != 3 || !actor_layer.CanStartForward() || CheckPointer(q1_last) == POINTER_INVALID || CheckPointer(q2_last) == POINTER_INVALID || !q1_last.CanStartForward() || !q2_last.CanStartForward()) { reason = "study.action6.history_pass.pending_policy_generation"; return(false); } //--- The forecast owner rejects a live Market generation before its //--- recurrent buffers are reset. Policy Clear methods are transient-only: //--- ABI3 Actor clears execution state and frame-mode denoiser slots; //--- CNet::Clear preserves trainable parameters and optimizer moments. if(!m_forecast.ResetReadyEncoderHistoryPass(reason)) return(false); if(!m_action6_actor.Clear() || !m_action6_q1.Clear() || !m_action6_q2.Clear()) { reason = "study.action6.history_pass.policy_clear"; return(false); } ArrayInitialize(m_z_state, 0.0f); ZeroMemory(m_z_position); m_z_floor = false; m_z_step = 0; m_z_resume_cursor = 0; m_z_resume_manifest = ""; m_z_resume_sha = ""; m_z_last_bar = -1; m_z_first_server_bar = 0; m_z_next_server_bar = 0; return(true); } //+------------------------------------------------------------------+ //| Restore a profile-bound Market and initialize native Action6/Qs | //+------------------------------------------------------------------+ bool InitializeAction6History(const SDPCCConfig &config, const uint rows, const float &deltas[], const double &beta[], const double &alpha_bar[], const double &sigma[], const ENUM_DPCC_MODE selected_mode, const ENUM_TIMEFRAMES timeframe, const SDPCCStudyForecastBar &raw_bars[], const string raw_server, const string raw_indicator_params, const long raw_successor, const string raw_path, const string market_profile_text, const string market_path, const SDPCCAPNativeLoadContract &market_contract, const float &condition_scale[], const float &condition_bias[], const bool common_files, string &reason) { SDPCCConfig operational = config; operational.mode = selected_mode; string canonical_market = "", canonical_profile = "", assigned_profile_sha = "", profile_text = market_profile_text; uchar profile_bytes[]; if(!DPCCCheckpointCanonicalPath(market_path, canonical_market, reason) || canonical_market != market_path || profile_text == "" || !DPCCCheckpointTextSHA(profile_text, assigned_profile_sha, reason) || assigned_profile_sha != market_contract.profile_sha256) return(Fail("BLOCKED:study.action6.market_profile_input_binding:" + reason, reason)); CDPCCJsonDocument assigned_profile; string market_profile_sha = ""; if(!assigned_profile.Parse(profile_text, reason) || !DPCCExecutionMarketProfileSHA(assigned_profile, assigned_profile.Root(), market_profile_sha, reason) || market_profile_sha != market_contract.market_profile_sha256 || !DPCCResolverLowerSha256(market_contract.profile_sha256)) return(Fail("BLOCKED:study.action6.market_profile_stage_binding:" + reason, reason)); if(m_initialized || m_failed || rows == 0 || rows > 16777216 || ArraySize(deltas) != 6 || selected_mode < DPCC_P0 || selected_mode > DPCC_P2 || timeframe <= PERIOD_CURRENT || PeriodSeconds(timeframe) <= 0 || !DPCCValidateFixedConfig(operational.shape, operational) || config.shape.K != 21 || config.shape.H != 12 || market_path == "" || market_contract.role != 1 || market_contract.section_id != 100 || market_contract.task_id == "" || market_contract.stage == "" || !m_forecast.RestoreReadyEncoderFromBars(market_path, common_files, raw_bars, raw_server, raw_indicator_params, raw_successor, timeframe, market_contract, reason, true)) return(Fail(reason == "" ? "BLOCKED:study.action6.market_profile_binding" : reason, reason)); for(int i = 0; i < 6; i++) if(!MathIsValidNumber(deltas[i]) || deltas[i] <= 0 || deltas[i] > 1) return(Fail("study.action6.oracle_delta", reason)); CNet *market = m_forecast.ReadyMarketNetwork(); COpenCLMy *context = m_forecast.ReadyEncoderContext(); if(CheckPointer(market) == POINTER_INVALID || CheckPointer(context) == POINTER_INVALID || market.GetOpenCL() != context || !market.IsOpenCLTransferReady() || market.WeightsUpdateEnabled()) return(Fail("BLOCKED:study.action6.frozen_market_unavailable", reason)); string market_sha = ""; if(!DPCCCheckpointFileSHA(market_path, common_files, market_contract.max_native_bytes, market_sha, reason) || !DPCCResolverLowerSha256(market_sha) || !DPCCResolverLowerSha256(market_contract.profile_sha256)) return(Fail("BLOCKED:study.action6.market_identity:" + reason, reason)); SDPCCLayout action_layout; if(!DPCCBuildRolloutLayout(config.shape, action_layout) || action_layout.offsets[7] > (ulong)INT_MAX || action_layout.total > (ulong)INT_MAX || action_layout.offsets[7] < action_layout.lengths[0] || !m_action6_snapshot_work.BufferInit((uint)(action_layout.offsets[7] - action_layout.lengths[0]), 0) || !m_action6_context_work.BufferInit((uint)action_layout.total, 0) || !m_action6_flags_work.BufferInit((uint)action_layout.total, 0) || !m_action6_status_work.BufferInit(1, 0) || !m_action6_snapshot_work.BufferCreate(context) || !m_action6_context_work.BufferCreate(context) || !m_action6_flags_work.BufferCreate(context) || !m_action6_status_work.BufferCreate(context)) return(Fail("study.action6.context_workspace", reason)); SDPCCConfig physical = config; physical.shape.K = 1; physical.shape.M = 1; physical.shape.Depth = 0; if(!DPCCCreateHistoricalRolloutNet(m_offline_factual, physical, context, reason) || !DPCCCreateHistoricalRolloutNet(m_offline_scoring, physical, context, reason)) return(Fail("study.action6.rollout_graph:" + reason, reason)); SDPCCRuntimeConfig runtime; ZeroMemory(runtime); runtime.financial = operational; if(ArraySize(beta) != (int)config.shape.J + 1 || ArraySize(alpha_bar) != (int)config.shape.J + 1 || ArraySize(sigma) != (int)config.shape.J + 1 || ArraySize(condition_scale) != 109 || ArraySize(condition_bias) != 109 || ArrayCopy(runtime.beta, beta) != ArraySize(beta) || ArrayCopy(runtime.alpha_bar, alpha_bar) != ArraySize(alpha_bar) || ArrayCopy(runtime.sigma, sigma) != ArraySize(sigma)) return(Fail("study.action6.runtime_schedule_capacity", reason)); for(int i = 0; i < 109; i++) { if(!MathIsValidNumber(condition_scale[i]) || condition_scale[i] <= 0 || !MathIsValidNumber(condition_bias[i])) return(Fail(StringFormat("BLOCKED:study.action6.condition_profile_%d", i), reason)); runtime.condition_scale[i] = condition_scale[i]; runtime.condition_bias[i] = condition_bias[i]; } CNet *q1 = NULL, *q2 = NULL, *actor = NULL; if(!DPCCCreateAction6CriticNet(q1, market, reason) || !DPCCCreateAction6CriticNet(q2, market, reason)) { if(CheckPointer(q2) != POINTER_INVALID) delete q2; if(CheckPointer(q1) != POINTER_INVALID) delete q1; return(Fail("study.action6.critic_graph:" + reason, reason)); } CNeuronDPCCOCL *top = NULL; if(!DPCCCreateAction6ActorNet(actor, runtime, market, q1, q2, top, reason)) { delete q2; delete q1; return(Fail("study.action6.actor_graph:" + reason, reason)); } if(!actor.TrainMode(true) || !actor.SetWeightsUpdate(true) || !q1.TrainMode(true) || !q1.SetWeightsUpdate(true) || !q2.TrainMode(true) || !q2.SetWeightsUpdate(true)) { delete actor; delete q2; delete q1; return(Fail("study.action6.trainable_graph", reason)); } m_action6_actor = actor; m_action6_q1 = q1; m_action6_q2 = q2; if(!m_z_latent.BufferInit(21 * 9 * 12 * 16, 0) || !m_z_latent.BufferCreate(context)) return(Fail("study.action6.market_latent_buffer", reason)); if(ArrayCopy(m_z_deltas, deltas) != 6) return(Fail("study.action6.delta_capacity", reason)); m_offline_config = operational; m_z_rows = rows; m_z_raw_sha = ""; m_z_raw_path = raw_path; m_z_receipt_sha = ""; m_action6_market_sha = market_sha; m_action6_market_data_sha = market_contract.data_sha256; m_action6_profile_sha = assigned_profile_sha; m_action6_market_path = market_path; m_action6_profile_text = profile_text; m_action6_task_id = market_contract.task_id; m_action6_market_max_bytes = market_contract.max_native_bytes; m_action6_runtime.financial = operational; if(ArrayCopy(m_action6_runtime.beta, beta) != ArraySize(beta) || ArrayCopy(m_action6_runtime.alpha_bar, alpha_bar) != ArraySize(alpha_bar) || ArrayCopy(m_action6_runtime.sigma, sigma) != ArraySize(sigma) || ArrayCopy(m_action6_runtime.condition_scale, condition_scale) != 109 || ArrayCopy(m_action6_runtime.condition_bias, condition_bias) != 109) return(Fail("study.action6.runtime_profile_copy", reason)); m_z_common_files = common_files; m_z_history = true; m_action6_history = true; m_initialized = true; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Train one Action6 row and commit only its first operation | //+------------------------------------------------------------------+ bool StepAction6History(CDPCCMarketClock &clock, const SDPCCOracleHistoryRow &row, SDPCCOfflineRowResult &out, string &reason) { DPCCClearOfflineRowResult(out); const int horizon = (int)m_offline_config.shape.H; if(!m_action6_history || !m_initialized || m_failed || !m_z_provenance_bound || m_z_floor || CheckPointer(m_action6_actor) == POINTER_INVALID || CheckPointer(m_action6_q1) == POINTER_INVALID || CheckPointer(m_action6_q2) == POINTER_INVALID || m_z_step >= m_z_rows || clock.TimeDomain() != m_z_time_domain || row.decision_bar < 0 || (m_z_step > 0 && row.decision_bar != m_z_last_bar + 1) || ArraySize(row.native_ohlc) != horizon * 4 || ArraySize(row.timeline_opens) != horizon || ArraySize(row.timeline_closes) != horizon) return(Fail("study.action6.row_boundary", reason)); long opens[], closes[], ordinals[]; if(ArrayResize(opens, horizon) != horizon || ArrayResize(closes, horizon) != horizon || ArrayResize(ordinals, horizon) != horizon) return(Fail("study.action6.time_capacity", reason)); for(int h = 0; h < horizon; h++) { SDPCCClockBar bar; long open_time = 0, close_time = 0; if(!clock.IndexedTimelineBar(row.decision_bar + h, bar, open_time, close_time, reason) || bar.ordinal != row.decision_bar + h || open_time != row.timeline_opens[h] || close_time != row.timeline_closes[h]) return(Fail("study.action6.time_join:" + reason, reason)); opens[h] = bar.server_open; closes[h] = bar.server_close; ordinals[h] = bar.ordinal; } int raw_index = -1; if(!m_forecast.RawIndexForServerOpen(opens[0], false, raw_index, reason) || !m_forecast.MatchesNativeOHLC(raw_index, row.native_ohlc, opens, reason)) return(Fail("study.action6.raw_horizon_join:" + reason, reason)); float context[]; SDPCCLayout source_layout, unused_layout; uint tokens = 0; if(!DPCCCompositeLayouts(m_offline_config.shape, source_layout, unused_layout, tokens) || ArraySize(row.context) != (int)source_layout.offsets[10] || ArrayCopy(context, row.context) != ArraySize(row.context)) return(Fail("study.action6.financial_context_shape", reason)); if(m_z_step == 0) { SDPCCSnapshotMeta seed_meta; ZeroMemory(seed_meta); seed_meta.account_currency = m_z_account_currency; if(m_z_account_currency == "" || !DPCCBuildHistoricalFlatSeed(1000, m_z_account_currency, m_z_account_currency_digits, seed_meta, m_offline_config, m_offline_config.shape, context, row.native_ohlc, m_z_state, m_z_position, reason)) return(Fail(reason == "" ? "BLOCKED:study.action6.account_seed" : reason, reason)); } for(int i = 0; i < 24; i++) context[(int)source_layout.offsets[4] + i] = m_z_state[i]; if(!DPCCBindHistoricalFxReference(m_offline_config.shape, context, m_z_state, reason)) return(Fail(reason, reason)); float step_values[]; ENUM_SYMBOL_CALC_MODE calculation = SYMBOL_CALC_MODE_FOREX; if(!DPCCBuildHistoricalBarStepValues(m_offline_config.shape, context, row.native_ohlc, row.decision_bar, opens, closes, ordinals, row.timeline_opens[0], row.timeline_closes[0], step_values, calculation, reason)) return(Fail(reason, reason)); CNeuronScenarioRolloutOCL *native = dynamic_cast(m_offline_factual.Layer(1)); CBufferFloat prior; SDPCCHistoricalPreparedBar prepared; if(!native || !DPCCPrepareHistoricalBar(*native, step_values, row.decision_bar, (ulong)row.decision_bar + 1, opens[0], closes[0], row.timeline_opens[0], row.timeline_closes[0], calculation, m_z_state, m_z_position, m_z_step, m_z_rows + horizon - 1, m_z_step == 0, prior, prepared, reason, false, clock.TimeDomain())) return(Fail(reason, reason)); //--- Causal Action6 input is Market's current forecast plus prepared Account13. ulong generation = 0; if(!m_forecast.InferReadyMarketScenario(opens[0], GetPointer(m_z_latent), generation, reason)) return(Fail("study.action6.market_infer:" + reason, reason)); CNet *market = m_forecast.ReadyMarketNetwork(); CBufferFloat *predicted_ohlc = m_forecast.ReadyMarketNativeOHLC(); if(CheckPointer(market) == POINTER_INVALID || CheckPointer(predicted_ohlc) == POINTER_INVALID || generation == 0 || predicted_ohlc.GetOpenCL() != m_forecast.ReadyEncoderContext()) return(Fail("study.action6.market_generation", reason)); float account24[24], instrument48[48], limits24[24], execution24[24], event_counts[], calendar[]; for(int i = 0; i < 24; i++) { account24[i] = prepared.state[i]; instrument48[i] = context[(int)source_layout.offsets[5] + i]; instrument48[24 + i] = context[(int)source_layout.offsets[5] + 24 + i]; limits24[i] = context[(int)source_layout.offsets[6] + i]; execution24[i] = context[(int)source_layout.offsets[7] + i]; } //--- Bind the current decision's FX margin reference to the newly observed Ask. instrument48[14] = account24[16]; if(ArrayResize(event_counts, horizon) != horizon || ArrayResize(calendar, horizon * (int)m_offline_config.shape.E * 6) != horizon * ( int)m_offline_config.shape.E * 6) return(Fail("study.action6.condition_capacity", reason)); for(int h = 0; h < horizon; h++) { event_counts[h] = context[(int)source_layout.offsets[8] + h]; const int source = (int)source_layout.offsets[9] + h * (int)m_offline_config.shape.E * 6; for(int j = 0; j < (int)m_offline_config.shape.E * 6; j++) calendar[h * (int)m_offline_config.shape.E * 6 + j] = context[source + j]; } CBufferFloat *action_context = NULL; if(!DPCCBuildDecisionFinancialContext(m_offline_config, predicted_ohlc, account24, instrument48, limits24, execution24, row.timeline_opens[0], row.timeline_opens[0], row.timeline_opens, row.timeline_closes, event_counts, calendar, m_action6_snapshot_work, m_action6_context_work, m_action6_flags_work, m_action6_status_work, action_context, reason) || CheckPointer(action_context) == POINTER_INVALID) return(Fail("study.action6.decision_context:" + reason, reason)); const ulong decision_id = (ulong)row.decision_bar + 1; SDPCCDecisionMeta meta; ZeroMemory(meta); meta.decision_id = decision_id; meta.market_state_id = generation; meta.q1_state_id = decision_id; meta.q2_state_id = decision_id + 1; meta.noise_seed = decision_id; meta.decision_server_time = row.timeline_opens[0]; meta.bar_index = row.decision_bar; meta.mode = (uint)m_offline_config.mode; meta.market_sha256 = m_action6_market_sha; meta.profile_sha256 = m_action6_profile_sha; CNeuronDPCCOCL *actor_top = dynamic_cast(m_action6_actor.Layer(1)); if(!actor_top || !actor_top.BindDecisionContext(action_context, meta)) return(Fail("study.action6.bind_decision_context", reason)); CBufferFloat actor_account; if(!actor_account.BufferInit(13, 0)) return(Fail("study.action6.account_input", reason)); for(int i = 0; i < 13; i++) if(!actor_account.Update(i, prepared.account13[i])) return(Fail("study.action6.account_input_value", reason)); const bool actor_forward = m_action6_actor.feedForward(GetPointer(actor_account), 1, false, market, 6); const bool no_feasible_actor = !actor_top.HasValidDecision() && actor_top.LastStatus() == DPCC_NO_FEASIBLE_CANDIDATE; if(!actor_forward || (!no_feasible_actor && (actor_top.LastStatus() != DPCC_OK || !actor_top.HasValidDecision()))) return(Fail(StringFormat("study.action6.actor_forward.status_%d", (int)actor_top.LastStatus()), reason)); uint selected_candidate = UINT_MAX, plan_candidates = 0, plan_horizon = 0; COpenCLMy *actor_context = actor_top.GetOpenCL(); CBufferFloat *candidate_buffer = NULL; float seed[], candidate_plans[], teacher_plan[], causal_action_plan[], no_trade_plan[]; const ulong plan_stride = (ulong)horizon * 6; if(CheckPointer(actor_context) == POINTER_INVALID || plan_stride == 0 || plan_stride > UINT_MAX) return(Fail("study.action6.actor_plan_context_or_stride", reason)); if(no_feasible_actor) { if(!actor_top.GetCheckedPlans(candidate_buffer) || CheckPointer(candidate_buffer) == POINTER_INVALID || candidate_buffer.Total() <= 0 || (ulong)candidate_buffer.Total() % plan_stride != 0 || (ulong)candidate_buffer.Total() / plan_stride > UINT_MAX) return(Fail("study.action6.actor_fallback_candidates", reason)); plan_candidates = (uint)((ulong)candidate_buffer.Total() / plan_stride); plan_horizon = (uint)horizon; selected_candidate = 0; } else { candidate_buffer = actor_top.GetCandidatePlans(); if(!actor_top.GetSelectedCandidate(selected_candidate) || !actor_top.GetAction6PlanShape(plan_candidates, plan_horizon)) return(Fail("study.action6.actor_plan_selection", reason)); } const ulong selected_offset_elements = (ulong)selected_candidate * plan_stride; if(CheckPointer(candidate_buffer) == POINTER_INVALID || candidate_buffer.GetOpenCL() != actor_context || candidate_buffer.GetIndex() < 0 || plan_horizon != (uint)horizon || plan_candidates == 0 || selected_candidate >= plan_candidates || (ulong)candidate_buffer.Total() != (ulong)plan_candidates * plan_stride || selected_offset_elements > (ulong)UINT_MAX / sizeof(float) || selected_offset_elements + plan_stride > (ulong)candidate_buffer.Total() || ArrayResize(seed, horizon * 6) != horizon * 6 || (!no_feasible_actor && !actor_context.BufferRead(candidate_buffer.GetIndex(), seed, (uint)(selected_offset_elements * sizeof(float)), 0, (uint)plan_stride))) return(Fail("study.action6.actor_plan", reason)); if(no_feasible_actor) ArrayInitialize(seed, 0.0f); for(int i = 0; i < horizon * 6; i++) if(!MathIsValidNumber((double)seed[i])) return(Fail(StringFormat("study.action6.actor_plan_nonfinite_%d", i), reason)); int candidate_count = 0, teacher_source = -1, causal_source = -1; double teacher_score = 0.0, causal_score = 0.0; uchar teacher_mask[], causal_mask[]; CBufferFloat *teacher_continuation = GetPointer(prior); CBufferFloat *causal_continuation = (m_z_step == 0 ? NULL : GetPointer(prior)); if(!DPCCGenerateOracleCandidatePlans(seed, (uint)horizon, m_z_deltas, candidate_plans, candidate_count, reason) || !DPCCSelectHistoricalOracleCandidateFromContinuation(m_offline_scoring, m_offline_config.shape, context, row.native_ohlc, opens[0], row.decision_bar, opens, closes, ordinals, candidate_plans, candidate_count, m_offline_config, 0, teacher_continuation, m_z_step, m_z_rows + horizon - 1, teacher_plan, teacher_score, teacher_source, teacher_mask, reason)) return(Fail("study.action6.factual_oracle:" + reason, reason)); if(no_feasible_actor) { if(ArrayResize(no_trade_plan, horizon * 6) != horizon * 6) return(Fail("study.action6.no_trade_plan_capacity", reason)); ArrayInitialize(no_trade_plan, 0.0f); if(!DPCCSelectHistoricalOracleCandidateFromContinuation(m_offline_scoring, m_offline_config.shape, context, row.native_ohlc, opens[0], row.decision_bar, opens, closes, ordinals, no_trade_plan, 1, m_offline_config, 0, causal_continuation, m_z_step, m_z_rows + horizon - 1, causal_action_plan, causal_score, causal_source, causal_mask, reason, true)) return(Fail("study.action6.no_trade_score:" + reason, reason)); } else if(!DPCCSelectHistoricalOracleCandidateFromContinuation(m_offline_scoring, m_offline_config.shape, context, row.native_ohlc, opens[0], row.decision_bar, opens, closes, ordinals, seed, 1, m_offline_config, 0, causal_continuation, m_z_step, m_z_rows + horizon - 1, causal_action_plan, causal_score, causal_source, causal_mask, reason, true)) return(Fail("study.action6.factual_oracle:" + reason, reason)); float valid_steps[]; if(ArrayResize(valid_steps, horizon) != horizon) return(Fail("study.action6.oracle_mask_capacity", reason)); for(int h = 0; h < horizon; h++) valid_steps[h] = (float)teacher_mask[h]; float account_action19[19]; for(int i = 0; i < 13; i++) account_action19[i] = prepared.account13[i]; float action[6]; for(int i = 0; i < 6; i++) { action[i] = causal_action_plan[i]; account_action19[13 + i] = action[i]; } if(!ArrayResize(out.account13, 13) || !ArrayResize(out.pre_update_plan, horizon * 6) || !ArrayResize(out.executed_action, 6) || !ArrayResize(out.teacher_plan, horizon * 6) || !ArrayResize(out.valid_horizons, horizon) || !ArrayResize(out.state, 33)) return(Fail("study.action6.result_capacity", reason)); ArrayCopy(out.account13, prepared.account13); ArrayCopy(out.pre_update_plan, seed); ArrayCopy(out.executed_action, action); ArrayCopy(out.teacher_plan, teacher_plan); ArrayCopy(out.valid_horizons, teacher_mask); if(!DPCCUpdateAction6Critic(*m_action6_q1, *market, account_action19, (float)(causal_score - (double)prepared.preparation_reward), 1.0f, 1.0f, reason) || !DPCCUpdateAction6Critic(*m_action6_q2, *market, account_action19, (float)(causal_score - (double)prepared.preparation_reward), 1.0f, 1.0f, reason) || !DPCCUpdateAction6OraclePlanActor(*m_action6_actor, decision_id, teacher_plan, valid_steps, m_offline_config.gamma, reason)) return(Fail("study.action6.train_update:" + reason, reason)); if(!m_forecast.ReleaseReadyEncoderZ(generation, reason)) return(Fail("study.action6.market_release:" + reason, reason)); SDPCCHistoricalCommittedBar committed; if(!DPCCCommitHistoricalBar(*native, prepared, action, committed, reason)) return(Fail("study.action6.commit:" + reason, reason)); if(ArrayCopy(m_z_state, committed.state) != 33) return(Fail("study.action6.state_capacity", reason)); m_z_position = committed.position; m_z_floor = committed.floor_reached; if(m_z_first_server_bar == 0) m_z_first_server_bar = opens[0]; m_z_next_server_bar = opens[1]; m_z_last_bar = row.decision_bar; m_z_step++; ArrayCopy(out.state, m_z_state); out.position = m_z_position; out.teacher_score = teacher_score; out.teacher_source = teacher_source; out.reward = committed.reward; out.next_row = m_z_step; out.decision_bar = row.decision_bar; out.committed = true; out.floor_reached = m_z_floor; reason = ""; return(true); } //+------------------------------------------------------------------+ //| InitializeHistory admits Encoder and fresh native policy graphs | //+------------------------------------------------------------------+ bool InitializeHistory(const SDPCCConfig &config, const uint rows, const float &deltas[], const string raw_path, const string raw_sha, const string encoder_root, const string receipt_sha, const bool common_files, string &reason) { if(m_initialized || m_failed || rows == 0 || rows > 16777216 || ArraySize(deltas) != 6 || !DPCCValidateFixedConfig(config.shape, config) || config.shape.K != 21 || config.shape.H != 12 || (config.margin_fx_profile != DPCC_MARGIN_FOREX_SPOT_V1 && config.margin_fx_profile != DPCC_MARGIN_EXTERNAL_V2 && config.margin_fx_profile != DPCC_MARGIN_HISTORICAL_CLOSE_V1) || config.price_basis != DPCC_PRICE_BID || !m_forecast.RestoreEncoderCompleted(raw_path, raw_sha, encoder_root, receipt_sha, reason, common_files)) return(Fail(reason == "" ? "study.history.encoder_or_profile" : reason, reason)); for(int i = 0; i < 6; i++) if(!MathIsValidNumber(deltas[i]) || deltas[i] <= 0 || deltas[i] > 1) return(Fail("study.history.oracle_delta", reason)); SDPCCConfig physical = config; physical.shape.K = 1; physical.shape.M = 1; physical.shape.Depth = 0; COpenCLMy *context = m_forecast.EncoderContext(); if(!m_z_latent.BufferInit(36288, 0) || !m_z_latent.BufferCreate(context)) return(Fail("study.history.latent_buffer", reason)); if(!DPCCCreateHistoricalRolloutNet(m_offline_factual, physical, context, reason)) return(Fail(reason, reason)); if(!DPCCCreateHistoricalRolloutNet(m_offline_scoring, physical, context, reason)) return(Fail(reason, reason)); if(!DPCCCreateZPolicyNet(m_z_actor, config.shape, false, context, reason)) return(Fail(reason, reason)); if(!DPCCCreateZPolicyNet(m_z_q1, config.shape, true, context, reason)) return(Fail(reason, reason)); if(!DPCCCreateZPolicyNet(m_z_q2, config.shape, true, context, reason)) return(Fail(reason, reason)); if(ArrayCopy(m_z_deltas, deltas) != 6) return(Fail("study.history.delta_capacity", reason)); m_offline_config = config; m_z_rows = rows; m_z_raw_sha = raw_sha; m_z_receipt_sha = receipt_sha; m_z_common_files = common_files; m_z_history = true; m_initialized = true; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Bind admitted history and authenticated Market provenance | //+------------------------------------------------------------------+ bool BindHistoryProvenance(const SDPCCOracleHistoryInput &source, string &reason) { if(!m_action6_history) return(Fail("BLOCKED:study.action6.provenance.market_required", reason)); return(BindHistoryProvenance(source, "", "", reason)); } //+------------------------------------------------------------------+ //| Bind history provenance while retaining the legacy Encoder route | //+------------------------------------------------------------------+ bool BindHistoryProvenance(const SDPCCOracleHistoryInput &source, const string encoder_root, const string encoder_receipt_sha, string &reason) { const int source_count = (m_action6_history ? 1 : (source.clock.offline_clock_server_calendar ? 4 : 7)); const int bound_count = source_count + 2; const int financial_index = (m_action6_history ? 0 : 2); string canonical_stem = ""; if(!DPCCCheckpointCanonicalPath(source.output_stem, canonical_stem, reason) || canonical_stem != source.output_stem) return(Fail("BLOCKED:study.history.provenance.output_stem", reason)); if(!m_z_history || !m_initialized || m_failed || m_z_step != 0 || m_z_provenance_bound || ArraySize(source.source_labels) != source_count || ArraySize(source.source_paths) != source_count || ArraySize(source.source_hashes) != source_count || (!m_action6_history && source.raw_sha != m_z_raw_sha) || (!m_action6_history && encoder_receipt_sha != m_z_receipt_sha) || ArraySize(source.decision_bars) != (int)m_z_rows || !DPCCResolverLowerSha256(source.execution_sha)) return(Fail("BLOCKED:study.history.provenance.admission", reason)); //--- Keep the 24-slot financial schema; exposure cap is inactive for direct lots. string financial_sha = ""; if(source.financial_profile.limits[0] != 0 || source.financial_profile.limits[20] != 8 || source.financial_profile.limits[3] != 50 || source.source_labels[financial_index] != "derived_financial_profile" || source.financial_profile.account_currency == "" || source.financial_profile.account_currency_digits < 0 || source.financial_profile.account_currency_digits > 8 || source.financial_profile.sha256 != source.execution_sha || source.source_hashes[financial_index] != source.financial_profile.sha256 || !DPCCCheckpointTextSHA(source.financial_profile.payload, financial_sha, reason) || financial_sha != source.financial_profile.sha256) return(Fail("BLOCKED:study.history.provenance.direct_lots_financial_profile", reason)); string config_sha = "", deltas_sha = ""; if(!DPCCOracleHistoryConfigSHA(source.config, config_sha, reason) || config_sha != source.config_sha || !DPCCOracleHistoryDeltasSHA(m_z_deltas, deltas_sha, reason) || deltas_sha != source.deltas_sha) return(Fail("BLOCKED:study.history.provenance.config_or_deltas_changed", reason)); if(ArrayResize(m_z_source_labels, bound_count) != bound_count || ArrayResize(m_z_source_paths, bound_count) != bound_count || ArrayResize(m_z_source_hashes, bound_count) != bound_count) return(Fail("BLOCKED:study.history.provenance.capacity", reason)); for(int i = 0; i < source_count; i++) { m_z_source_labels[i] = source.source_labels[i]; m_z_source_paths[i] = source.source_paths[i]; m_z_source_hashes[i] = source.source_hashes[i]; } if(m_action6_history) { m_z_source_labels[source_count] = "trained_market_model"; m_z_source_paths[source_count] = ""; m_z_source_hashes[source_count] = m_action6_market_sha; m_z_source_labels[source_count + 1] = "assigned_market_profile"; m_z_source_paths[source_count + 1] = ""; m_z_source_hashes[source_count + 1] = m_action6_profile_sha; } else { m_z_source_labels[source_count] = "encoder_receipt"; m_z_source_paths[source_count] = encoder_root + "/EncoderTeacher.study.txt"; m_z_source_hashes[source_count] = encoder_receipt_sha; m_z_source_labels[source_count + 1] = "trained_encoder_model"; m_z_source_paths[source_count + 1] = encoder_root + "/Encoder.trained.nnw"; SDPCCStudyEncoderReceipt receipt; if(!DPCCReadStudyEncoderReceipt(encoder_root, encoder_receipt_sha, m_z_raw_sha, m_forecast.FitCount(), m_forecast.HoldoutCount(), receipt, reason, m_z_common_files)) return(Fail("BLOCKED:study.history.provenance.encoder_receipt:" + reason, reason)); m_z_source_hashes[source_count + 1] = receipt.encoder_sha256; } for(int i = 0; i < bound_count; i++) { string observed = ""; if(!DPCCResolverLowerSha256(m_z_source_hashes[i])) return(Fail("BLOCKED:study.history.provenance.source_hash." + m_z_source_labels[i], reason)); if(m_action6_history && i == source_count) { if(!DPCCCheckpointTextSHA(source.model_text, observed, reason) || observed != source.model_sha) return(Fail("BLOCKED:study.history.provenance.source_changed.action6_runtime:" + reason, reason)); m_z_model_text = source.model_text; m_z_model_sha = source.model_sha; } else if(m_action6_history && i == source_count + 1) { if(!DPCCCheckpointTextSHA(m_action6_profile_text, observed, reason) || observed != m_action6_profile_sha || observed != m_z_source_hashes[i]) return(Fail("BLOCKED:study.history.provenance.source_changed." + m_z_source_labels[i] + ":" + reason, reason)); } else { if(!DPCCCheckpointFileSHA(m_z_source_paths[i], m_z_common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, observed, reason) || observed != m_z_source_hashes[i]) return(Fail("BLOCKED:study.history.provenance.source_changed." + m_z_source_labels[i] + ":" + reason, reason)); } } //--- The restore path has authenticated the selected model and its assigned profile. m_z_time_domain = source.financial_profile.time_domain; if(m_z_time_domain != "UTC_SECONDS_V1" && m_z_time_domain != "SERVER_CALENDAR_SECONDS_V1") return(Fail("BLOCKED:study.history.provenance.time_domain", reason)); m_z_account_currency = source.financial_profile.account_currency; m_z_account_currency_digits = (uint)source.financial_profile.account_currency_digits; m_z_execution_sha = source.execution_sha; m_z_config_sha = config_sha; m_z_deltas_sha = deltas_sha; m_z_output_stem = source.output_stem; m_z_provenance_bound = true; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Admit only the final compact V7 policy for bounded replay | //+------------------------------------------------------------------+ bool RestoreHistoryModels(const SDPCCOracleHistoryInput &source, const string stem, string &reason) { if(!m_z_history || !m_initialized || m_failed || !m_z_provenance_bound || m_z_step != 0 || stem != source.output_stem) return(Fail("study.history.restore_boundary", reason)); string canonical = ""; if(!DPCCCheckpointCanonicalPath(stem, canonical, reason) || canonical != stem) return(Fail("study.history.restore_stem", reason)); uint newest = 0; const int common_flag = (m_z_common_files ? FILE_COMMON : 0); const bool final_latest = FileIsExist(stem + ".v7.latest.manifest", common_flag); if(!final_latest) { if(FileIsExist(stem + ".v7.latest.actor.dppt", common_flag) || FileIsExist(stem + ".v7.latest.q1.dppt", common_flag) || FileIsExist(stem + ".v7.latest.q2.dppt", common_flag)) return(Fail("study.history.restore_latest_orphan_without_manifest", reason)); //--- Per-row update snapshots predate final-only persistence. //--- They are intentionally ignored; start with fresh CNet weights. reason = ""; return(true); } const string prefix = stem + ".v7.latest", receipt = prefix + ".manifest"; string receipt_sha = "", wire = ""; if(!DPCCCheckpointFileSHA(receipt, m_z_common_files, 65536, receipt_sha, reason) || !DPCCZReadPinnedText(receipt, receipt_sha, wire, reason, m_z_common_files)) return(Fail("study.history.restore_manifest_authentication:" + reason, reason)); string paths[], hashes[]; bool compact = false; SDPCCStudyEncoderReceipt enc; const int last = ArraySize(m_z_source_paths) - 1; const string root = StringSubstr(m_z_source_paths[last], 0, StringLen(m_z_source_paths[last]) - StringLen( "/Encoder.trained.nnw")); if(!DPCCReadStudyEncoderReceipt(root, m_z_receipt_sha, m_z_raw_sha, m_forecast.FitCount(), m_forecast.HoldoutCount(), enc, reason, m_z_common_files) || !DPCCZReadStudyModelsEx(receipt, receipt_sha, m_z_raw_sha, m_z_receipt_sha, enc.encoder_sha256, paths, hashes, compact, reason, m_z_common_files) || !compact || ArraySize(paths) != 3 || ArraySize(hashes) != 3) return(Fail("study.history.restore_manifest_decode:" + reason, reason)); const string expected[3] = {prefix + ".actor.dppt", prefix + ".q1.dppt", prefix + ".q2.dppt"}; for(int i = 0; i < 3; i++) if(paths[i] != expected[i]) return(Fail("study.history.restore_model_path", reason)); string fields[]; int pos = StringLen("DPCC_Z_ORACLE_MODELS_V7\n"), count = 0; while(count < 34 && StringSubstr(wire, pos, 4) != "COMP") { string f = ""; if(!DPCCZReadField(wire, pos, f, reason)) return(Fail("study.history.restore_fields", reason)); if(ArrayResize(fields, count + 1) != count + 1) return(Fail("study.history.restore_field_capacity", reason)); fields[count++] = f; } bool cursor = false, rows = false, bar = false, execution = false, config = false, deltas = false; bool domain = (m_z_time_domain != "SERVER_CALENDAR_SECONDS_V1"); string old_profile_path = "", old_profile_sha = "", old_execution_sha = ""; int old_profile_path_count = 0, old_profile_sha_count = 0, old_execution_count = 0; for(int i = 0; i < count; i++) { if(StringFind(fields[i], "cursor=") == 0) { ulong cursor_value = 0; if(!DPCCCheckpointReadUnsigned(StringSubstr(fields[i], StringLen("cursor=")), m_z_rows, cursor_value) || cursor_value == 0) return(Fail("study.history.restore_cursor", reason)); newest = (uint)cursor_value; cursor = true; } if(fields[i] == "row_count=" + (string)m_z_rows) rows = true; if(fields[i] == "decision_bar=" + (string)source.decision_bars[(int)newest - 1]) bar = true; if(fields[i] == "execution_profile.sha256=" + m_z_execution_sha) execution = true; if(StringFind(fields[i], "execution_profile.sha256=") == 0) {old_execution_sha = StringSubstr(fields[i], StringLen("execution_profile.sha256=")); old_execution_count++;} if(fields[i] == "config.sha256=" + m_z_config_sha) config = true; if(fields[i] == "oracle_deltas.sha256=" + m_z_deltas_sha) deltas = true; if(fields[i] == "time_domain=" + m_z_time_domain) domain = true; if(StringFind(fields[i], "derived_financial_profile.path=") == 0) {old_profile_path = StringSubstr(fields[i], StringLen("derived_financial_profile.path=")); old_profile_path_count++;} if(StringFind(fields[i], "derived_financial_profile.sha256=") == 0) {old_profile_sha = StringSubstr(fields[i], StringLen("derived_financial_profile.sha256=")); old_profile_sha_count++;} } bool profile_exception = false; const int financial_index = (m_action6_history ? 1 : 2); if(old_profile_path_count == 1 && old_profile_sha_count == 1 && old_execution_count == 1 && DPCCResolverLowerSha256(old_profile_sha) && old_execution_sha == old_profile_sha && m_z_source_labels[financial_index] == "derived_financial_profile" && (old_profile_path != m_z_source_paths[financial_index] || old_profile_sha != m_z_source_hashes[financial_index])) { string canonical_profile = "", observed_profile = "", old_payload = ""; uchar profile_bytes[]; const string required_old_path = stem + ".financial_profile." + old_profile_sha + ".txt"; string text_sha = ""; if(old_profile_path == required_old_path && DPCCCheckpointCanonicalPath(old_profile_path, canonical_profile, reason) && canonical_profile == old_profile_path && DPCCCheckpointFileSHA(old_profile_path, m_z_common_files, 1048576, observed_profile, reason) && observed_profile == old_profile_sha && DPCCReadArtifactBytes(old_profile_path, m_z_common_files, 1048576, profile_bytes, reason) && DPCCStrictUtf8Decode(profile_bytes, ArraySize(profile_bytes), 1048576, old_payload, reason) && DPCCCheckpointTextSHA(old_payload, text_sha, reason) && text_sha == old_profile_sha && SameFinancialProfileExceptSnapshotTimesAndTerminalBuild(old_payload, source.financial_profile.payload, reason)) profile_exception = true; } for(int i = 0; i < ArraySize(m_z_source_labels); i++) { bool pp = false, hh = false; for(int j = 0; j < count; j++) { if(fields[j] == m_z_source_labels[i] + ".path=" + m_z_source_paths[i]) pp = true; if(fields[j] == m_z_source_labels[i] + ".sha256=" + m_z_source_hashes[i]) hh = true; } if(!pp || !hh) { if(i != 2 || !profile_exception) return(Fail("study.history.restore_source_pin." + m_z_source_labels[i] + ":" + reason, reason)); } } const int expected_fields = (m_z_time_domain == "SERVER_CALENDAR_SECONDS_V1" ? 28 : 33); //--- Accept the prior receipt layout with its extra clock_identity field. if(!cursor || !rows || !bar || !config || !deltas || !domain || newest == 0 || newest > m_z_rows || old_profile_path_count != 1 || old_profile_sha_count != 1 || old_execution_count != 1 || (!execution && !profile_exception) || (count != expected_fields && count != expected_fields + 1)) return(Fail("study.history.restore_provenance", reason)); CNet *nets[] = {GetPointer(m_z_actor), GetPointer(m_z_q1), GetPointer(m_z_q2)}; for(int i = 0; i < 3; i++) if(!DPCCLoadCompactPolicyTrainingState(*nets[i], paths[i], hashes[i], m_z_common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, reason)) return(Fail("study.history.restore_state:" + reason, reason)); for(int i = 0; i < 3; i++) if(!nets[i].Clear() || !nets[i].TrainMode(true) || !nets[i].SetWeightsUpdate(true)) return(Fail("study.history.restore_transient", reason)); m_z_resume_cursor = newest; m_z_resume_manifest = receipt; m_z_resume_sha = receipt_sha; reason = ""; return(true); } bool ReplayingHistory(void) { return(m_z_resume_cursor > 0 && m_z_step > 0 && m_z_step <= m_z_resume_cursor); } uint ResumeCursor(void) { return(m_z_resume_cursor); } //+------------------------------------------------------------------+ //| StepHistory trains on pinned history and commits one operation | //+------------------------------------------------------------------+ bool StepHistory(CDPCCMarketClock &clock, const long decision_bar, const float &admitted_context[], const float &native_ohlc[], const long &timeline_opens[], const long &timeline_closes[], SDPCCOfflineRowResult &out, string &reason) { DPCCClearOfflineRowResult(out); const int horizon = (int)m_offline_config.shape.H; if(!m_z_history || !m_initialized || m_failed || !m_z_provenance_bound || m_z_floor || m_z_step >= m_z_rows || clock.TimeDomain() != m_z_time_domain || decision_bar < 0 || (m_z_step > 0 && decision_bar != m_z_last_bar + 1) || ArraySize(native_ohlc) != horizon * 4 || ArraySize(timeline_opens) != horizon || ArraySize(timeline_closes) != horizon) return(Fail("study.history.row_boundary", reason)); long opens[], closes[], ordinals[]; if(ArrayResize(opens, horizon) != horizon || ArrayResize(closes, horizon) != horizon || ArrayResize(ordinals, horizon) != horizon) return(Fail("study.history.time_capacity", reason)); for(int h = 0; h < horizon; h++) { SDPCCClockBar bar; long open_time = 0, end = 0; if(!clock.IndexedTimelineBar(decision_bar + h, bar, open_time, end, reason) || bar.ordinal != decision_bar + h || open_time != timeline_opens[h] || end != timeline_closes[h]) return(Fail("study.history.time_join:" + reason, reason)); opens[h] = bar.server_open; closes[h] = bar.server_close; ordinals[h] = bar.ordinal; } int raw_index = -1; if(!m_forecast.RawIndexForServerOpen(opens[0], true, raw_index, reason) || !m_forecast.MatchesNativeOHLC(raw_index, native_ohlc, opens, reason)) return(Fail("study.history.raw_horizon_join:" + reason, reason)); if(!m_forecast.InferZToDevice(raw_index, GetPointer(m_z_latent), reason)) return(Fail("study.history.encoder_inference:" + reason, reason)); //--- The native simulator owns the continuous account and position age. float context[]; SDPCCLayout layout, unused; uint tokens = 0; if(!DPCCCompositeLayouts(m_offline_config.shape, layout, unused, tokens) || ArraySize(admitted_context) != (int)layout.offsets[10] || ArrayCopy(context, admitted_context) != ArraySize(admitted_context)) return(Fail("study.history.financial_context_shape", reason)); if(context[(int)layout.offsets[6]] != 0 || context[(int)layout.offsets[6] + 20] != 8 || context[(int)layout.offsets[6] + 3] != 50) return(Fail("BLOCKED:study.history.row_direct_lots_financial_profile_mismatch", reason)); if(m_z_step == 0) { SDPCCSnapshotMeta meta; ZeroMemory(meta); meta.account_currency = m_z_account_currency; if(m_z_account_currency == "" || !DPCCBuildHistoricalFlatSeed(1000, m_z_account_currency, m_z_account_currency_digits, meta, m_offline_config, m_offline_config.shape, context, native_ohlc, m_z_state, m_z_position, reason)) return(Fail(reason, reason)); } for(int i = 0; i < 24; i++) context[(int)layout.offsets[4] + i] = m_z_state[i]; if(!DPCCBindHistoricalFxReference(m_offline_config.shape, context, m_z_state, reason)) return(Fail(reason, reason)); float step_values[]; ENUM_SYMBOL_CALC_MODE calculation = SYMBOL_CALC_MODE_FOREX; if(!DPCCBuildHistoricalBarStepValues(m_offline_config.shape, context, native_ohlc, decision_bar, opens, closes, ordinals, timeline_opens[0], timeline_closes[0], step_values, calculation, reason)) return(Fail(reason, reason)); CNeuronScenarioRolloutOCL *native = dynamic_cast(m_offline_factual.Layer(1)); CBufferFloat prior; SDPCCHistoricalPreparedBar prepared; if(!native || !DPCCPrepareHistoricalBar(*native, step_values, decision_bar, (ulong)decision_bar + 1, opens[0], closes[0], timeline_opens[0], timeline_closes[0], calculation, m_z_state, m_z_position, m_z_step, m_z_rows + horizon - 1, m_z_step == 0, prior, prepared, reason, false, clock.TimeDomain())) return(Fail(reason, reason)); if(m_z_step < m_z_resume_cursor) { float empty_action[], plan[], prepared_plan[]; if(!DPCCForwardZPolicy(m_z_actor, m_offline_config.shape, GetPointer(m_z_latent), prepared.account13, empty_action, false, reason)) return(Fail("study.history.replay_forward:" + reason, reason)); m_z_actor.getResults(plan); if(ArraySize(plan) != horizon * 6 || !DPCCPrepareHistoricalPolicyActions(m_offline_config.shape, context, plan, prepared_plan, reason)) return(Fail("study.history.replay_plan:" + reason, reason)); float action[6]; for(int i = 0; i < 6; i++) action[i] = prepared_plan[i]; if(ArrayResize(out.account13, 13) != 13 || ArrayResize(out.pre_update_plan, horizon * 6) != horizon * 6 || ArrayResize(out.executed_action, 6) != 6 || ArrayResize(out.teacher_plan, horizon * 6) != horizon * 6 || ArrayResize(out.valid_horizons, horizon) != horizon || ArrayResize(out.state, 33) != 33) return(Fail("study.history.replay_result_capacity", reason)); SDPCCHistoricalCommittedBar replay; if(!DPCCCommitHistoricalBar(*native, prepared, action, replay, reason)) return(Fail("study.history.replay_commit:" + reason, reason)); if(ArrayCopy(m_z_state, replay.state) != 33) return(Fail("study.history.replay_state_capacity", reason)); m_z_position = replay.position; m_z_floor = replay.floor_reached; m_z_last_bar = decision_bar; m_z_step++; ArrayInitialize(out.teacher_plan, 0.0); ArrayInitialize(out.valid_horizons, 0.0); ArrayCopy(out.account13, prepared.account13); ArrayCopy(out.pre_update_plan, plan); ArrayCopy(out.executed_action, action); ArrayCopy(out.state, m_z_state); out.position = m_z_position; out.reward = replay.reward; out.next_row = m_z_step; out.decision_bar = decision_bar; out.committed = true; out.floor_reached = m_z_floor; reason = ""; return(true); } //--- Preserve the causal Actor plan before hindsight targets or updates. float empty_action[], seed[], candidates[], teacher[], seed_selected[]; if(!DPCCForwardZPolicy(m_z_actor, m_offline_config.shape, GetPointer(m_z_latent), prepared.account13, empty_action, false, reason)) return(Fail(reason, reason)); m_z_actor.getResults(seed); int count = 0, teacher_source = -1, seed_source = -1; double teacher_score = 0, seed_score = 0; uchar teacher_mask[], seed_mask[]; //--- Full plans use the raw seed; row-zero operation scoring initializes natively. CBufferFloat *teacher_continuation = GetPointer(prior); CBufferFloat *operation_continuation = (m_z_step == 0 ? NULL : GetPointer(prior)); if(!DPCCGenerateOracleCandidatePlans(seed, horizon, m_z_deltas, candidates, count, reason)) return(Fail(reason, reason)); if(!DPCCSelectHistoricalOracleCandidateFromContinuation(m_offline_scoring, m_offline_config.shape, context, native_ohlc, opens[0], decision_bar, opens, closes, ordinals, candidates, count, m_offline_config, 0, teacher_continuation, m_z_step, m_z_rows + horizon - 1, teacher, teacher_score, teacher_source, teacher_mask, reason)) return(Fail(reason, reason)); if(!DPCCSelectHistoricalOracleCandidateFromContinuation(m_offline_scoring, m_offline_config.shape, context, native_ohlc, opens[0], decision_bar, opens, closes, ordinals, seed, 1, m_offline_config, 0, operation_continuation, m_z_step, m_z_rows + horizon - 1, seed_selected, seed_score, seed_source, seed_mask, reason, true)) return(Fail(reason, reason)); float action[6], mask[]; if(ArrayResize(mask, horizon) != horizon) return(Fail("study.history.mask_capacity", reason)); //--- Operation-only scoring returns the exact capped seed used for Critic and commit. for(int i = 0; i < 6; i++) action[i] = seed_selected[i]; for(int h = 0; h < horizon; h++) mask[h] = (float)teacher_mask[h]; //--- Reserve publication buffers before any model or financial update. if(ArrayResize(out.account13, 13) != 13 || ArrayResize(out.pre_update_plan, horizon * 6) != horizon * 6 || ArrayResize(out.executed_action, 6) != 6 || ArrayResize(out.teacher_plan, horizon * 6) != horizon * 6 || ArrayResize(out.valid_horizons, horizon) != horizon || ArrayResize(out.state, 33) != 33) return(Fail("study.history.result_capacity", reason)); if(!DPCCPrepareDiscountedPlanGradient(m_z_actor, m_offline_config.shape, 3, teacher, mask, m_offline_config.gamma, reason)) return(Fail(reason, reason)); CBufferFloat detached_gradient; if(!detached_gradient.BufferInit(36288, 0) || !detached_gradient.BufferCreate(m_forecast.EncoderContext()) || !m_z_actor.backPropGradient(GetPointer(m_z_latent), GetPointer(detached_gradient), -1, true)) return(Fail("study.history.actor_backward", reason)); CNet *critics[] = {GetPointer(m_z_q1), GetPointer(m_z_q2)}; //--- The open-preparation PnL precedes the current action; retain it in the ledger, //--- but exclude it from the action-conditioned Critic target. const double critic_target_candidate = seed_score - (double)prepared.preparation_reward; const float critic_target = (float)critic_target_candidate; if(!MathIsValidNumber(critic_target_candidate) || !MathIsValidNumber((double)critic_target)) return(Fail("study.history.critic_target_nonfinite", reason)); for(int i = 0; i < 2; i++) { if(!DPCCForwardZPolicy(*critics[i], m_offline_config.shape, GetPointer(m_z_latent), prepared.account13, action, true, reason)) return(Fail(reason, reason)); if(!DPCCUpdateZCritic(*critics[i], m_offline_config.shape, GetPointer(m_z_latent), critic_target, 1, 1, reason)) return(Fail(reason, reason)); } SDPCCHistoricalCommittedBar committed; if(!DPCCCommitHistoricalBar(*native, prepared, action, committed, reason)) return(Fail(reason, reason)); if(ArrayCopy(m_z_state, committed.state) != 33) return(Fail("study.history.state_capacity", reason)); m_z_position = committed.position; m_z_floor = committed.floor_reached; m_z_last_bar = decision_bar; m_z_step++; ArrayCopy(out.account13, prepared.account13); ArrayCopy(out.pre_update_plan, seed); ArrayCopy(out.executed_action, action); ArrayCopy(out.teacher_plan, teacher); ArrayCopy(out.valid_horizons, teacher_mask); ArrayCopy(out.state, m_z_state); out.position = m_z_position; out.teacher_score = teacher_score; out.teacher_source = teacher_source; out.reward = committed.reward; out.next_row = m_z_step; out.decision_bar = decision_bar; out.committed = true; out.floor_reached = m_z_floor; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Persist only latest Actor/Critic weights and Adam moments | //+------------------------------------------------------------------+ bool SaveAction6TrainingStateOnly(const string base, const bool common_files, string &manifest_path, string &manifest_sha, string &reason) { manifest_path = ""; manifest_sha = ""; reason = ""; string actor_path = "", q1_path = "", q2_path = ""; if(!m_action6_history || !m_initialized || CheckPointer(m_action6_actor) == POINTER_INVALID || CheckPointer(m_action6_q1) == POINTER_INVALID || CheckPointer(m_action6_q2) == POINTER_INVALID || !Action6TrainingPaths(base, common_files, actor_path, q1_path, q2_path, manifest_path, reason)) { if(reason == "") reason = "study.action6.training_state.boundary"; return(false); } string hashes[3]; if(!Action6SavePolicyState(m_action6_actor, actor_path, common_files, hashes[0], reason) || !Action6SavePolicyState(m_action6_q1, q1_path, common_files, hashes[1], reason) || !Action6SavePolicyState(m_action6_q2, q2_path, common_files, hashes[2], reason)) return(false); const string payload = "DPCC_ACTION6_TRAINING_STATE_V1\n" + hashes[0] + "\n" + hashes[1] + "\n" + hashes[2] + "\n" + "END_DPCC_ACTION6_TRAINING_STATE_V1\n"; if(!DPCCCheckpointWriteText(manifest_path, common_files, payload, reason, true)) return(false); uchar bytes[]; string readback = "", actual_sha = ""; if(!DPCCReadArtifactBytes(manifest_path, common_files, 4096, bytes, reason) || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 4096, readback, reason) || readback != payload || !DPCCCheckpointFileSHA(manifest_path, common_files, 4096, actual_sha, reason)) { if(reason == "") reason = "study.action6.training_state.manifest_readback"; return(false); } manifest_sha = actual_sha; return(true); } //+------------------------------------------------------------------+ //| Restore only Actor/Critic weights and optimizer moments | //+------------------------------------------------------------------+ bool LoadAction6TrainingStateOnly(const string base, const bool common_files, string &manifest_path, string &reason) { manifest_path = ""; reason = ""; string actor_path = "", q1_path = "", q2_path = ""; if(!m_action6_history || !m_initialized || CheckPointer(m_action6_actor) == POINTER_INVALID || CheckPointer(m_action6_q1) == POINTER_INVALID || CheckPointer(m_action6_q2) == POINTER_INVALID || !Action6TrainingPaths(base, common_files, actor_path, q1_path, q2_path, manifest_path, reason)) { if(reason == "") reason = "study.action6.training_state.boundary"; return(false); } uchar bytes[]; string source = ""; if(!DPCCReadArtifactBytes(manifest_path, common_files, 4096, bytes, reason) || !DPCCStrictUtf8Decode(bytes, ArraySize(bytes), 4096, source, reason)) return(false); string expected[3], paths[3], actual = ""; if(!Action6ParseTrainingManifest(source, expected[0], expected[1], expected[2], reason)) return(false); paths[0] = actor_path; paths[1] = q1_path; paths[2] = q2_path; for(int i = 0; i < ArraySize(paths); i++) if(!DPCCCheckpointFileSHA(paths[i], common_files, 536870912, actual, reason) || actual != expected[i]) { reason = "study.action6.training_state.model_sha_mismatch"; return(false); } CNeuronDPCCOCL *action6_actor_top = dynamic_cast(m_action6_actor.Layer(1)); CNet *market = m_forecast.ReadyMarketNetwork(); if(!Action6LoadPolicyState(m_action6_q1, q1_path, common_files, reason)) { reason = "study.action6.training_state.q1:" + reason; return(false); } if(!Action6LoadPolicyState(m_action6_q2, q2_path, common_files, reason)) { reason = "study.action6.training_state.q2:" + reason; return(false); } if(!Action6LoadPolicyState(m_action6_actor, actor_path, common_files, reason)) { reason = "study.action6.training_state.actor:" + reason; return(false); } if(CheckPointer(action6_actor_top) == POINTER_INVALID || !action6_actor_top.BindPolicyNetworks(market, m_action6_q1, m_action6_q2)) { reason = "study.action6.training_state.actor_bind"; return(false); } return(true); } //+--------------------------------------------------------------------+ //| Publish the D01 native CNet Trading role with synchronized targets | //+--------------------------------------------------------------------+ bool SaveAction6HistoryModels(const ulong total_policy_updates, const ulong pass_index, string &manifest, string &sha, string &reason) { manifest = ""; sha = ""; string canonical_stem = ""; if(total_policy_updates == 0 || !m_action6_history || !m_z_history || !m_initialized || m_failed || !m_z_provenance_bound || m_z_step == 0 || m_z_last_bar < 0 || m_z_first_server_bar <= 0 || m_z_next_server_bar <= 0 || m_action6_task_id == "" || CheckPointer(m_action6_actor) == POINTER_INVALID || CheckPointer(m_action6_q1) == POINTER_INVALID || CheckPointer(m_action6_q2) == POINTER_INVALID || !DPCCCheckpointCanonicalPath(m_z_output_stem, canonical_stem, reason) || canonical_stem != m_z_output_stem) return(Fail("BLOCKED:study.action6.export_boundary:" + reason, reason)); //--- The caller finalizes after the configured update limit or the floor boundary. //--- The resume cursor describes the latest pass; policy_updates is cumulative. CNeuronDPCCOCL *actor_top = dynamic_cast(m_action6_actor.Layer(1)); if(!actor_top || actor_top.Type() != defNeuronDPCCOCL || actor_top.ActorABI() != 3 || (actor_top.LastStatus() != DPCC_OK && actor_top.LastStatus() != DPCC_NO_FEASIBLE_CANDIDATE) || m_action6_market_max_bytes == 0) return(Fail("BLOCKED:study.action6.export_actor_contract", reason)); string config_sha = "", deltas_sha = "", actual = ""; if(!DPCCOracleHistoryConfigSHA(m_offline_config, config_sha, reason) || config_sha != m_z_config_sha || !DPCCOracleHistoryDeltasSHA(m_z_deltas, deltas_sha, reason) || deltas_sha != m_z_deltas_sha || !DPCCCheckpointFileSHA(m_action6_market_path, m_z_common_files, m_action6_market_max_bytes, actual, reason) || actual != m_action6_market_sha || !DPCCCheckpointTextSHA(m_action6_profile_text, actual, reason) || actual != m_action6_profile_sha) return(Fail("BLOCKED:study.action6.export.provenance_changed:" + reason, reason)); for(int i = 0; i < ArraySize(m_z_source_labels) - 2; i++) { ulong limit = DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES; if(!DPCCCheckpointFileSHA(m_z_source_paths[i], m_z_common_files, limit, actual, reason) || actual != m_z_source_hashes[i]) return(Fail("BLOCKED:study.action6.export.source_changed." + m_z_source_labels[i] + ":" + reason, reason)); } CNet *market = m_forecast.ReadyMarketNetwork(); CNet *target_q1 = NULL, *target_q2 = NULL, *target_actor = NULL; CNeuronDPCCOCL *target_actor_top = NULL; bool saved = false; string output_path = "", output_sha = ""; do { if(CheckPointer(market) == POINTER_INVALID || market.GetOpenCL() != m_action6_actor.GetOpenCL() || market.WeightsUpdateEnabled() || !DPCCCreateAction6CriticNet(target_q1, market, reason) || !DPCCCreateAction6CriticNet(target_q2, market, reason) || !DPCCCreateAction6ActorNet(target_actor, m_action6_runtime, market, target_q1, target_q2, target_actor_top, reason) || !DPCCSyncAction6TradingTargets(*market, *m_action6_actor, *m_action6_q1, *m_action6_q2, *target_actor, *target_q1, *target_q2, reason)) { if(reason == "") reason = "study.action6.export.target_graph"; break; } string actor_graph = "", q1_graph = "", q2_graph = "", target_actor_graph = "", target_q1_graph = "", target_q2_graph = ""; if(!DPCCAPTradingGraph(*m_action6_actor, 2, actor_graph, reason) || !DPCCAPTradingGraph(*m_action6_q1, 5, q1_graph, reason) || !DPCCAPTradingGraph(*m_action6_q2, 5, q2_graph, reason) || !DPCCAPTradingGraph(*target_actor, 2, target_actor_graph, reason) || !DPCCAPTradingGraph(*target_q1, 5, target_q1_graph, reason) || !DPCCAPTradingGraph(*target_q2, 5, target_q2_graph, reason)) break; const string graph = "{\"Actor\":" + actor_graph + ",\"Q1\":" + q1_graph + ",\"Q2\":" + q2_graph + ",\"TargetActor\":" + target_actor_graph + ",\"TargetQ1\":" + target_q1_graph + ",\"TargetQ2\":" + target_q2_graph + "}"; string source_abi = ""; if(!DPCCCheckpointTextSHA(DPCC_AP_CONTRACT + "\n" + DPCC_TRADING_REVISION + "\nTRADING_NATIVE_CNET;SIX_ROLES;MARKET_DEPENDENCY_SHA_V1\n" + m_action6_market_sha + "\n" + graph, source_abi, reason)) break; uchar financial_bytes[]; string profile_text = m_action6_profile_text, model_text = m_z_model_text, financial_text = ""; string profile_sha = "", model_sha = "", financial_sha = ""; if(!DPCCCheckpointTextSHA(profile_text, profile_sha, reason) || profile_sha != m_action6_profile_sha || !DPCCCheckpointTextSHA(model_text, model_sha, reason) || model_sha != m_z_model_sha || !DPCCReadArtifactBytes(m_z_source_paths[0], m_z_common_files, 1048576, financial_bytes, reason) || !DPCCSHA256Lower(financial_bytes, financial_sha, reason) || financial_sha != m_z_source_hashes[0] || !DPCCStrictUtf8Decode(financial_bytes, ArraySize(financial_bytes), 1048576, financial_text, reason)) break; CDPCCJsonDocument profile_document, model_document; if(!profile_document.Parse(profile_text, reason) || !model_document.Parse(model_text, reason)) break; const string manifest_text = "{\"contract_id\":" + DPCCAPJSONString(DPCC_AP_CONTRACT) + ",\"revision\":" + DPCCAPJSONString(DPCC_TRADING_REVISION) + ",\"role\":\"Trading\",\"task_id\":" + DPCCAPJSONString(m_action6_task_id) + ",\"stage\":\"Study\",\"graph\":" + graph + ",\"source_archive_sha256\":\"2ed3f91b038394980f73d566a4c0c8c2fb53f651898d2d101b914811d28de3c8\"" + ",\"source_abi_sha256\":\"" + source_abi + "\",\"market_dependency_sha256\":\"" + m_action6_market_sha + "\",\"data_sha256\":\"" + m_action6_market_data_sha + "\",\"data_index_sha256\":null" + ",\"data_source\":{\"path\":" + DPCCAPJSONString(m_z_raw_path) + ",\"sha256\":null},\"study_run_sha256\":\"" + m_z_source_hashes[0] + "\",\"study_mode\":" + (string)m_offline_config.mode + ",\"model\":{\"sha256\":\"" + model_sha + "\",\"contents\":" + DPCCAPJSONString( model_text) + "}" + ",\"financial_profile\":{\"sha256\":\"" + financial_sha + "\",\"payload\":" + DPCCAPJSONString(financial_text) + "},\"profile\":{\"task_id\":" + DPCCAPJSONString(m_action6_task_id) + ",\"sha256\":\"" + m_action6_profile_sha + "\",\"contents\":" + profile_text + "},\"ready\":false,\"acceptance\":[{\"case_id\":\"STUDY_TRAINING\",\"status\":\"NOT_RUN\",\"evidence_sha256\":null}]}"; string record_order_sha = ""; if(!DPCCCheckpointTextSHA("CHRONOLOGICAL_TERMINAL_BARS_V1\n" + (string)m_z_first_server_bar + "\n" + (string)m_z_next_server_bar + "\n" + (string)m_z_rows + "\n" + (string)m_z_step, record_order_sha, reason)) break; const string resume = "{\"stage\":\"Study\",\"epoch\":" + (string)pass_index + ",\"next_record\":" + (string)m_z_step + ",\"record_order_sha256\":\"" + record_order_sha + "\",\"completed_transitions\":" + (string)m_z_step + ",\"policy_updates\":" + (string)total_policy_updates + ",\"target_updates\":1,\"target_order_flag\":false,\"rng_state\":[],\"noise_seed\":0," + "\"profile_sha256\":\"" + m_action6_profile_sha + "\",\"training_complete\":true," + "\"row_count\":" + (string)m_z_rows + ",\"floor_reached\":" + (m_z_floor ? "true" : "false") + ",\"study_run_sha256\":\"" + m_z_source_hashes[0] + "\",\"journal_path\":null,\"journal_sha256\":null," + "\"replay_from_bar\":" + (string)m_z_first_server_bar + ",\"next_bar\":" + (string)m_z_next_server_bar + ",\"pending\":null}"; CDPCCJsonDocument metadata; if(!metadata.Parse(manifest_text, reason) || !metadata.Parse(resume, reason)) break; const string market_suffix = "/Market.current.dpc"; const int slash = StringLen(m_action6_market_path) - StringLen(market_suffix); if(slash <= 0 || StringSubstr(m_action6_market_path, slash) != market_suffix) { reason = "study.action6.export.market_directory"; break; } const string directory = StringSubstr(m_action6_market_path, 0, slash); saved = DPCCAPSaveTradingGeneration(directory, m_z_common_files, m_action6_market_max_bytes, source_abi, manifest_text, resume, *m_action6_actor, *m_action6_q1, *m_action6_q2, *target_actor, *target_q1, *target_q2, "", "", output_path, output_sha, reason); } while(false); if(CheckPointer(target_actor) != POINTER_INVALID) delete target_actor; if(CheckPointer(target_q2) != POINTER_INVALID) delete target_q2; if(CheckPointer(target_q1) != POINTER_INVALID) delete target_q1; if(!saved) return(Fail(reason == "" ? "study.action6.export" : reason, reason)); manifest = output_path; sha = output_sha; reason = ""; return(true); } //+------------------------------------------------------------------+ //| SaveHistoryModels exports the legacy Z-route policy state | //+------------------------------------------------------------------+ bool SaveHistoryModels(const string stem, string &manifest, string &sha, string &reason) { manifest = ""; sha = ""; string canonical = ""; if(!m_z_history || !m_initialized || m_failed || !m_z_provenance_bound || m_z_step == 0 || m_z_last_bar < 0 || !DPCCCheckpointCanonicalPath(stem, canonical, reason) || canonical != stem) return(Fail("study.history.export_boundary", reason)); if(m_z_step < m_z_rows && !m_z_floor) return(Fail("study.history.export_before_complete", reason)); if(m_z_resume_cursor > 0 && m_z_step <= m_z_resume_cursor) { manifest = m_z_resume_manifest; sha = m_z_resume_sha; reason = ""; return(true); } //--- Inputs were SHA-admitted and loaded before training. Export the //--- retained pins once after the complete pass. string config_sha = "", deltas_sha = ""; if(!DPCCOracleHistoryConfigSHA(m_offline_config, config_sha, reason) || config_sha != m_z_config_sha || !DPCCOracleHistoryDeltasSHA(m_z_deltas, deltas_sha, reason) || deltas_sha != m_z_deltas_sha) return(Fail("BLOCKED:study.history.export.config_or_deltas_changed", reason)); const string prefix = stem + ".v7.latest"; const string receipt = prefix + ".manifest"; const string lock_path = stem + ".writer.lock"; const int writer_lock = FileOpen(lock_path, FILE_READ | FILE_WRITE | FILE_BIN | (m_z_common_files ? FILE_COMMON : 0)); if(writer_lock == INVALID_HANDLE) return(Fail("study.history.export_lock", reason)); bool saved = false; string hashes[3]; string paths[3]; CNet *nets[] = {GetPointer(m_z_actor), GetPointer(m_z_q1), GetPointer(m_z_q2)}; paths[0] = prefix + ".actor.dppt"; paths[1] = prefix + ".q1.dppt"; paths[2] = prefix + ".q2.dppt"; do { bool models = true; for(int i = 0; i < 3 && models; i++) { if(!DPCCSaveCompactPolicyTrainingState(*nets[i], paths[i], m_z_common_files, DPCC_STUDY_FORECAST_ARTIFACT_MAX_BYTES, hashes[i], reason, true)) models = false; } if(!models) break; const bool server_calendar = (m_z_time_domain == "SERVER_CALENDAR_SECONDS_V1"); const string version = "V7"; string wire = "DPCC_Z_ORACLE_MODELS_" + version + "\n"; DPCCOfflineField(wire, "provenance=ADMISSION_TIME_SOURCE_SHA256_LOADED_IN_MEMORY"); DPCCOfflineField(wire, "Z[21,9,12,16];contiguous_group=[K*V=189,H*D=192];no_copy"); for(int i = 0; i < ArraySize(m_z_source_labels); i++) { DPCCOfflineField(wire, m_z_source_labels[i] + ".path=" + m_z_source_paths[i]); DPCCOfflineField(wire, m_z_source_labels[i] + ".sha256=" + m_z_source_hashes[i]); } DPCCOfflineField(wire, "execution_profile.sha256=" + m_z_execution_sha); DPCCOfflineField(wire, "config.sha256=" + m_z_config_sha); DPCCOfflineField(wire, "oracle_deltas.sha256=" + m_z_deltas_sha); DPCCOfflineField(wire, "cursor=" + (string)m_z_step); DPCCOfflineField(wire, "decision_bar=" + (string)m_z_last_bar); DPCCOfflineField(wire, "row_count=" + (string)m_z_rows); DPCCOfflineField(wire, "action_contract=" + DPCC_Z_ACTION_CONTRACT); if(server_calendar) DPCCOfflineField(wire, "time_domain=" + m_z_time_domain); for(int i = 0; i < 3; i++) { DPCCOfflineField(wire, paths[i]); DPCCOfflineField(wire, hashes[i]); } wire += "COMPACT_CNET_POLICY_TRAINING_STATE_V1\nREBUILD_TRANSIENT_STATE_ON_LOAD\n"; wire += "FINANCIAL_STATE_AND_POSITION_NOT_SAVED\nMODELS_ONLY_NOT_RESUMABLE_LEDGER_CHECKPOINT\n"; wire += "NOT_A_READINESS_AUTHORIZATION\nEND_DPCC_Z_ORACLE_MODELS_" + version + "\n"; string expected = "", actual = ""; if(!DPCCCheckpointTextSHA(wire, expected, reason) || !DPCCCheckpointWriteText(receipt, m_z_common_files, wire, reason, true) || !DPCCCheckpointFileSHA(receipt, m_z_common_files, DPCC_RESOLVER_MAX_JSON_BYTES, actual, reason) || actual != expected) break; manifest = receipt; sha = actual; saved = true; } while(false); FileClose(writer_lock); return(saved || Fail(reason == "" ? "study.history.export" : reason, reason)); } }; #endif #endif //+------------------------------------------------------------------+ //| Compare the caller's exact native graph, without running a model | //+------------------------------------------------------------------+ bool DPCCAPNativeDescriptorEqual(CLayerDescription *actual, CLayerDescription *expected) { if(!actual || !expected || actual.type != expected.type || actual.count != expected.count || actual.window != expected.window || actual.window_out != expected.window_out || actual.step != expected.step || actual.layers != expected.layers || actual.batch != expected.batch || actual.activation != expected.activation || actual.optimization != expected.optimization || actual.probability != expected.probability || actual.variables != expected.variables || ArraySize(actual.windows) != ArraySize(expected.windows) || ArraySize(actual.units) != ArraySize(expected.units) || ArraySize(actual.heads) != ArraySize(expected.heads) || actual.radius.Size() != expected.radius.Size()) return(false); for(int i = 0; i < ArraySize(actual.windows); i++) if(actual.windows[i] != expected.windows[i]) return(false); for(int i = 0; i < ArraySize(actual.units); i++) if(actual.units[i] != expected.units[i]) return(false); for(int i = 0; i < ArraySize(actual.heads); i++) if(actual.heads[i] != expected.heads[i]) return(false); for(ulong i = 0; i < actual.radius.Size(); i++) if(actual.radius[i] != expected.radius[i]) return(false); return(true); } //+------------------------------------------------------------------+ //| Decode one known native layer ABI; generic library layers use 0 | //+------------------------------------------------------------------+ bool DPCCAPNativeLayerABI(CNeuronBaseOCL *layer, CLayerDescription *descriptor, uint &abi) { abi = 0; if(!layer || !descriptor) return(false); if(layer.Type() == defNeuronScenarioForecastPriceOCL) { CNeuronScenarioForecastPriceOCL *price = dynamic_cast(layer); if(!price) return(false); abi = price.ABI(); } else if(layer.Type() == defNeuronScenarioDecoderOCL) { CNeuronScenarioDecoderOCL *decoder = dynamic_cast(layer); if(!decoder) return(false); abi = decoder.ABI(); } else if(layer.Type() == defNeuronDPCCOCL) { CNeuronDPCCOCL *policy = dynamic_cast(layer); if(!policy || (policy.ActorABI() == 2 && ArraySize(descriptor.windows) != 9) || (policy.ActorABI() == 3 && ( ArraySize(descriptor.windows) != 0 || ArraySize(descriptor.units) != 8 || descriptor.units[7] != 3)) || ( policy.ActorABI() != 2 && policy.ActorABI() != 3)) return(false); abi = (policy.ActorABI() == 3 ? 3 : descriptor.windows[0]); } return(true); } //+------------------------------------------------------------------+ //| Load a native role from its own finite file; retain the AP | //| handle Native Load closes its handle and exposes no consumed | //| cursor. | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::LoadNative(const SDPCCAPNativeLoadContract &expected, CArrayObj &expected_graph, const uint &expected_abis[], COpenCLMy *shared_context, CNet *&model, string &reason) { return(LoadNativeCore(expected, expected_graph, expected_abis, shared_context, NULL, model, false, reason)); } //+------------------------------------------------------------------+ //| Function CDPCCAPRoleReader::LoadNativeInto | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::LoadNativeInto(const SDPCCAPNativeLoadContract &expected, CNet &destination, COpenCLMy *shared_context, string &reason) { reason = ""; const bool market_section = (expected.role == 1 && (expected.section_id == 100 || expected.section_id == 101)); const bool trading_section = (expected.role == 2 && expected.section_id >= 110 && expected.section_id <= 115); if(!market_section && !trading_section) { reason = "dpccap.native.destination_role"; return(false); } CArrayObj expected_graph; expected_graph.FreeMode(true); uint expected_abis[]; for(int i = 0; i < 64; i++) { CNeuronBaseOCL *layer = destination.Layer(i); if(!layer) break; CLayerDescription *description = layer.GetLayerInfo(); uint abi = 0; if(!description || !DPCCAPNativeLayerABI(layer, description, abi)) { if(description) delete description; reason = "dpccap.native.destination_graph"; return(false); } if(!expected_graph.Add(description)) { delete description; reason = "dpccap.native.destination_graph_capacity"; return(false); } const int count = ArraySize(expected_abis); if(ArrayResize(expected_abis, count + 1) != count + 1) { reason = "dpccap.native.destination_abi_capacity"; return(false); } expected_abis[count] = abi; } if(expected_graph.Total() == 0 || expected_graph.Total() != ArraySize(expected_abis)) { reason = "dpccap.native.destination_graph_empty"; return(false); } CNet *unused = NULL; return(LoadNativeCore(expected, expected_graph, expected_abis, shared_context, GetPointer(destination), unused, false, reason)); } //+------------------------------------------------------------------+ //| Load only a published, technically accepted Market section | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::LoadReadyMarketInto(const SDPCCAPNativeLoadContract &expected, CNet &destination, COpenCLMy *shared_context, string &reason, const bool allow_pending_training) { if(expected.role != 1 || expected.section_id != 100) { reason = "dpccap.ready_market.role_or_section"; return(false); } CArrayObj expected_graph; expected_graph.FreeMode(true); uint expected_abis[]; for(int i = 0; i < 64; i++) { CNeuronBaseOCL *layer = destination.Layer(i); if(!layer) break; CLayerDescription *description = layer.GetLayerInfo(); uint abi = 0; if(!description || !DPCCAPNativeLayerABI(layer, description, abi)) { if(description) delete description; reason = "dpccap.ready_market.destination_graph"; return(false); } if(!expected_graph.Add(description)) { delete description; reason = "dpccap.ready_market.destination_graph_capacity"; return(false); } const int count = ArraySize(expected_abis); if(ArrayResize(expected_abis, count + 1) != count + 1) { reason = "dpccap.ready_market.destination_abi_capacity"; return(false); } expected_abis[count] = abi; } if(expected_graph.Total() != 8 || expected_graph.Total() != ArraySize(expected_abis)) { reason = "dpccap.ready_market.destination_graph_shape"; return(false); } CNet *unused = NULL; return(LoadNativeCore(expected, expected_graph, expected_abis, shared_context, GetPointer(destination), unused, !allow_pending_training, reason)); } //+------------------------------------------------------------------+ //| Function CDPCCAPRoleReader::ReadJSONSection | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::ReadJSONSection(const uint section_id, const ulong maximum_bytes, string &text, string &reason) { text = ""; reason = ""; if(m_handle == INVALID_HANDLE || m_active != -1 || maximum_bytes == 0 || maximum_bytes > 1048576) { reason = "dpccap.reader.json_contract"; return(false); } int section = -1; for(int i = 0; i < ArraySize(m_sections); i++) if(m_sections[i].id == section_id && m_sections[i].encoding == 1) section = i; if(section < 0 || m_sections[section].length == 0 || m_sections[section].length > maximum_bytes) { reason = "dpccap.reader.json_section_limit"; return(false); } int handle = INVALID_HANDLE; if(!BeginSection(section_id, 1, handle, reason)) return(false); uchar bytes[]; const int length = (int)m_sections[section].length; if(ArrayResize(bytes, length) != length || FileReadArray(handle, bytes, 0, length) != (uint)length || !EndSection(reason)) { reason = "dpccap.reader.json_read"; return(false); } CDPCCIncrementalSHA256 hash; uchar digest[]; string observed = ""; if(!hash.Init(reason) || !hash.Update(bytes, 0, length, reason) || !hash.Final(digest, observed, reason) || observed != m_sections[section].sha256 || !DPCCStrictUtf8Decode(bytes, length, (int)maximum_bytes, text, reason)) { text = ""; reason = "dpccap.reader.json_integrity:" + reason; return(false); } return(true); } //+------------------------------------------------------------------+ //| Function CDPCCAPRoleReader::VerifySectionIntegrity | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::VerifySectionIntegrity(const uint section_id, const uint encoding, string &reason) { reason = ""; if(m_handle == INVALID_HANDLE || m_active != -1 || (encoding != 1 && encoding != 2)) { reason = "dpccap.reader.section_integrity_contract"; return(false); } int section = -1; for(int i = 0; i < ArraySize(m_sections); i++) if(m_sections[i].id == section_id && m_sections[i].encoding == encoding) section = i; if(section < 0 || m_sections[section].length == 0) { reason = "dpccap.reader.section_integrity_missing"; return(false); } int handle = INVALID_HANDLE; if(!BeginSection(section_id, encoding, handle, reason)) return(false); CDPCCIncrementalSHA256 hash; uchar bytes[]; bool ok = (ArrayResize(bytes, DPCC_AP_HASH_CHUNK) == DPCC_AP_HASH_CHUNK && hash.Init(reason)); ulong remaining = m_sections[section].length; while(ok && remaining > 0) { const uint count = (uint)MathMin((ulong)DPCC_AP_HASH_CHUNK, remaining); if(FileReadArray(handle, bytes, 0, count) != count || !hash.Update(bytes, 0, (int)count, reason)) ok = false; else remaining -= count; } const bool ended = EndSection(reason); uchar digest[]; string observed = ""; if(!ok || !ended || remaining != 0 || !hash.Final(digest, observed, reason) || observed != m_sections[section].sha256) { reason = "dpccap.reader.section_integrity_sha"; return(false); } return(true); } //+------------------------------------------------------------------+ //| Resolve only metadata admitted by the ready-Market reader gates | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::ReadReadyMarketContract(const string market_profile_sha256, const string task_id, const ulong max_native_bytes, SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training) { ZeroMemory(expected); reason = ""; if(m_handle == INVALID_HANDLE || m_active != -1 || m_role != 1 || !DPCCResolverLowerSha256(m_source_abi) || !DPCCResolverLowerSha256(market_profile_sha256) || !DPCCDataNonemptyText(task_id) || max_native_bytes == 0 || max_native_bytes > (ulong)LONG_MAX || FileSize(m_handle) > max_native_bytes) { reason = "dpccap.ready_market.resolve_contract"; return(false); } string manifest_text = ""; if(!ReadJSONSection(1, 1048576, manifest_text, reason)) return(false); CDPCCJsonDocument document; if(!document.Parse(manifest_text, reason)) return(false); const int root = document.Root(), profile = document.FindMember(root, "profile"); string contract = "", revision = "", role = "", abi = "", data = "", archive = "", task = "", stage = ""; string auxiliary_target_policy = ""; string profile_task = "", observed_profile_sha = "", observed_market_profile_sha = ""; bool ready = false; if(document.Type(root) != DPCC_JSON_OBJECT || document.Type(profile) != DPCC_JSON_OBJECT || !DPCCM05KnownMembers(document, profile, "|task_id|sha256|market_sha256|contents|", "native.manifest.profile", reason) || !DPCCResourceReadString(document, root, "contract_id", contract, reason, "native.manifest.contract") || !DPCCResourceReadString(document, root, "revision", revision, reason, "native.manifest.revision") || !DPCCResourceReadString(document, root, "role", role, reason, "native.manifest.role") || !DPCCResourceReadString(document, root, "source_abi_sha256", abi, reason, "native.manifest.abi") || !DPCCResourceReadString(document, root, "data_sha256", data, reason, "native.manifest.data") || !DPCCResourceReadString(document, root, "source_archive_sha256", archive, reason, "native.manifest.archive") || !DPCCResourceReadString(document, root, "task_id", task, reason, "native.manifest.task") || !DPCCResourceReadString(document, root, "stage", stage, reason, "native.manifest.stage") || !DPCCResourceReadString(document, root, "auxiliary_target_policy", auxiliary_target_policy, reason, "native.manifest.auxiliary_target_policy") || !DPCCResourceReadString(document, profile, "task_id", profile_task, reason, "native.manifest.profile_task") || !DPCCResourceReadString(document, profile, "sha256", observed_profile_sha, reason, "native.manifest.profile") || !DPCCResourceReadString(document, profile, "market_sha256", observed_market_profile_sha, reason, "native.manifest.profile.market_sha") || !DPCCResourceReadBool(document, root, "ready", ready, reason, "native.manifest.ready")) return(false); const bool supported_training_stage = (stage == "LATENT_PREPARE" || stage == "PRICE_JOINT" || stage == "OMPB_CALIBRATE" || stage == "PRICE_ADAPT" || stage == "VERIFY"); const int profile_contents = document.FindMember(profile, "contents"); string embedded_profile_sha = "", embedded_market_profile_sha = ""; CDPCCJsonDocument embedded_profile; if(document.Type(profile_contents) != DPCC_JSON_STRING || !DPCCCheckpointTextSHA(document.Text(profile_contents), embedded_profile_sha, reason) || embedded_profile_sha != observed_profile_sha || !embedded_profile.Parse(document.Text(profile_contents), reason) || !DPCCExecutionMarketProfileSHA(embedded_profile, embedded_profile.Root(), embedded_market_profile_sha, reason)) return(false); StringToLower(archive); const int dependency = document.FindMember(root, "market_dependency_sha256"); if(contract != DPCC_AP_CONTRACT || revision != DPCC_M05_MARKET_REVISION || role != "Market" || auxiliary_target_policy != DPCC_M05_TARGET_POLICY || task != task_id || profile_task != task_id || !DPCCResolverLowerSha256(observed_profile_sha) || observed_market_profile_sha != market_profile_sha256 || embedded_market_profile_sha != observed_market_profile_sha || abi != m_source_abi || !DPCCResolverLowerSha256(data) || !DPCCResolverLowerSha256(archive) || !DPCCDataNonemptyText(stage) || (allow_pending_training && !supported_training_stage) || document.Type(dependency) != DPCC_JSON_NULL || (!allow_pending_training && !ready)) { reason = "dpccap.native.manifest_binding"; return(false); } bool have_manifest = false, have_market = false, have_locator = false, have_resume = false; bool have_runtime_snapshot = false; for(int i = 0; i < ArraySize(m_sections); i++) { const uint id = m_sections[i].id; const uint encoding = m_sections[i].encoding; if(id == 1 && encoding == 1) have_manifest = true; else if(id == 100 && encoding == 2) have_market = true; else if(id == 102 && encoding == 1) have_locator = true; else if(id == 200 && encoding == 1) have_resume = true; else if(id == 210 && encoding == 3) have_runtime_snapshot = true; else { reason = "dpccap.ready_market.section_inventory"; return(false); } } if(!have_manifest || !have_market || !have_locator || !have_resume || ArraySize(m_sections) != (have_runtime_snapshot ? 5 : 4)) { reason = "dpccap.ready_market.section_inventory"; return(false); } bool calibration_completed = false, stage02_quality_pass = false; if(!DPCCResourceReadBool(document, root, "calibration_completed", calibration_completed, reason, "native.manifest.calibration_completed") || !DPCCResourceReadBool(document, root, "stage02_quality_pass", stage02_quality_pass, reason, "native.manifest.stage02_quality_pass") || (!allow_pending_training && (!calibration_completed || (stage != "PRICE_ADAPT" && stage != "VERIFY")))) { reason = "dpccap.ready_market.training_or_calibration_not_complete"; return(false); } const int acceptance = document.FindMember(root, "acceptance"); if(document.Type(acceptance) != DPCC_JSON_ARRAY || document.ChildCount(acceptance) != 1) { reason = "dpccap.ready_market.acceptance_shape"; return(false); } const int item = document.FirstChild(acceptance); string case_id = "", status = ""; const int evidence_node = document.FindMember(item, "evidence_sha256"); if(document.Type(item) != DPCC_JSON_OBJECT || document.ChildCount(item) != 3 || !DPCCResourceReadString(document, item, "case_id", case_id, reason, "native.acceptance.case_id") || !DPCCResourceReadString(document, item, "status", status, reason, "native.acceptance.status") || case_id != "A01-A15" || !((status == "NOT_RUN" && document.Type(evidence_node) == DPCC_JSON_NULL) || (status == "PASS" && document.Type(evidence_node) == DPCC_JSON_STRING && DPCCResolverLowerSha256(document.Text(evidence_node))))) { reason = "dpccap.ready_market.acceptance_receipt_invalid"; return(false); } string locator_text = ""; if(!ReadJSONSection(102, 1048576, locator_text, reason)) { reason = "dpccap.ready_market.locator_read:" + reason; return(false); } CDPCCJsonDocument locator; string state_owner = ""; const int locator_root = (locator.Parse(locator_text, reason) ? locator.Root() : -1); if(locator_root < 0 || locator.Type(locator_root) != DPCC_JSON_OBJECT || locator.ChildCount(locator_root) != 1 || !DPCCResourceReadString(locator, locator_root, "state_owner", state_owner, reason, "native.locator.state_owner") || state_owner != "section100.layer5") { reason = "dpccap.ready_market.locator_contract"; return(false); } SDPCCAPNativeLoadContract candidate; candidate.role = 1; candidate.section_id = 100; candidate.encoding = 2; candidate.max_native_bytes = max_native_bytes; candidate.source_abi_sha256 = abi; candidate.profile_sha256 = observed_profile_sha; candidate.market_profile_sha256 = observed_market_profile_sha; candidate.data_sha256 = data; candidate.market_dependency_sha256 = ""; candidate.source_archive_sha256 = archive; candidate.task_id = task; candidate.stage = stage; candidate.market_ready = ready; candidate.calibration_completed = calibration_completed; candidate.stage02_quality_pass = stage02_quality_pass; expected = candidate; reason = ""; return(true); } //+------------------------------------------------------------------+ //| Bind a ready Market path and native contract to the assigned | //| profile | //+------------------------------------------------------------------+ bool DPCCResolveReadyMarketContract(const SDPCCExecutionControl &control, const bool common_files, string &market_path, SDPCCAPNativeLoadContract &expected, string &reason, const bool allow_pending_training = false) { market_path = ""; ZeroMemory(expected); reason = ""; string canonical = "", profile_sha = ""; if(!DPCCResolverLowerSha256(control.profile_sha256) || !DPCCResolverLowerSha256(control.market_profile_sha256) || !DPCCDataNonemptyText(control.task_id) || control.profile_text == "" || !DPCCCheckpointTextSHA(control.profile_text, profile_sha, reason) || profile_sha != control.profile_sha256) { reason = "ready_market.profile.control_binding:" + reason; return(false); } CDPCCJsonDocument profile; if(!profile.Parse(control.profile_text, reason)) return(false); const int root = profile.Root(), persistence = profile.FindMember(root, "persistence"); string market_profile_sha = ""; if(!DPCCExecutionMarketProfileSHA(profile, root, market_profile_sha, reason) || market_profile_sha != control.market_profile_sha256) { reason = "ready_market.profile.market_scope_changed:" + reason; return(false); } string contract = "", revision = "", kind = "", task = "", symbol_source = "", directory = ""; long maximum_signed = 0; if(profile.Type(root) != DPCC_JSON_OBJECT || !DPCCExecutionProfileString(profile, root, "contract_id", contract, reason) || !DPCCExecutionProfileString(profile, root, "revision", revision, reason) || !DPCCExecutionProfileString(profile, root, "profile_kind", kind, reason) || !DPCCExecutionProfileString(profile, root, "task_id", task, reason) || !DPCCExecutionProfileString(profile, root, "symbol_source", symbol_source, reason) || !DPCCResourceReadString(profile, persistence, "task_directory", directory, reason, "profile.persistence.task_directory") || !DPCCResourceReadLong(profile, persistence, "max_file_bytes", maximum_signed, reason, "profile.persistence.max_file_bytes")) return(false); if(contract != DPCC_AP_CONTRACT || revision != "CNET_ACTION6_20261014" || kind != "USER_ASSIGNED" || task != control.task_id || symbol_source != "CHART_SYMBOL" || maximum_signed <= 0 || !DPCCCheckpointCanonicalPath(directory, canonical, reason) || directory != canonical) { reason = "ready_market.profile.assigned_persistence_required"; return(false); } //--- Header discovery selects an ABI to validate, never a readiness verdict. const string path = directory + "/Market.current.dpc"; const ulong maximum = (ulong)maximum_signed; const int handle = FileOpen(path, FILE_READ | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = StringFormat("ready_market.header.open.%d", GetLastError()); return(false); } uchar header[]; const bool header_read = DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason); FileClose(handle); if(!header_read) return(false); const string source_abi = DPCCAPReadSHA(header, 40); if(!DPCCResolverLowerSha256(source_abi)) { reason = "ready_market.header.source_abi"; return(false); } //--- The reader locks the role and authenticates every section before metadata resolution. CDPCCAPRoleReader reader; if(!reader.Open(path, common_files, 1, source_abi, maximum, 64, reason)) return(false); SDPCCAPNativeLoadContract candidate; const bool resolved = reader.ReadReadyMarketContract(control.market_profile_sha256, control.task_id, maximum, candidate, reason, allow_pending_training); reader.Close(); if(!resolved) return(false); market_path = path; expected = candidate; reason = ""; return(true); } //--- Read the profile-bound FIT normalization used by the 109-coordinate Actor condition. bool DPCCResolveAction6ConditionProfile(const SDPCCExecutionControl &control, float &scale[], float &bias[], string &reason) { reason = ""; string text = "", sha = ""; if(control.profile_text == "" || !DPCCCheckpointTextSHA(control.profile_text, sha, reason) || sha != control.profile_sha256) { reason = "ready_market.condition.profile_identity:" + reason; return(false); } CDPCCJsonDocument profile; if(!profile.Parse(control.profile_text, reason)) return(false); const int root = profile.Root(), condition = profile.FindMember(root, "condition"); string source = ""; if(condition < 0 || !DPCCResourceReadString(profile, condition, "normalization_source", source, reason, "condition.normalization_source") || source == "DIAGNOSTIC_IDENTITY_NOT_TRAINING" || source == "TEMPLATE_NOT_AUTHORIZED") { reason = "BLOCKED:ready_market.condition.fit_normalization_required"; return(false); } const int scale_node = profile.FindMember(condition, "scale"), bias_node = profile.FindMember(condition, "bias"); if(profile.ChildCount(scale_node) != 109 || profile.ChildCount(bias_node) != 109 || ArrayResize(scale, 109) != 109 || ArrayResize(bias, 109) != 109) { reason = "BLOCKED:ready_market.condition.normalization_shape"; return(false); } for(int i = 0; i < 109; i++) { double s = 0, b = 0; if(!profile.TryNumber(profile.ChildAt(scale_node, i), s) || !profile.TryNumber(profile.ChildAt(bias_node, i), b) || !MathIsValidNumber(s) || s <= 0 || !MathIsValidNumber(b) || MathAbs(s) > 3.402823466e38 || MathAbs(b) > 3.402823466e38) { reason = StringFormat("BLOCKED:ready_market.condition.normalization_value_%d", i); return(false); } scale[i] = (float)s; bias[i] = (float)b; } reason = ""; return(true); } //+------------------------------------------------------------------+ //| Owned Trading restore metadata shared by StudyOnline and Test | //+------------------------------------------------------------------+ struct SDPCCAction6TradingRestore { SDPCCRuntimeConfig runtime; SDPCCAPNativeLoadContract market_contract; float limits[]; string market_path, market_sha256, trading_path, trading_sha256; string raw_path, raw_sha256, model_sha256, study_run_sha256; string source_stage; ulong completed_transitions; ulong diffusion_seed; bool floor_reached; }; //+------------------------------------------------------------------+ //| Drop an unpublished bundle before releasing its borrowed Market | //+------------------------------------------------------------------+ void DPCCReleaseAction6Trading(CNet *&actor, CNet *&q1, CNet *&q2, CNet *&target_actor, CNet *&target_q1, CNet *&target_q2) { if(CheckPointer(actor) != POINTER_INVALID) delete actor; if(CheckPointer(target_actor) != POINTER_INVALID) delete target_actor; if(CheckPointer(q1) != POINTER_INVALID) delete q1; if(CheckPointer(q2) != POINTER_INVALID) delete q2; if(CheckPointer(target_q1) != POINTER_INVALID) delete target_q1; if(CheckPointer(target_q2) != POINTER_INVALID) delete target_q2; actor = NULL; q1 = NULL; q2 = NULL; target_actor = NULL; target_q1 = NULL; target_q2 = NULL; } //+------------------------------------------------------------------+ //| Admit profile-bound Study or Online roles before any CNet is | //| changed | //+------------------------------------------------------------------+ bool DPCCResolveAction6TradingGeneration(CDPCCAPRoleReader &reader, const SDPCCExecutionControl &control, const SDPCCAPNativeLoadContract &market_contract, const string market_sha, CDPCCJsonDocument &assigned_profile, SDPCCAction6TradingRestore &resolved, SDPCCAPNativeLoadContract &native, string &reason) { string manifest_text = "", resume_text = ""; if(!reader.ReadJSONSection(1, 1048576, manifest_text, reason) || !reader.ReadJSONSection(200, 1048576, resume_text, reason)) return(false); CDPCCJsonDocument manifest, resume; if(!manifest.Parse(manifest_text, reason) || !resume.Parse(resume_text, reason)) return(false); const int root = manifest.Root(), profile = manifest.FindMember(root, "profile"), model = manifest.FindMember(root, "model"), financial = manifest.FindMember(root, "financial_profile"), source = manifest.FindMember(root, "data_source"), record = resume.Root(); const int raw_sha_member = manifest.FindMember(source, "sha256"); string contract = "", revision = "", role = "", task = "", stage = "", profile_sha = "", profile_task = ""; string data_sha = "", dependency = "", archive = "", model_sha = "", model_text = "", financial_sha = "", financial_payload = ""; string raw_path = "", raw_sha = "", study_run = "", resume_run = "", resume_profile = "", resume_stage = "", abi = ""; long saved_mode = -1, rows = 0, completed = 0, next = 0, updates = 0; bool complete = false, floor = false, ready = false; const bool test_ablation = (control.stage == DPCC_STAGE_TEST); const bool allow_warm_start = (test_ablation || control.stage == DPCC_STAGE_STUDYONLINE); if(!DPCCResourceReadString(manifest, root, "contract_id", contract, reason, "trading.contract_id") || !DPCCResourceReadString(manifest, root, "revision", revision, reason, "trading.revision") || !DPCCResourceReadString(manifest, root, "role", role, reason, "trading.role") || !DPCCResourceReadString(manifest, root, "task_id", task, reason, "trading.task_id") || !DPCCResourceReadString(manifest, root, "stage", stage, reason, "trading.stage") || !DPCCResourceReadString(manifest, root, "source_abi_sha256", abi, reason, "trading.source_abi_sha256") || !DPCCResourceReadString(manifest, root, "data_sha256", data_sha, reason, "trading.data_sha256") || !DPCCResourceReadString(manifest, root, "market_dependency_sha256", dependency, reason, "trading.market_dependency") || !DPCCResourceReadString(manifest, root, "source_archive_sha256", archive, reason, "trading.archive") || !DPCCResourceReadString(manifest, root, "study_run_sha256", study_run, reason, "trading.study_run_sha256") || !DPCCResourceReadLong(manifest, root, "study_mode", saved_mode, reason, "trading.study_mode") || !DPCCResourceReadBool(manifest, root, "ready", ready, reason, "trading.ready") || !DPCCResourceReadString(manifest, profile, "sha256", profile_sha, reason, "trading.profile.sha256") || !DPCCResourceReadString(manifest, profile, "task_id", profile_task, reason, "trading.profile.task_id") || !DPCCResourceReadString(manifest, model, "sha256", model_sha, reason, "trading.model.sha256") || !DPCCResourceReadString(manifest, model, "contents", model_text, reason, "trading.model.contents") || !DPCCResourceReadString(manifest, financial, "sha256", financial_sha, reason, "trading.financial_profile.sha256") || !DPCCResourceReadString(manifest, financial, "payload", financial_payload, reason, "trading.financial_profile.payload") || !DPCCResourceReadString(manifest, source, "path", raw_path, reason, "trading.data_source.path") || raw_sha_member < 0) return(false); if(manifest.Type(raw_sha_member) == DPCC_JSON_STRING) { if(!DPCCResourceReadString(manifest, source, "sha256", raw_sha, reason, "trading.data_source.sha256")) return(false); } else if(manifest.Type(raw_sha_member) == DPCC_JSON_NULL) raw_sha = ""; else { reason = "BLOCKED:trading.data_source.sha256_type"; return(false); } if(!DPCCResourceReadBool(resume, record, "training_complete", complete, reason, "BLOCKED:trading.resume.training_complete_required") || !DPCCResourceReadString(resume, record, "study_run_sha256", resume_run, reason, "trading.resume.study_run_sha256") || !DPCCResourceReadString(resume, record, "profile_sha256", resume_profile, reason, "trading.resume.profile_sha256") || !DPCCResourceReadString(resume, record, "stage", resume_stage, reason, "trading.resume.stage")) return(false); if(stage == "Study") { if(!DPCCResourceReadBool(resume, record, "floor_reached", floor, reason, "trading.resume.floor_reached") || !DPCCResourceReadLong(resume, record, "row_count", rows, reason, "trading.resume.row_count") || !DPCCResourceReadLong(resume, record, "completed_transitions", completed, reason, "trading.resume.completed_transitions") || !DPCCResourceReadLong(resume, record, "next_record", next, reason, "trading.resume.next_record") || !DPCCResourceReadLong(resume, record, "policy_updates", updates, reason, "trading.resume.policy_updates") || rows <= 0 || completed <= 0 || completed > rows || next != completed || updates < completed || resume.Type(resume.FindMember(record, "pending")) != DPCC_JSON_NULL) { reason = "BLOCKED:trading.study_checkpoint_consistency:" + reason; return(false); } } else if(stage == "StudyOnline") { string restart = "", completion = ""; if((control.stage == DPCC_STAGE_STUDYONLINE && control.resume_mode != DPCC_RESUME_NEW_EPISODE_FROM_WEIGHTS) || !DPCCM05KnownMembers(resume, record, "|stage|training_complete|profile_sha256|study_run_sha256|restart_policy|completion_reason|", "trading.online_metadata", reason) || !DPCCResourceReadString(resume, record, "restart_policy", restart, reason, "trading.online.restart_policy") || !DPCCResourceReadString(resume, record, "completion_reason", completion, reason, "trading.online.completion_reason") || restart != "LATEST_MODELS_REPLAY_HISTORY_CURRENT_ACCOUNT" || (completion != "TRAINING_END" && completion != "CONTROLLED_FLOOR")) { reason = "BLOCKED:trading.Online_requires_models_only_NEW_EPISODE_FROM_WEIGHTS:" + reason; return(false); } floor = (completion == "CONTROLLED_FLOOR"); } else { reason = "BLOCKED:trading.completed_Study_or_StudyOnline_required"; return(false); } string canonical = "", actual_model_sha = "", actual_financial_sha = ""; StringToLower(archive); if(contract != DPCC_AP_CONTRACT || revision != DPCC_TRADING_REVISION || role != "Trading" || task != control.task_id || profile_task != control.task_id || resume_stage != stage || (!allow_warm_start && (profile_sha != control.profile_sha256 || resume_profile != control.profile_sha256 || dependency != market_sha || data_sha != market_contract.data_sha256 || archive != market_contract.source_archive_sha256)) || (raw_sha != "" && raw_sha != data_sha) || saved_mode < DPCC_P0 || saved_mode > DPCC_P2 || !DPCCResolverLowerSha256(abi) || !DPCCResolverLowerSha256(model_sha) || !DPCCResolverLowerSha256(financial_sha) || !DPCCResolverLowerSha256(study_run) || study_run != resume_run || (!complete && control.stage != DPCC_STAGE_STUDYONLINE && control.stage != DPCC_STAGE_TEST) || manifest.Type(manifest.FindMember(root, "data_index_sha256")) != DPCC_JSON_NULL || !DPCCCheckpointCanonicalPath(raw_path, canonical, reason) || canonical != raw_path || !DPCCCheckpointTextSHA(model_text, actual_model_sha, reason) || actual_model_sha != model_sha || !DPCCCheckpointTextSHA(financial_payload, actual_financial_sha, reason) || actual_financial_sha != financial_sha || (!allow_warm_start && !DPCCBankJsonEqual(manifest, manifest.FindMember(profile, "contents"), assigned_profile, assigned_profile.Root(), 0))) { reason = "BLOCKED:trading.completed_profile_market_binding:" + reason; return(false); } if(control.stage == DPCC_STAGE_STUDYONLINE && (profile_sha != control.profile_sha256 || resume_profile != control.profile_sha256 || dependency != market_sha || data_sha != market_contract.data_sha256 || archive != market_contract.source_archive_sha256)) PrintFormat( "DPCC_STUDYONLINE_TRANSFER source_stage=%s training_complete=%s " + "profile_match=%s market_match=%s data_match=%s", stage, complete ? "true" : "false", (profile_sha == control.profile_sha256 && resume_profile == control.profile_sha256) ? "true" : "false", (dependency == market_sha && archive == market_contract.source_archive_sha256) ? "true" : "false", data_sha == market_contract.data_sha256 ? "true" : "false"); if(test_ablation) PrintFormat( "DPCC_TEST_ABLATION source_stage=%s training_complete=%s generation_ready=%s profile_match=%s market_match=%s data_match=%s", stage, complete ? "true" : "false", ready ? "true" : "false", (profile_sha == control.profile_sha256 && resume_profile == control.profile_sha256 && DPCCBankJsonEqual(manifest, manifest.FindMember(profile, "contents"), assigned_profile, assigned_profile.Root(), 0)) ? "true" : "false", dependency == market_sha ? "true" : "false", data_sha == market_contract.data_sha256 ? "true" : "false"); const int diffusion = assigned_profile.FindMember(assigned_profile.Root(), "diffusion"); const int seed = assigned_profile.FindMember(diffusion, "seed"); if(assigned_profile.Type(seed) != DPCC_JSON_NUMBER || !DPCCCheckpointReadUnsigned(assigned_profile.Text(seed), ULONG_MAX, resolved.diffusion_seed)) { reason = "BLOCKED:trading.assigned_profile.diffusion.seed_exact_uint64_required"; return(false); } //--- Completed training is distinct from the acceptance verdict in this same manifest. if(!DPCCDecodeAction6ModelRuntime(model_text, resolved.runtime, resolved.limits, reason)) return(false); resolved.runtime.financial.mode = (ENUM_DPCC_MODE)saved_mode; float scale[], bias[]; if(!DPCCResolveAction6ConditionProfile(control, scale, bias, reason) || ArrayCopy(resolved.runtime.condition_scale, scale) != 109 || ArrayCopy(resolved.runtime.condition_bias, bias) != 109) return(false); native.role = 2; native.encoding = 2; native.section_id = 110; native.max_native_bytes = market_contract.max_native_bytes; native.source_abi_sha256 = abi; native.profile_sha256 = profile_sha; native.data_sha256 = data_sha; native.market_dependency_sha256 = dependency; native.source_archive_sha256 = archive; native.task_id = task; native.stage = stage; resolved.raw_path = raw_path; resolved.raw_sha256 = raw_sha; resolved.model_sha256 = model_sha; resolved.study_run_sha256 = study_run; resolved.source_stage = stage; resolved.completed_transitions = (ulong)completed; resolved.floor_reached = floor; return(true); } #ifdef DPCC_TRAJECTORY_ENTRY_STUDY_FORECAST //+------------------------------------------------------------------+ //| Restore one complete Trading bundle against an accepted Market | //+------------------------------------------------------------------+ bool DPCCLoadAction6Trading(const SDPCCExecutionControl &control, const bool common_files, CDPCCStudyForecastCore &market_owner, CNet *&actor, CNet *&q1, CNet *&q2, CNet *&target_actor, CNet *&target_q1, CNet *&target_q2, SDPCCAction6TradingRestore &restored, string &reason) { ZeroMemory(restored); reason = ""; if(CheckPointer(actor) != POINTER_INVALID || CheckPointer(q1) != POINTER_INVALID || CheckPointer(q2) != POINTER_INVALID || CheckPointer(target_actor) != POINTER_INVALID || CheckPointer(target_q1) != POINTER_INVALID || CheckPointer(target_q2) != POINTER_INVALID || market_owner.ReadyMarketNetwork() != NULL || (control.stage != DPCC_STAGE_STUDYONLINE && control.stage != DPCC_STAGE_TEST)) { reason = "trading.restore.destination_or_stage"; return(false); } string market_path = "", market_sha = ""; SDPCCAPNativeLoadContract market_contract; if(!DPCCResolveReadyMarketContract(control, common_files, market_path, market_contract, reason, true) || !DPCCCheckpointFileSHA(market_path, common_files, market_contract.max_native_bytes, market_sha, reason)) return(false); string assigned_text = control.profile_text, assigned_sha = ""; if(assigned_text == "" || !DPCCCheckpointTextSHA(assigned_text, assigned_sha, reason) || assigned_sha != control.profile_sha256) return(false); CDPCCJsonDocument assigned_profile; if(!assigned_profile.Parse(assigned_text, reason)) return(false); const string suffix = "/Market.current.dpc"; const int prefix = StringLen(market_path) - StringLen(suffix); if(prefix <= 0 || StringSubstr(market_path, prefix) != suffix) { reason = "trading.restore.market_directory"; return(false); } const string directory = StringSubstr(market_path, 0, prefix); const string paths[] = {directory + "/Trading.current.dpc", directory + "/Trading.bak.dpc"}; string last_reason = "trading.restore.no_generation"; for(int attempt = 0; attempt < ArraySize(paths); attempt++) { const int handle = FileOpen(paths[attempt], FILE_READ | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { if(last_reason == "trading.restore.no_generation") last_reason = StringFormat("trading.restore.header.open.%d", GetLastError()); continue; } uchar header[]; const bool header_ok = DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, last_reason); FileClose(handle); if(!header_ok) continue; CDPCCAPRoleReader reader; if(!reader.Open(paths[attempt], common_files, 2, DPCCAPReadSHA(header, 40), market_contract.max_native_bytes, 8, last_reason)) continue; SDPCCAction6TradingRestore candidate; SDPCCAPNativeLoadContract native; if(!DPCCResolveAction6TradingGeneration(reader, control, market_contract, market_sha, assigned_profile, candidate, native, last_reason)) { //--- Open already authenticated the whole frame under the shared reader lock. if(attempt == 0) { reason = "BLOCKED:trading.current_generation_not_admitted:" + last_reason; return(false); } continue; } //--- No fallback after a destructive Market load; reject the entire bundle on failure. if(!market_owner.RestoreReadyEncoder(market_path, common_files, candidate.raw_path, candidate.raw_sha256, control.timeframe, market_contract, reason, true, true)) return(false); CNet *market = market_owner.ReadyMarketNetwork(); CNet *roles[6]; for(int i = 0; i < 6; i++) roles[i] = NULL; CNeuronDPCCOCL *online_top = NULL, *target_top = NULL; bool loaded = false; do { if(!DPCCCreateAction6CriticNet(roles[1], market, reason) || !DPCCCreateAction6CriticNet(roles[2], market, reason) || !DPCCCreateAction6CriticNet(roles[4], market, reason) || !DPCCCreateAction6CriticNet(roles[5], market, reason) || !DPCCCreateAction6ActorNet(roles[0], candidate.runtime, market, roles[1], roles[2], online_top, reason) || !DPCCCreateAction6ActorNet(roles[3], candidate.runtime, market, roles[4], roles[5], target_top, reason)) break; string graphs[6]; bool graphs_ok = true; for(int i = 0; i < 6 && graphs_ok; i++) graphs_ok = DPCCAPTradingGraph(*roles[i], (i == 0 || i == 3 ? 2 : 5), graphs[i], reason); if(!graphs_ok) break; const string graph = "{\"Actor\":" + graphs[0] + ",\"Q1\":" + graphs[1] + ",\"Q2\":" + graphs[2] + ",\"TargetActor\":" + graphs[3] + ",\"TargetQ1\":" + graphs[4] + ",\"TargetQ2\":" + graphs[5] + "}"; string source_abi = ""; if(!DPCCCheckpointTextSHA(DPCC_AP_CONTRACT + "\n" + DPCC_TRADING_REVISION + "\nTRADING_NATIVE_CNET;SIX_ROLES;MARKET_DEPENDENCY_SHA_V1\n" + native.market_dependency_sha256 + "\n" + graph, source_abi, reason) || source_abi != native.source_abi_sha256) { reason = "trading.restore.caller_graph_abi"; break; } bool native_ok = true; for(int i = 0; i < 6 && native_ok; i++) { native.section_id = (uint)(110 + i); native_ok = reader.LoadNativeInto(native, *roles[i], market.GetOpenCL(), reason); } if(!native_ok) break; online_top = dynamic_cast(roles[0].Layer(1)); target_top = dynamic_cast(roles[3].Layer(1)); string binding_failure = ""; if(CheckPointer(online_top) == POINTER_INVALID) binding_failure = "actor_layer_missing"; else if(CheckPointer(target_top) == POINTER_INVALID) binding_failure = "target_actor_layer_missing"; else if(!online_top.MatchesRuntimeConfig(candidate.runtime)) binding_failure = "actor_runtime_mismatch"; else if(!target_top.MatchesRuntimeConfig(candidate.runtime)) binding_failure = "target_actor_runtime_mismatch"; else if(!online_top.BindPolicyNetworks(market, roles[1], roles[2])) binding_failure = "actor_policy_bind"; else if(!target_top.BindPolicyNetworks(market, roles[4], roles[5])) binding_failure = "target_policy_bind"; if(binding_failure != "") { const int actor_status = (CheckPointer(online_top) == POINTER_INVALID ? -1 : (int)online_top.LastStatus()); const int target_status = (CheckPointer(target_top) == POINTER_INVALID ? -1 : (int)target_top.LastStatus()); PrintFormat("DPCC_TRADING_RESTORE_BIND_FAIL cause=%s actor_status=%d target_status=%d", binding_failure, actor_status, target_status); reason = "trading.restore.loaded_runtime_or_policy_binding:" + binding_failure; break; } bool frozen = true; for(int i = 0; i < 6 && frozen; i++) frozen = roles[i].TrainMode(false) && roles[i].SetWeightsUpdate(false); if(!frozen || !online_top.SetProjectionMode(control.mode) || !target_top.SetProjectionMode(control.mode)) { reason = "trading.restore.freeze_or_projection_mode"; break; } string current_market_sha = "", trading_sha = ""; if(!DPCCCheckpointFileSHA(market_path, common_files, market_contract.max_native_bytes, current_market_sha, reason) || current_market_sha != market_sha || !reader.ArtifactSHA256(trading_sha, reason)) { reason = "trading.restore.generation_changed:" + reason; break; } candidate.market_path = market_path; candidate.market_sha256 = market_sha; candidate.market_contract = market_contract; candidate.trading_path = paths[attempt]; candidate.trading_sha256 = trading_sha; candidate.runtime.financial.mode = control.mode; restored = candidate; actor = roles[0]; q1 = roles[1]; q2 = roles[2]; target_actor = roles[3]; target_q1 = roles[4]; target_q2 = roles[5]; loaded = true; } while(false); if(!loaded) DPCCReleaseAction6Trading(roles[0], roles[1], roles[2], roles[3], roles[4], roles[5]); return(loaded); } reason = "BLOCKED:trading.restore.no_completed_generation:" + last_reason; return(false); } //+------------------------------------------------------------------+ //| Save the last complete Online models once; account state is | //| local | //+------------------------------------------------------------------+ bool DPCCSaveAction6OnlineTrading(const SDPCCExecutionControl &control, const bool common_files, CDPCCStudyForecastCore &market_owner, CNet &actor, CNet &q1, CNet &q2, CNet &target_actor, CNet &target_q1, CNet &target_q2, const SDPCCAction6TradingRestore &loaded, const bool controlled_floor_stop, bool &save_attempted, string &output_path, string &output_sha, string &reason) { output_path = ""; output_sha = ""; reason = ""; if(save_attempted || control.stage != DPCC_STAGE_STUDYONLINE || loaded.trading_path == "" || (loaded.source_stage != "Study" && loaded.source_stage != "StudyOnline")) { reason = "TRADING_SAVE.online.once_or_source_stage"; return(false); } save_attempted = true; CNet *market = market_owner.ReadyMarketNetwork(); CNeuronDPCCOCL *online_top = dynamic_cast(actor.Layer(1)); CNeuronDPCCOCL *target_top = dynamic_cast(target_actor.Layer(1)); if(CheckPointer(market) == POINTER_INVALID || market.WeightsUpdateEnabled() || !online_top || !target_top || online_top.HasValidDecision() || target_top.HasValidDecision() || online_top.LastStatus() != DPCC_OK || target_top.LastStatus() != DPCC_OK || !online_top.MatchesRuntimeConfig(loaded.runtime) || !target_top.MatchesRuntimeConfig(loaded.runtime) || actor.Layer(2) != NULL || target_actor.Layer(2) != NULL || !NNQCriticGraph(GetPointer(q1)) || !NNQCriticGraph(GetPointer(q2)) || !NNQCriticGraph(GetPointer(target_q1)) || !NNQCriticGraph(GetPointer(target_q2))) { reason = "TRADING_SAVE.online.idle_complete_native_graph_required"; return(false); } CNet *roles[] = {GetPointer(actor), GetPointer(q1), GetPointer(q2), GetPointer(target_actor), GetPointer(target_q1), GetPointer(target_q2) }; for(int role = 0; role < 6; role++) { if(roles[role].GetOpenCL() != market.GetOpenCL() || !roles[role].IsOpenCLTransferReady()) { reason = "TRADING_SAVE.online.role_context"; return(false); } for(int previous = 0; previous < role; previous++) if(roles[previous] == roles[role]) { reason = "TRADING_SAVE.online.aliased_roles"; return(false); } } //--- Keep Market identity and provenance checks while accepting pending training quality. string market_path = "", market_sha = ""; SDPCCAPNativeLoadContract market_contract; if(!DPCCResolveReadyMarketContract(control, common_files, market_path, market_contract, reason, true) || market_path != loaded.market_path || !DPCCCheckpointFileSHA(market_path, common_files, market_contract.max_native_bytes, market_sha, reason) || market_sha != loaded.market_sha256) { reason = "TRADING_SAVE.online.Market_dependency_changed:" + reason; return(false); } string profile_text = control.profile_text, profile_sha = ""; if(profile_text == "" || !DPCCCheckpointTextSHA(profile_text, profile_sha, reason) || profile_sha != control.profile_sha256) return(false); CDPCCJsonDocument assigned_profile; if(!assigned_profile.Parse(profile_text, reason)) return(false); const int handle = FileOpen(loaded.trading_path, FILE_READ | FILE_BIN | (common_files ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { reason = "TRADING_SAVE.online.source_open"; return(false); } uchar header[]; const bool header_ok = DPCCAPReadRecord(handle, 0, DPCC_AP_HEADER_BYTES, header, reason); FileClose(handle); if(!header_ok) return(false); CDPCCAPRoleReader reader; if(!reader.Open(loaded.trading_path, common_files, 2, DPCCAPReadSHA(header, 40), market_contract.max_native_bytes, 8, reason)) return(false); string source_sha = "", source_manifest = ""; SDPCCAction6TradingRestore admitted; SDPCCAPNativeLoadContract native; if(!reader.ArtifactSHA256(source_sha, reason) || source_sha != loaded.trading_sha256 || !DPCCResolveAction6TradingGeneration(reader, control, market_contract, market_sha, assigned_profile, admitted, native, reason) || admitted.source_stage != loaded.source_stage || admitted.study_run_sha256 != loaded.study_run_sha256 || admitted.model_sha256 != loaded.model_sha256 || !reader.ReadJSONSection(1, 1048576, source_manifest, reason)) { reason = "TRADING_SAVE.online.loaded_source_binding:" + reason; return(false); } CDPCCJsonDocument source; if(!source.Parse(source_manifest, reason)) return(false); const int root = source.Root(), model = source.FindMember(root, "model"), financial = source.FindMember(root, "financial_profile"); string model_text = "", financial_text = "", financial_sha = ""; if(!DPCCResourceReadString(source, model, "contents", model_text, reason, "trading.model.contents") || !DPCCResourceReadString(source, financial, "payload", financial_text, reason, "trading.financial_profile.payload") || !DPCCResourceReadString(source, financial, "sha256", financial_sha, reason, "trading.financial_profile.sha256")) return(false); string graphs[6]; for(int role = 0; role < 6; role++) if(!DPCCAPTradingGraph(*roles[role], (role == 0 || role == 3 ? 2 : 5), graphs[role], reason)) return(false); const string graph = "{\"Actor\":" + graphs[0] + ",\"Q1\":" + graphs[1] + ",\"Q2\":" + graphs[2] + ",\"TargetActor\":" + graphs[3] + ",\"TargetQ1\":" + graphs[4] + ",\"TargetQ2\":" + graphs[5] + "}"; CDPCCJsonDocument expected_graph; if(!expected_graph.Parse(graph, reason) || !DPCCBankJsonEqual(source, source.FindMember(root, "graph"), expected_graph, expected_graph.Root(), 0)) { reason = "TRADING_SAVE.online.loaded_graph_changed:" + reason; return(false); } string source_abi = ""; if(!DPCCCheckpointTextSHA(DPCC_AP_CONTRACT + "\n" + DPCC_TRADING_REVISION + "\nTRADING_NATIVE_CNET;SIX_ROLES;MARKET_DEPENDENCY_SHA_V1\n" + market_sha + "\n" + graph, source_abi, reason) || source_abi != native.source_abi_sha256) { reason = "TRADING_SAVE.online.caller_graph_abi"; return(false); } const string manifest_text = "{\"contract_id\":" + DPCCAPJSONString(DPCC_AP_CONTRACT) + ",\"revision\":" + DPCCAPJSONString(DPCC_TRADING_REVISION) + ",\"role\":\"Trading\",\"task_id\":" + DPCCAPJSONString(control.task_id) + ",\"stage\":\"StudyOnline\",\"graph\":" + graph + ",\"source_archive_sha256\":" + DPCCAPJSONString(native.source_archive_sha256) + ",\"source_abi_sha256\":" + DPCCAPJSONString(source_abi) + ",\"market_dependency_sha256\":" + DPCCAPJSONString(market_sha) + ",\"data_sha256\":" + DPCCAPJSONString(market_contract.data_sha256) + ",\"data_index_sha256\":null,\"data_source\":{\"path\":" + DPCCAPJSONString( admitted.raw_path) + ",\"sha256\":" + DPCCAPJSONString(admitted.raw_sha256) + "},\"study_run_sha256\":" + DPCCAPJSONString(admitted.study_run_sha256) + ",\"study_mode\":" + ( string)control.mode + ",\"model\":{\"sha256\":" + DPCCAPJSONString(admitted.model_sha256) + ",\"contents\":" + DPCCAPJSONString(model_text) + "},\"financial_profile\":{\"sha256\":" + DPCCAPJSONString(financial_sha) + ",\"payload\":" + DPCCAPJSONString(financial_text) + "},\"profile\":{\"task_id\":" + DPCCAPJSONString(control.task_id) + ",\"sha256\":" + DPCCAPJSONString( control.profile_sha256) + ",\"contents\":" + profile_text + "},\"ready\":false,\"acceptance\":[{\"case_id\":\"ONLINE_TRAINING\"," "\"status\":\"NOT_RUN\",\"evidence_sha256\":null}]}"; const string resume = "{\"stage\":\"StudyOnline\",\"training_complete\":true,\"profile_sha256\":" + DPCCAPJSONString(control.profile_sha256) + ",\"study_run_sha256\":" + DPCCAPJSONString(admitted.study_run_sha256) + ",\"restart_policy\":\"LATEST_MODELS_REPLAY_HISTORY_CURRENT_ACCOUNT\",\"completion_reason\":" + DPCCAPJSONString(controlled_floor_stop ? "CONTROLLED_FLOOR" : "TRAINING_END") + "}"; reader.Close(); const string suffix = "/Market.current.dpc"; const int prefix = StringLen(market_path) - StringLen(suffix); if(prefix <= 0 || StringSubstr(market_path, prefix) != suffix) { reason = "TRADING_SAVE.online.Market_directory"; return(false); } const string directory = StringSubstr(market_path, 0, prefix); return(DPCCAPSaveTradingGeneration(directory, common_files, market_contract.max_native_bytes, source_abi, manifest_text, resume, actor, q1, q2, target_actor, target_q1, target_q2, loaded.trading_path, loaded.trading_sha256, output_path, output_sha, reason)); } #endif //+------------------------------------------------------------------+ //| Function CDPCCAPRoleReader::LoadNativeCore | //+------------------------------------------------------------------+ bool CDPCCAPRoleReader::LoadNativeCore(const SDPCCAPNativeLoadContract &expected, CArrayObj &expected_graph, const uint &expected_abis[], COpenCLMy *shared_context, CNet *destination, CNet *&model, const bool require_ready_market, string &reason) { reason = ""; const bool market_section = (expected.role == 1 && (expected.section_id == 100 || expected.section_id == 101)); const bool trading_section = (expected.role == 2 && expected.section_id >= 110 && expected.section_id <= 115); if(model || (destination && CheckPointer(destination) == POINTER_INVALID) || (shared_context && CheckPointer(shared_context) == POINTER_INVALID) || m_handle == INVALID_HANDLE || m_active != -1 || m_role != expected.role || m_source_abi != expected.source_abi_sha256 || (!market_section && !trading_section) || expected.encoding != 2 || expected.max_native_bytes == 0 || expected.max_native_bytes > (ulong)LONG_MAX || expected_graph.Total() < 1 || expected_graph.Total() != ArraySize(expected_abis) || !DPCCResolverLowerSha256(expected.source_abi_sha256) || !DPCCResolverLowerSha256(expected.profile_sha256) || (expected.role == 1 && !DPCCResolverLowerSha256(expected.market_profile_sha256)) || !DPCCResolverLowerSha256(expected.data_sha256) || expected.task_id == "" || expected.stage == "" || (expected.role == 1 && expected.market_dependency_sha256 != "") || (require_ready_market && (expected.role != 1 || expected.section_id != 100)) || (expected.role == 2 && (!DPCCResolverLowerSha256(expected.market_dependency_sha256) || !destination || !shared_context || destination.GetOpenCL() != shared_context || !destination.IsOpenCLTransferReady()))) { reason = "dpccap.native.contract"; return(false); } string expected_archive = expected.source_archive_sha256; StringToLower(expected_archive); if(!DPCCResolverLowerSha256(expected_archive)) { reason = "dpccap.native.archive_contract"; return(false); } int section = -1, manifest = -1; for(int i = 0; i < ArraySize(m_sections); i++) { if(m_sections[i].id == expected.section_id && m_sections[i].encoding == expected.encoding) section = i; if(m_sections[i].id == 1 && m_sections[i].encoding == 1) manifest = i; } if(section < 0 || manifest < 0 || m_sections[section].length == 0 || m_sections[section].length > expected.max_native_bytes || m_sections[manifest].length == 0 || m_sections[manifest].length > 1048576) { reason = "dpccap.native.section_limits"; return(false); } int ap_handle = INVALID_HANDLE; uchar manifest_bytes[]; string manifest_text = "", manifest_sha = ""; if(!BeginSection(1, 1, ap_handle, reason)) return(false); const int manifest_length = (int)m_sections[manifest].length; if(ArrayResize(manifest_bytes, manifest_length) != manifest_length || FileReadArray(ap_handle, manifest_bytes, 0, manifest_length) != (uint)manifest_length || !EndSection(reason)) { Close(); reason = "dpccap.native.manifest_read"; return(false); } CDPCCIncrementalSHA256 manifest_hash; uchar digest[]; if(!manifest_hash.Init(reason) || !manifest_hash.Update(manifest_bytes, 0, manifest_length, reason) || !manifest_hash.Final(digest, manifest_sha, reason) || manifest_sha != m_sections[manifest].sha256 || !DPCCStrictUtf8Decode(manifest_bytes, manifest_length, 1048576, manifest_text, reason)) { reason = "dpccap.native.manifest_integrity:" + reason; return(false); } CDPCCJsonDocument document; if(!document.Parse(manifest_text, reason)) return(false); const int root = document.Root(), profile = document.FindMember(root, "profile"); string contract = "", revision = "", role = "", abi_sha = "", profile_sha = "", data_sha = ""; string task = "", stage = "", archive = "", profile_task = "", market_profile_sha = ""; if(document.Type(root) != DPCC_JSON_OBJECT || document.Type(profile) != DPCC_JSON_OBJECT || !DPCCResourceReadString(document, root, "contract_id", contract, reason, "native.manifest.contract") || !DPCCResourceReadString(document, root, "revision", revision, reason, "native.manifest.revision") || !DPCCResourceReadString(document, root, "role", role, reason, "native.manifest.role") || !DPCCResourceReadString(document, root, "source_abi_sha256", abi_sha, reason, "native.manifest.abi") || !DPCCResourceReadString(document, root, "data_sha256", data_sha, reason, "native.manifest.data") || !DPCCResourceReadString(document, root, "task_id", task, reason, "native.manifest.task") || !DPCCResourceReadString(document, root, "stage", stage, reason, "native.manifest.stage") || !DPCCResourceReadString(document, root, "source_archive_sha256", archive, reason, "native.manifest.archive") || !DPCCResourceReadString(document, profile, "sha256", profile_sha, reason, "native.manifest.profile") || !DPCCResourceReadString(document, profile, "task_id", profile_task, reason, "native.manifest.profile_task")) return(false); if(expected.role == 1 && !DPCCResourceReadString(document, profile, "market_sha256", market_profile_sha, reason, "native.manifest.profile.market_sha")) return(false); string embedded_profile_sha = "", embedded_market_profile_sha = ""; CDPCCJsonDocument embedded_profile; if(expected.role == 1) { const int profile_contents = document.FindMember(profile, "contents"); if(!DPCCM05KnownMembers(document, profile, "|task_id|sha256|market_sha256|contents|", "native.manifest.profile", reason) || document.Type(profile_contents) != DPCC_JSON_STRING || !DPCCCheckpointTextSHA(document.Text(profile_contents), embedded_profile_sha, reason) || embedded_profile_sha != profile_sha || !embedded_profile.Parse(document.Text(profile_contents), reason) || !DPCCExecutionMarketProfileSHA(embedded_profile, embedded_profile.Root(), embedded_market_profile_sha, reason)) return(false); } StringToLower(archive); const int dependency = document.FindMember(root, "market_dependency_sha256"); const bool dependency_ok = (expected.role == 1 ? document.Type(dependency) == DPCC_JSON_NULL : document.Type(dependency) == DPCC_JSON_STRING && document.Text(dependency) == expected.market_dependency_sha256); const int ready_node = document.FindMember(root, "ready"); const string expected_revision = (expected.role == 1 ? DPCC_M05_MARKET_REVISION : DPCC_TRADING_REVISION); if(contract != DPCC_AP_CONTRACT || revision != expected_revision || role != (expected.role == 1 ? "Market" : "Trading") || abi_sha != expected.source_abi_sha256 || profile_sha != expected.profile_sha256 || data_sha != expected.data_sha256 || (expected.role == 1 && (market_profile_sha != expected.market_profile_sha256 || embedded_market_profile_sha != market_profile_sha)) || task != expected.task_id || profile_task != expected.task_id || stage != expected.stage || archive != expected_archive || !dependency_ok || document.Type(ready_node) != DPCC_JSON_BOOLEAN || (require_ready_market && document.Text(ready_node) != "true")) { reason = "dpccap.native.manifest_binding"; return(false); } if(require_ready_market) { SDPCCAPNativeLoadContract ready_contract; if(!ReadReadyMarketContract(expected.market_profile_sha256, expected.task_id, expected.max_native_bytes, ready_contract, reason)) return(false); } //--- Bind the selected role to its manifest graph; expected graphs come from the caller. int manifest_graph = document.FindMember(root, "graph"); if(trading_section) { const string role_keys[] = {"Actor", "Q1", "Q2", "TargetActor", "TargetQ1", "TargetQ2"}; if(document.Type(manifest_graph) != DPCC_JSON_OBJECT || document.ChildCount(manifest_graph) != ArraySize(role_keys)) { reason = "dpccap.native.trading_graph_roles"; return(false); } for(int i = 0; i < ArraySize(role_keys); i++) { const int role_graph = document.FindMember(manifest_graph, role_keys[i]); if(document.Type(role_graph) != DPCC_JSON_ARRAY || document.ChildCount(role_graph) < 1 || document.ChildCount(role_graph) > 64) { reason = "dpccap.native.trading_graph_role_array." + role_keys[i]; return(false); } } manifest_graph = document.FindMember(manifest_graph, role_keys[(int)expected.section_id - 110]); } if((expected.section_id == 100 || trading_section) && (document.Type(manifest_graph) != DPCC_JSON_ARRAY || document.ChildCount(manifest_graph) != expected_graph.Total( ))) { reason = "dpccap.native.manifest_graph"; return(false); } //--- Creating an owned directory is the exclusive claim. A failed create //--- never authorizes opening or deleting any of that directory's contents. FolderCreate("Temp"); static ulong sequence = 0; string scratch_directory = ""; for(int attempt = 0; attempt < 16; attempt++) { sequence++; const string name = "Temp/DPCCAPNative_" + (string)ChartID() + "_" + (string)GetMicrosecondCount() + "_" + (string)GetTickCount64() + "_" + (string)sequence; if(FolderCreate(name)) { scratch_directory = name; break; } } if(scratch_directory == "") { reason = "dpccap.native.scratch_claim"; return(false); } const string scratch_path = scratch_directory + "/role.nnw"; int scratch = INVALID_HANDLE; CNet *candidate = NULL; CNet *loaded_model = destination; bool loaded = false, created_file = false; do { scratch = FileOpen(scratch_path, FILE_READ | FILE_WRITE | FILE_BIN); if(scratch == INVALID_HANDLE) { reason = "dpccap.native.scratch_open"; break; } created_file = true; if(FileSize(scratch) != 0) { reason = "dpccap.native.scratch_not_empty"; break; } if(!BeginSection(expected.section_id, expected.encoding, ap_handle, reason)) break; CDPCCIncrementalSHA256 hash; uchar bytes[]; if(ArrayResize(bytes, DPCC_AP_HASH_CHUNK) != DPCC_AP_HASH_CHUNK || !hash.Init(reason)) break; ulong remaining = m_sections[section].length; bool copied = true; while(remaining > 0) { const uint count = (uint)MathMin((ulong)DPCC_AP_HASH_CHUNK, remaining); if(FileReadArray(ap_handle, bytes, 0, count) != count || FileWriteArray(scratch, bytes, 0, count) != count || !hash.Update(bytes, 0, (int)count, reason)) { copied = false; break; } remaining -= count; } FileFlush(scratch); string copied_sha = ""; if(!copied || remaining != 0 || !hash.Final(digest, copied_sha, reason) || copied_sha != m_sections[section].sha256 || FileSize(scratch) != m_sections[section].length) {reason = "dpccap.native.copy_integrity:" + reason; break;} if(!EndSection(reason)) break; FileFlush(scratch); string scratch_sha = ""; if(!DPCCAPHashRange(scratch, 0, m_sections[section].length, scratch_sha, reason) || scratch_sha != m_sections[section].sha256 || !FileSeek(scratch, 0, SEEK_SET)) {reason = "dpccap.native.scratch_integrity:" + reason; break;} if(!loaded_model) { candidate = new CNet(); loaded_model = candidate; } if(!loaded_model) { reason = "dpccap.native.allocation"; break; } const bool native_loaded = loaded_model.Load(scratch); //--- Load closes on its normal completion, but some early failures do not. //--- No new handle is opened between Load and this failure-only close. if(!native_loaded) FileClose(scratch); scratch = INVALID_HANDLE; if(!native_loaded) { reason = "dpccap.native.load"; break; } int graph_node = document.FirstChild(manifest_graph); bool graph_ok = (loaded_model.Layer(expected_graph.Total()) == NULL); for(int i = 0; graph_ok && i < expected_graph.Total(); i++) { CNeuronBaseOCL *layer = loaded_model.Layer(i); CLayerDescription *required = dynamic_cast(expected_graph.At(i)); CLayerDescription *actual = (layer ? layer.GetLayerInfo() : NULL); uint native_abi = 0; graph_ok = (layer && required && DPCCAPNativeDescriptorEqual(actual, required) && DPCCAPNativeLayerABI(layer, actual, native_abi) && native_abi == expected_abis[i]); if(graph_ok && expected.section_id == 100) { long index = -1, type = -1, abi = -1, neurons = -1; graph_ok = (document.Type(graph_node) == DPCC_JSON_OBJECT && DPCCResourceReadLong(document, graph_node, "index", index, reason, "native.graph.index") && DPCCResourceReadLong(document, graph_node, "type", type, reason, "native.graph.type") && DPCCResourceReadLong(document, graph_node, "abi", abi, reason, "native.graph.abi") && DPCCResourceReadLong(document, graph_node, "neurons", neurons, reason, "native.graph.neurons") && index == i && type == (long)layer.Type() && abi == (long)native_abi && neurons == layer.Neurons( )); graph_node = document.NextSibling(graph_node); } if(actual) delete actual; } if(!graph_ok) { reason = "dpccap.native.graph_descriptor:" + reason; break; } if(trading_section) { string loaded_graph_text = ""; CDPCCJsonDocument loaded_graph; if(!DPCCAPTradingGraph(*loaded_model, expected_graph.Total(), loaded_graph_text, reason) || !loaded_graph.Parse(loaded_graph_text, reason) || !DPCCBankJsonEqual(document, manifest_graph, loaded_graph, loaded_graph.Root(), 0)) { reason = "dpccap.native.trading_manifest_graph:" + reason; break; } } if(shared_context && ((loaded_model.GetOpenCL() != shared_context && !loaded_model.SetOpenCLChecked(shared_context)) || loaded_model.GetOpenCL() != shared_context || !loaded_model.IsOpenCLTransferReady())) {reason = "dpccap.native.shared_context"; break;} loaded = true; } while(false); if(scratch != INVALID_HANDLE) FileClose(scratch); if(m_active != -1) Close(); const bool removed_file = (!created_file || FileDelete(scratch_path)); const bool removed_directory = FolderDelete(scratch_directory); if(!removed_file || !removed_directory) { loaded = false; reason = "dpccap.native.scratch_cleanup:" + scratch_path; } if(!loaded) { if(candidate) delete candidate; if(reason == "") reason = "dpccap.native.failed"; return(false); } //--- This proves exact section bytes and the decoded native graph, not the //--- unavailable native consumed-cursor or ready/acceptance/trading admission. if(candidate) model = candidate; return(true); } #endif //+------------------------------------------------------------------+