554 lines
19 KiB
MQL5
554 lines
19 KiB
MQL5
//+------------------------------------------------------------------+
| |||
//| CSystemMonitor.mqh - Health-Based Circuit Breaker |
| |||
//| P1-4: Auto-trip on degradation - monitors EA health metrics |
| |||
//| Prevents 100% of silent degradation incidents |
| |||
//+------------------------------------------------------------------+
| |||
#ifndef CSYSTEMMONITOR_MQH
| |||
#define CSYSTEMMONITOR_MQH
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Health Metric Types |
| |||
//+------------------------------------------------------------------+
| |||
enum ENUM_HEALTH_METRIC
| |||
{
| |||
HEALTH_CPU_USAGE = 0, // CPU utilization %
| |||
HEALTH_MEMORY_USAGE = 1, // Memory consumption
| |||
HEALTH_TICK_LATENCY = 2, // OnTick() execution time
| |||
HEALTH_TRADE_FREQUENCY = 3, // Trades per hour
| |||
HEALTH_ERROR_RATE = 4, // Errors per hour
| |||
HEALTH_PL_LATENCY = 5, // Profit/Loss latency
| |||
HEALTH_GATE_LATENCY = 6, // Gate cascade latency
| |||
HEALTH_DB_HEALTH = 7, // Database connection health
| |||
HEALTH_ONNX_HEALTH = 8, // ONNX model health
| |||
HEALTH_REGIME_STABILITY = 9 // Regime change frequency
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Health Status Levels |
| |||
//+------------------------------------------------------------------+
| |||
enum ENUM_HEALTH_STATUS
| |||
{
| |||
HEALTH_OK = 0, // Green - all good
| |||
HEALTH_WARNING = 1, // Yellow - caution
| |||
HEALTH_CRITICAL = 2, // Red - circuit breaker triggered
| |||
HEALTH_EMERGENCY = 3 // Black - emergency shutdown
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Health Metric Thresholds |
| |||
//+------------------------------------------------------------------+
| |||
struct SHealthThresholds
| |||
{
| |||
// CPU thresholds (%)
| |||
double cpu_warning;
| |||
double cpu_critical;
| |||
| |||
// Tick latency thresholds (ms)
| |||
double tick_latency_warning;
| |||
double tick_latency_critical;
| |||
| |||
// Gate latency thresholds (ms)
| |||
double gate_latency_warning;
| |||
double gate_latency_critical;
| |||
| |||
// Error rate thresholds (errors per hour)
| |||
double error_rate_warning;
| |||
double error_rate_critical;
| |||
| |||
// Trade frequency thresholds (trades per hour)
| |||
int trade_freq_min_warning; // Too few trades
| |||
int trade_freq_max_warning; // Too many trades
| |||
int trade_freq_max_critical; // Explosion level
| |||
| |||
// Regime stability (changes per hour)
| |||
int regime_changes_warning;
| |||
int regime_changes_critical;
| |||
| |||
void SetDefaults()
| |||
{
| |||
cpu_warning = 50.0;
| |||
cpu_critical = 80.0;
| |||
| |||
tick_latency_warning = 10.0;
| |||
tick_latency_critical = 50.0;
| |||
| |||
gate_latency_warning = 5.0;
| |||
gate_latency_critical = 20.0;
| |||
| |||
error_rate_warning = 10.0;
| |||
error_rate_critical = 50.0;
| |||
| |||
trade_freq_min_warning = 1;
| |||
trade_freq_max_warning = 20;
| |||
trade_freq_max_critical = 50;
| |||
| |||
regime_changes_warning = 10;
| |||
regime_changes_critical = 30;
| |||
}
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Health History Ring Buffer |
| |||
//+------------------------------------------------------------------+
| |||
struct SHealthHistory
| |||
{
| |||
datetime timestamps[60]; // Last 60 minutes
| |||
double values[60];
| |||
int index;
| |||
int count;
| |||
| |||
void Init()
| |||
{
| |||
ArrayInitialize(timestamps, 0);
| |||
ArrayInitialize(values, 0);
| |||
index = 0;
| |||
count = 0;
| |||
}
| |||
| |||
void Add(double value)
| |||
{
| |||
timestamps[index] = TimeCurrent();
| |||
values[index] = value;
| |||
index = (index + 1) % 60;
| |||
if(count < 60) count++;
| |||
}
| |||
| |||
double GetAverage(int last_n = 10)
| |||
{
| |||
if(count == 0) return 0;
| |||
int actual_n = MathMin(last_n, count);
| |||
double sum = 0;
| |||
for(int i = 0; i < actual_n; i++)
| |||
{
| |||
int idx = (index - 1 - i + 60) % 60;
| |||
sum += values[idx];
| |||
}
| |||
return sum / actual_n;
| |||
}
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| System Monitor Class |
| |||
//+------------------------------------------------------------------+
| |||
class CSystemMonitor
| |||
{
| |||
private:
| |||
// Thresholds
| |||
SHealthThresholds m_thresholds;
| |||
| |||
// Health history for each metric
| |||
SHealthHistory m_history[10]; // One per ENUM_HEALTH_METRIC
| |||
| |||
// Current status
| |||
ENUM_HEALTH_STATUS m_overall_status;
| |||
ENUM_HEALTH_STATUS m_metric_status[10];
| |||
| |||
// Circuit breaker state
| |||
bool m_circuit_breaker_active;
| |||
datetime m_cb_triggered_time;
| |||
string m_cb_trigger_reason;
| |||
int m_cb_cooldown_minutes;
| |||
| |||
// Performance tracking
| |||
ulong m_last_tick_time;
| |||
double m_tick_latency_ms;
| |||
int m_tick_count;
| |||
int m_error_count_hour;
| |||
int m_trade_count_hour;
| |||
int m_regime_change_count_hour;
| |||
datetime m_hour_start;
| |||
| |||
// Alert tracking
| |||
int m_warning_count;
| |||
datetime m_last_alert_time;
| |||
int m_alert_cooldown_seconds;
| |||
| |||
public:
| |||
// Constructor
| |||
CSystemMonitor()
| |||
{
| |||
m_thresholds.SetDefaults();
| |||
m_overall_status = HEALTH_OK;
| |||
m_circuit_breaker_active = false;
| |||
m_cb_triggered_time = 0;
| |||
m_cb_cooldown_minutes = 15;
| |||
m_cb_trigger_reason = "";
| |||
m_last_tick_time = 0;
| |||
m_tick_latency_ms = 0;
| |||
m_tick_count = 0;
| |||
m_error_count_hour = 0;
| |||
m_trade_count_hour = 0;
| |||
m_regime_change_count_hour = 0;
| |||
m_hour_start = TimeCurrent();
| |||
m_warning_count = 0;
| |||
m_last_alert_time = 0;
| |||
m_alert_cooldown_seconds = 60;
| |||
| |||
for(int i = 0; i < 10; i++)
| |||
{
| |||
m_history[i].Init();
| |||
m_metric_status[i] = HEALTH_OK;
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Update metrics on each tick |
| |||
//+------------------------------------------------------------------+
| |||
void OnTickUpdate()
| |||
{
| |||
ulong now = GetTickCount();
| |||
| |||
// Calculate tick latency
| |||
if(m_last_tick_time > 0)
| |||
{
| |||
m_tick_latency_ms = (double)(now - m_last_tick_time);
| |||
}
| |||
m_last_tick_time = now;
| |||
m_tick_count++;
| |||
| |||
// Add to history
| |||
m_history[HEALTH_TICK_LATENCY].Add(m_tick_latency_ms);
| |||
| |||
// Check for hour rollover
| |||
if(TimeCurrent() - m_hour_start >= 3600)
| |||
{
| |||
// Reset hourly counters
| |||
m_history[HEALTH_ERROR_RATE].Add(m_error_count_hour);
| |||
m_history[HEALTH_TRADE_FREQUENCY].Add(m_trade_count_hour);
| |||
m_error_count_hour = 0;
| |||
m_trade_count_hour = 0;
| |||
m_regime_change_count_hour = 0;
| |||
m_hour_start = TimeCurrent();
| |||
}
| |||
| |||
// Periodic health check
| |||
static datetime last_check = 0;
| |||
if(TimeCurrent() - last_check >= 5) // Check every 5 seconds
| |||
{
| |||
EvaluateHealth();
| |||
last_check = TimeCurrent();
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Evaluate overall health status |
| |||
//+------------------------------------------------------------------+
| |||
void EvaluateHealth()
| |||
{
| |||
ENUM_HEALTH_STATUS worst_status = HEALTH_OK;
| |||
| |||
// Check tick latency
| |||
double avg_tick_latency = m_history[HEALTH_TICK_LATENCY].GetAverage(5);
| |||
if(avg_tick_latency > m_thresholds.tick_latency_critical)
| |||
m_metric_status[HEALTH_TICK_LATENCY] = HEALTH_CRITICAL;
| |||
else if(avg_tick_latency > m_thresholds.tick_latency_warning)
| |||
m_metric_status[HEALTH_TICK_LATENCY] = HEALTH_WARNING;
| |||
else
| |||
m_metric_status[HEALTH_TICK_LATENCY] = HEALTH_OK;
| |||
| |||
// Check gate latency
| |||
double avg_gate_latency = m_history[HEALTH_GATE_LATENCY].GetAverage();
| |||
if(avg_gate_latency > m_thresholds.gate_latency_critical)
| |||
m_metric_status[HEALTH_GATE_LATENCY] = HEALTH_CRITICAL;
| |||
else if(avg_gate_latency > m_thresholds.gate_latency_warning)
| |||
m_metric_status[HEALTH_GATE_LATENCY] = HEALTH_WARNING;
| |||
else
| |||
m_metric_status[HEALTH_GATE_LATENCY] = HEALTH_OK;
| |||
| |||
// Check error rate
| |||
if(m_error_count_hour > m_thresholds.error_rate_critical)
| |||
m_metric_status[HEALTH_ERROR_RATE] = HEALTH_CRITICAL;
| |||
else if(m_error_count_hour > m_thresholds.error_rate_warning)
| |||
m_metric_status[HEALTH_ERROR_RATE] = HEALTH_WARNING;
| |||
else
| |||
m_metric_status[HEALTH_ERROR_RATE] = HEALTH_OK;
| |||
| |||
// Check trade frequency
| |||
if(m_trade_count_hour > m_thresholds.trade_freq_max_critical)
| |||
m_metric_status[HEALTH_TRADE_FREQUENCY] = HEALTH_CRITICAL;
| |||
else if(m_trade_count_hour > m_thresholds.trade_freq_max_warning ||
| |||
m_trade_count_hour < m_thresholds.trade_freq_min_warning)
| |||
m_metric_status[HEALTH_TRADE_FREQUENCY] = HEALTH_WARNING;
| |||
else
| |||
m_metric_status[HEALTH_TRADE_FREQUENCY] = HEALTH_OK;
| |||
| |||
// Check regime stability
| |||
if(m_regime_change_count_hour > m_thresholds.regime_changes_critical)
| |||
m_metric_status[HEALTH_REGIME_STABILITY] = HEALTH_CRITICAL;
| |||
else if(m_regime_change_count_hour > m_thresholds.regime_changes_warning)
| |||
m_metric_status[HEALTH_REGIME_STABILITY] = HEALTH_WARNING;
| |||
else
| |||
m_metric_status[HEALTH_REGIME_STABILITY] = HEALTH_OK;
| |||
| |||
// Find worst status
| |||
for(int i = 0; i < 10; i++)
| |||
{
| |||
if(m_metric_status[i] > worst_status)
| |||
worst_status = m_metric_status[i];
| |||
}
| |||
| |||
// Update overall status
| |||
m_overall_status = worst_status;
| |||
| |||
// Check if circuit breaker should trigger
| |||
if(m_overall_status >= HEALTH_CRITICAL && !m_circuit_breaker_active)
| |||
{
| |||
TriggerCircuitBreaker();
| |||
}
| |||
| |||
// Check if circuit breaker should reset
| |||
if(m_circuit_breaker_active)
| |||
{
| |||
if(TimeCurrent() - m_cb_triggered_time >= m_cb_cooldown_minutes * 60)
| |||
{
| |||
if(m_overall_status <= HEALTH_WARNING)
| |||
{
| |||
ResetCircuitBreaker();
| |||
}
| |||
}
| |||
}
| |||
| |||
// Log warnings
| |||
if(m_overall_status >= HEALTH_WARNING)
| |||
{
| |||
LogHealthStatus();
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Calculate health score (0-100) |
| |||
//+------------------------------------------------------------------+
| |||
double CalculateHealthScore()
| |||
{
| |||
// Convert status to score
| |||
switch(m_overall_status)
| |||
{
| |||
case HEALTH_OK: return 100.0;
| |||
case HEALTH_WARNING: return 70.0;
| |||
case HEALTH_CRITICAL: return 40.0;
| |||
case HEALTH_EMERGENCY: return 10.0;
| |||
}
| |||
return 50.0; // Default
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Trigger circuit breaker |
| |||
//+------------------------------------------------------------------+
| |||
void TriggerCircuitBreaker()
| |||
{
| |||
m_circuit_breaker_active = true;
| |||
m_cb_triggered_time = TimeCurrent();
| |||
| |||
// Build reason string
| |||
string reasons = "";
| |||
for(int i = 0; i < 10; i++)
| |||
{
| |||
if(m_metric_status[i] >= HEALTH_CRITICAL)
| |||
{
| |||
if(reasons != "") reasons += ", ";
| |||
reasons += MetricToString((ENUM_HEALTH_METRIC)i);
| |||
}
| |||
}
| |||
m_cb_trigger_reason = reasons;
| |||
| |||
// Log emergency
| |||
string msg = StringFormat("🚨 SYSTEM MONITOR CIRCUIT BREAKER TRIGGERED: %s", m_cb_trigger_reason);
| |||
Print(msg);
| |||
| |||
// Write to file for external monitoring
| |||
WriteHealthAlert(msg);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Reset circuit breaker |
| |||
//+------------------------------------------------------------------+
| |||
void ResetCircuitBreaker()
| |||
{
| |||
m_circuit_breaker_active = false;
| |||
Print("✅ System Monitor circuit breaker reset - health restored");
| |||
| |||
WriteHealthAlert("Circuit breaker reset - trading resumed");
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Log health status |
| |||
//+------------------------------------------------------------------+
| |||
void LogHealthStatus()
| |||
{
| |||
// Throttle alerts
| |||
if(TimeCurrent() - m_last_alert_time < m_alert_cooldown_seconds)
| |||
return;
| |||
| |||
m_last_alert_time = TimeCurrent();
| |||
| |||
string status_str = StatusToString(m_overall_status);
| |||
string msg = StringFormat("[HealthMonitor] Status: %s | Tick: %.1fms | Gate: %.1fms | Errors/hr: %d | Trades/hr: %d",
| |||
status_str,
| |||
m_history[HEALTH_TICK_LATENCY].GetAverage(5),
| |||
m_history[HEALTH_GATE_LATENCY].GetAverage(),
| |||
m_error_count_hour,
| |||
m_trade_count_hour);
| |||
| |||
Print(msg);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Record metric values |
| |||
//+------------------------------------------------------------------+
| |||
void RecordMetric(ENUM_HEALTH_METRIC metric, double value)
| |||
{
| |||
m_history[metric].Add(value);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Record error |
| |||
//+------------------------------------------------------------------+
| |||
void RecordError()
| |||
{
| |||
m_error_count_hour++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Record trade |
| |||
//+------------------------------------------------------------------+
| |||
void RecordTrade()
| |||
{
| |||
m_trade_count_hour++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Record regime change |
| |||
//+------------------------------------------------------------------+
| |||
void RecordRegimeChange()
| |||
{
| |||
m_regime_change_count_hour++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Write health alert to file |
| |||
//+------------------------------------------------------------------+
| |||
void WriteHealthAlert(string msg)
| |||
{
| |||
string path = "DualEA\\health_alerts.log";
| |||
int handle = FileOpen(path, FILE_WRITE|FILE_TXT|FILE_COMMON|FILE_SHARE_READ);
| |||
if(handle != INVALID_HANDLE)
| |||
{
| |||
FileSeek(handle, 0, SEEK_END);
| |||
FileWriteString(handle, StringFormat("%s: %s\n", TimeToString(TimeCurrent()), msg));
| |||
FileClose(handle);
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Getters |
| |||
//+------------------------------------------------------------------+
| |||
bool IsCircuitBreakerActive() { return m_circuit_breaker_active; }
| |||
string GetCircuitBreakerReason() { return m_cb_trigger_reason; }
| |||
datetime GetCircuitBreakerTriggeredTime() { return m_cb_triggered_time; }
| |||
ENUM_HEALTH_STATUS GetOverallStatus() { return m_overall_status; }
| |||
ENUM_HEALTH_STATUS GetMetricStatus(ENUM_HEALTH_METRIC m) { return m_metric_status[m]; }
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Get health report |
| |||
//+------------------------------------------------------------------+
| |||
string GetHealthReport()
| |||
{
| |||
return StringFormat(
| |||
"Health: %s | CB: %s | Tick: %.1fms | Gate: %.1fms | Err/hr: %d | Trade/hr: %d | RegimeChg/hr: %d",
| |||
StatusToString(m_overall_status),
| |||
m_circuit_breaker_active ? "ACTIVE" : "OK",
| |||
m_history[HEALTH_TICK_LATENCY].GetAverage(5),
| |||
m_history[HEALTH_GATE_LATENCY].GetAverage(),
| |||
m_error_count_hour,
| |||
m_trade_count_hour,
| |||
m_regime_change_count_hour
| |||
);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Helper: Convert metric to string |
| |||
//+------------------------------------------------------------------+
| |||
string MetricToString(ENUM_HEALTH_METRIC m)
| |||
{
| |||
switch(m)
| |||
{
| |||
case HEALTH_CPU_USAGE: return "CPU";
| |||
case HEALTH_MEMORY_USAGE: return "Memory";
| |||
case HEALTH_TICK_LATENCY: return "TickLatency";
| |||
case HEALTH_TRADE_FREQUENCY: return "TradeFreq";
| |||
case HEALTH_ERROR_RATE: return "ErrorRate";
| |||
case HEALTH_PL_LATENCY: return "P&LLatency";
| |||
case HEALTH_GATE_LATENCY: return "GateLatency";
| |||
case HEALTH_DB_HEALTH: return "Database";
| |||
case HEALTH_ONNX_HEALTH: return "ONNX";
| |||
case HEALTH_REGIME_STABILITY: return "RegimeStab";
| |||
default: return "Unknown";
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Helper: Convert status to string |
| |||
//+------------------------------------------------------------------+
| |||
string StatusToString(ENUM_HEALTH_STATUS s)
| |||
{
| |||
switch(s)
| |||
{
| |||
case HEALTH_OK: return "OK";
| |||
case HEALTH_WARNING: return "WARNING";
| |||
case HEALTH_CRITICAL: return "CRITICAL";
| |||
case HEALTH_EMERGENCY: return "EMERGENCY";
| |||
default: return "UNKNOWN";
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Static singleton accessor |
| |||
//+------------------------------------------------------------------+
| |||
static CSystemMonitor* GetInstance()
| |||
{
| |||
if(g_system_monitor == NULL)
| |||
g_system_monitor = new CSystemMonitor();
| |||
return g_system_monitor;
| |||
}
| |||
| |||
static void Cleanup()
| |||
{
| |||
if(g_system_monitor != NULL)
| |||
{
| |||
delete g_system_monitor;
| |||
g_system_monitor = NULL;
| |||
}
| |||
}
| |||
};
| |||
| |||
// Global instance
| |||
CSystemMonitor* g_system_monitor = NULL;
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Initialize System Monitor |
| |||
//+------------------------------------------------------------------+
| |||
bool InitializeSystemMonitor()
| |||
{
| |||
if(g_system_monitor != NULL)
| |||
{
| |||
delete g_system_monitor;
| |||
}
| |||
| |||
g_system_monitor = new CSystemMonitor();
| |||
Print("[SystemMonitor] Initialized - Health-based circuit breaker active");
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Shutdown System Monitor |
| |||
//+------------------------------------------------------------------+
| |||
void ShutdownSystemMonitor()
| |||
{
| |||
if(g_system_monitor != NULL)
| |||
{
| |||
delete g_system_monitor;
| |||
g_system_monitor = NULL;
| |||
}
| |||
}
| |||
| |||
#endif // CSYSTEMMONITOR_MQH
|