628 lines
20 KiB
MQL5
628 lines
20 KiB
MQL5
//+------------------------------------------------------------------+
| |||
//| CErrorRecovery.mqh - Centralized Error Recovery System |
| |||
//| Provides consistent error handling, retry logic, and recovery |
| |||
//+------------------------------------------------------------------+
| |||
#ifndef CERRORRECOVERY_MQH
| |||
#define CERRORRECOVERY_MQH
| |||
| |||
#include "CArchitectureConfig.mqh"
| |||
| |||
// Error severity levels
| |||
enum ENUM_ERROR_SEVERITY
| |||
{
| |||
ERR_SEVERITY_LOW = 0, // Log only, continue
| |||
ERR_SEVERITY_MEDIUM = 1, // Degrade gracefully
| |||
ERR_SEVERITY_HIGH = 2, // Block operations temporarily
| |||
ERR_SEVERITY_CRITICAL = 3 // Shutdown required
| |||
};
| |||
| |||
// Error categories for classification
| |||
enum ENUM_ERROR_CATEGORY
| |||
{
| |||
ERR_CAT_FILE_IO = 0,
| |||
ERR_CAT_NETWORK = 1,
| |||
ERR_CAT_MEMORY = 2,
| |||
ERR_CAT_TRADE = 3,
| |||
ERR_CAT_SYSTEM = 4,
| |||
ERR_CAT_UNKNOWN = 5
| |||
};
| |||
| |||
// Recovery action types
| |||
enum ENUM_RECOVERY_ACTION
| |||
{
| |||
RECOVERY_NONE = 0, // No action, just log
| |||
RECOVERY_RETRY = 1, // Retry operation
| |||
RECOVERY_FALLBACK = 2, // Use fallback mode
| |||
RECOVERY_RESET = 3, // Reset and reinitialize
| |||
RECOVERY_SHUTDOWN = 4 // Graceful shutdown
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Error Record Structure |
| |||
//+------------------------------------------------------------------+
| |||
struct SErrorRecord
| |||
{
| |||
int error_code;
| |||
ENUM_ERROR_CATEGORY category;
| |||
ENUM_ERROR_SEVERITY severity;
| |||
string context;
| |||
string message;
| |||
datetime timestamp;
| |||
int retry_count;
| |||
bool recovered;
| |||
| |||
SErrorRecord()
| |||
{
| |||
error_code = 0;
| |||
category = ERR_CAT_UNKNOWN;
| |||
severity = ERR_SEVERITY_LOW;
| |||
context = "";
| |||
message = "";
| |||
timestamp = 0;
| |||
retry_count = 0;
| |||
recovered = false;
| |||
}
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Error Recovery Manager |
| |||
//+------------------------------------------------------------------+
| |||
class CErrorRecovery
| |||
{
| |||
private:
| |||
static CErrorRecovery* s_instance;
| |||
| |||
// Error history for analysis
| |||
SErrorRecord m_error_history[];
| |||
int m_history_count;
| |||
int m_max_history;
| |||
| |||
// Retry configuration
| |||
int m_max_retries;
| |||
int m_retry_delay_ms;
| |||
| |||
// Circuit breaker state
| |||
int m_consecutive_errors;
| |||
datetime m_circuit_breaker_until;
| |||
int m_circuit_breaker_threshold;
| |||
int m_circuit_breaker_cooldown_sec;
| |||
| |||
// Category-specific thresholds
| |||
int m_category_errors[];
| |||
datetime m_category_last_error[];
| |||
| |||
// Statistics
| |||
int m_total_errors;
| |||
int m_recovered_errors;
| |||
int m_failed_errors;
| |||
| |||
// Logging
| |||
bool m_verbose;
| |||
string m_log_prefix;
| |||
| |||
public:
| |||
CErrorRecovery()
| |||
{
| |||
m_history_count = 0;
| |||
m_max_history = 100;
| |||
ArrayResize(m_error_history, m_max_history);
| |||
| |||
m_max_retries = 3;
| |||
m_retry_delay_ms = 100;
| |||
| |||
m_consecutive_errors = 0;
| |||
m_circuit_breaker_until = 0;
| |||
m_circuit_breaker_threshold = 5;
| |||
m_circuit_breaker_cooldown_sec = 60;
| |||
| |||
ArrayResize(m_category_errors, 6);
| |||
ArrayResize(m_category_last_error, 6);
| |||
for(int i = 0; i < 6; i++)
| |||
{
| |||
m_category_errors[i] = 0;
| |||
m_category_last_error[i] = 0;
| |||
}
| |||
| |||
m_total_errors = 0;
| |||
m_recovered_errors = 0;
| |||
m_failed_errors = 0;
| |||
| |||
m_verbose = false;
| |||
m_log_prefix = "[ErrorRecovery]";
| |||
}
| |||
| |||
~CErrorRecovery()
| |||
{
| |||
// Cleanup
| |||
}
| |||
| |||
static CErrorRecovery* Instance()
| |||
{
| |||
if(s_instance == NULL)
| |||
s_instance = new CErrorRecovery();
| |||
return s_instance;
| |||
}
| |||
| |||
static void Destroy()
| |||
{
| |||
if(s_instance != NULL)
| |||
{
| |||
delete s_instance;
| |||
s_instance = NULL;
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Configure recovery behavior |
| |||
//+------------------------------------------------------------------+
| |||
void SetMaxRetries(int max_retries) { m_max_retries = max_retries; }
| |||
void SetRetryDelay(int delay_ms) { m_retry_delay_ms = delay_ms; }
| |||
void SetCircuitBreakerThreshold(int threshold) { m_circuit_breaker_threshold = threshold; }
| |||
void SetCircuitBreakerCooldown(int cooldown_sec) { m_circuit_breaker_cooldown_sec = cooldown_sec; }
| |||
void SetVerbose(bool verbose) { m_verbose = verbose; }
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Main error handling entry point |
| |||
//+------------------------------------------------------------------+
| |||
bool HandleError(int error_code, const string context, const string message)
| |||
{
| |||
// Clear error to prevent contamination
| |||
ResetLastError();
| |||
| |||
// Check if in circuit breaker
| |||
if(IsCircuitBreakerActive())
| |||
{
| |||
Log("Circuit breaker active, operation blocked", ERR_SEVERITY_HIGH);
| |||
return false;
| |||
}
| |||
| |||
// Classify error
| |||
ENUM_ERROR_CATEGORY category = ClassifyError(error_code);
| |||
ENUM_ERROR_SEVERITY severity = DetermineSeverity(error_code, category);
| |||
| |||
// Create error record
| |||
SErrorRecord record;
| |||
record.error_code = error_code;
| |||
record.category = category;
| |||
record.severity = severity;
| |||
record.context = context;
| |||
record.message = message;
| |||
record.timestamp = TimeCurrent();
| |||
record.retry_count = 0;
| |||
record.recovered = false;
| |||
| |||
// Add to history
| |||
AddToHistory(record);
| |||
| |||
// Update statistics
| |||
m_total_errors++;
| |||
m_consecutive_errors++;
| |||
m_category_errors[category]++;
| |||
m_category_last_error[category] = TimeCurrent();
| |||
| |||
// Check circuit breaker
| |||
if(m_consecutive_errors >= m_circuit_breaker_threshold)
| |||
{
| |||
ActivateCircuitBreaker();
| |||
}
| |||
| |||
// Determine recovery action
| |||
ENUM_RECOVERY_ACTION action = DetermineRecoveryAction(error_code, category, severity);
| |||
| |||
// Execute recovery
| |||
bool recovered = ExecuteRecovery(action, record);
| |||
| |||
if(recovered)
| |||
{
| |||
m_recovered_errors++;
| |||
m_consecutive_errors = 0;
| |||
record.recovered = true;
| |||
UpdateHistoryRecord(record);
| |||
Log("Error recovered: " + message, ERR_SEVERITY_LOW);
| |||
}
| |||
else
| |||
{
| |||
m_failed_errors++;
| |||
Log("Error recovery failed: " + message, severity);
| |||
}
| |||
| |||
return recovered;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Classify error code to category |
| |||
//+------------------------------------------------------------------+
| |||
ENUM_ERROR_CATEGORY ClassifyError(int error_code)
| |||
{
| |||
switch(error_code)
| |||
{
| |||
// File I/O errors
| |||
case 5001: case 5002: case 5003: case 5004:
| |||
case 5005: case 5006: case 5007: case 5008:
| |||
case 5010: case 5011: case 5012: case 5013:
| |||
case 5014: case 5015: case 5016: case 5017:
| |||
case 5020:
| |||
return ERR_CAT_FILE_IO;
| |||
| |||
// Network/connection errors
| |||
case 5200: case 5201: case 5202: case 5203:
| |||
case 6750: case 6751: case 6752:
| |||
return ERR_CAT_NETWORK;
| |||
| |||
// Memory errors
| |||
case 4001: case 4002: case 4003:
| |||
return ERR_CAT_MEMORY;
| |||
| |||
// Trading errors
| |||
case 10004: case 10006: case 10007: case 10009:
| |||
case 10010: case 10013: case 10014: case 10015:
| |||
case 10016: case 10021: case 10022: case 10023:
| |||
case 10024: case 10025: case 10026: case 10027:
| |||
return ERR_CAT_TRADE;
| |||
| |||
// System errors
| |||
case 1: case 2: case 3: case 4:
| |||
return ERR_CAT_SYSTEM;
| |||
| |||
default:
| |||
return ERR_CAT_UNKNOWN;
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Determine error severity |
| |||
//+------------------------------------------------------------------+
| |||
ENUM_ERROR_SEVERITY DetermineSeverity(int error_code, ENUM_ERROR_CATEGORY category)
| |||
{
| |||
// Critical errors
| |||
switch(error_code)
| |||
{
| |||
case 4001: // ERR_WRONG_FUNCTION_POINTER
| |||
case 4002: // ERR_ARRAY_INDEX_OUT_OF_RANGE
| |||
case 2: // ERR_COMMON_ERROR
| |||
case 10026: // ERR_TRADE_MODIFY_DENIED
| |||
return ERR_SEVERITY_CRITICAL;
| |||
| |||
// High severity
| |||
case 10007: // ERR_TRADE_NOT_ALLOWED
| |||
case 10014: // ERR_TRADE_INVALID_STOPS
| |||
case 5001: // ERR_FILE_TOO_LARGE
| |||
case 5200: // ERR_INTERNET_CONNECTION_CLOSED
| |||
return ERR_SEVERITY_HIGH;
| |||
| |||
// Medium severity
| |||
case 10004: // ERR_REQUOTE
| |||
case 10006: // ERR_NO_PRICES
| |||
case 5004: // ERR_FILE_NOT_OPEN
| |||
case 6750: // ERR_SOCKET_INIT_FAILED
| |||
return ERR_SEVERITY_MEDIUM;
| |||
| |||
// Low severity (transient)
| |||
case 10009: // ERR_TRADE_ERROR
| |||
case 10010: // ERR_TRADE_TIMEOUT
| |||
case 5003: // ERR_FILE_INVALID_HANDLE
| |||
return ERR_SEVERITY_LOW;
| |||
}
| |||
| |||
// Category-based fallback
| |||
if(category == ERR_CAT_MEMORY)
| |||
return ERR_SEVERITY_HIGH;
| |||
if(category == ERR_CAT_SYSTEM)
| |||
return ERR_SEVERITY_HIGH;
| |||
| |||
return ERR_SEVERITY_MEDIUM;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Determine recovery action |
| |||
//+------------------------------------------------------------------+
| |||
ENUM_RECOVERY_ACTION DetermineRecoveryAction(int error_code, ENUM_ERROR_CATEGORY category, ENUM_ERROR_SEVERITY severity)
| |||
{
| |||
// Never retry critical errors
| |||
if(severity == ERR_SEVERITY_CRITICAL)
| |||
return RECOVERY_SHUTDOWN;
| |||
| |||
// Network errors: retry with fallback
| |||
if(category == ERR_CAT_NETWORK)
| |||
return RECOVERY_RETRY;
| |||
| |||
// File I/O errors: retry then fallback
| |||
if(category == ERR_CAT_FILE_IO)
| |||
return RECOVERY_RETRY;
| |||
| |||
// Trading errors: specific handling
| |||
if(category == ERR_CAT_TRADE)
| |||
{
| |||
switch(error_code)
| |||
{
| |||
case 10004: // Requote - retry immediately
| |||
case 10006: // No prices - retry later
| |||
return RECOVERY_RETRY;
| |||
| |||
case 10007: // Not allowed - check permissions then fallback
| |||
return RECOVERY_FALLBACK;
| |||
| |||
default:
| |||
return RECOVERY_NONE;
| |||
}
| |||
}
| |||
| |||
// Default: no recovery action
| |||
return RECOVERY_NONE;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Execute recovery action |
| |||
//+------------------------------------------------------------------+
| |||
bool ExecuteRecovery(ENUM_RECOVERY_ACTION action, SErrorRecord &record)
| |||
{
| |||
switch(action)
| |||
{
| |||
case RECOVERY_NONE:
| |||
return true; // Nothing to do
| |||
| |||
case RECOVERY_RETRY:
| |||
return ExecuteRetry(record);
| |||
| |||
case RECOVERY_FALLBACK:
| |||
return ExecuteFallback(record);
| |||
| |||
case RECOVERY_RESET:
| |||
return ExecuteReset(record);
| |||
| |||
case RECOVERY_SHUTDOWN:
| |||
ExecuteShutdown(record);
| |||
return false; // Cannot recover from shutdown
| |||
| |||
default:
| |||
return false;
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Retry logic with exponential backoff |
| |||
//+------------------------------------------------------------------+
| |||
bool ExecuteRetry(SErrorRecord &record)
| |||
{
| |||
if(record.retry_count >= m_max_retries)
| |||
{
| |||
Log("Max retries exceeded for error " + IntegerToString(record.error_code), ERR_SEVERITY_MEDIUM);
| |||
return false;
| |||
}
| |||
| |||
record.retry_count++;
| |||
| |||
// Calculate delay with exponential backoff
| |||
int delay = m_retry_delay_ms * (1 << (record.retry_count - 1));
| |||
if(delay > 5000) delay = 5000; // Cap at 5 seconds
| |||
| |||
Log("Retry " + IntegerToString(record.retry_count) + "/" + IntegerToString(m_max_retries) +
| |||
" after " + IntegerToString(delay) + "ms for " + record.context, ERR_SEVERITY_LOW);
| |||
| |||
// Use Sleep only in non-tick contexts
| |||
// In OnTick, this should be handled asynchronously
| |||
// For now, we return false to indicate retry needed
| |||
return false; // Caller must handle actual retry
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Fallback mode execution |
| |||
//+------------------------------------------------------------------+
| |||
bool ExecuteFallback(SErrorRecord &record)
| |||
{
| |||
Log("Activating fallback mode for " + record.context, ERR_SEVERITY_MEDIUM);
| |||
| |||
// Fallback behaviors by category
| |||
switch(record.category)
| |||
{
| |||
case ERR_CAT_FILE_IO:
| |||
// Switch to memory buffer mode
| |||
return true;
| |||
| |||
case ERR_CAT_NETWORK:
| |||
// Use cached data
| |||
return true;
| |||
| |||
case ERR_CAT_TRADE:
| |||
// Switch to shadow/simulation mode
| |||
return true;
| |||
| |||
default:
| |||
return false;
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Reset and reinitialize |
| |||
//+------------------------------------------------------------------+
| |||
bool ExecuteReset(SErrorRecord &record)
| |||
{
| |||
Log("Executing system reset for " + record.context, ERR_SEVERITY_HIGH);
| |||
| |||
// Clear error state
| |||
ResetLastError();
| |||
| |||
// Reset counters
| |||
m_consecutive_errors = 0;
| |||
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Graceful shutdown |
| |||
//+------------------------------------------------------------------+
| |||
void ExecuteShutdown(SErrorRecord &record)
| |||
{
| |||
Log("CRITICAL: Executing graceful shutdown - " + record.message, ERR_SEVERITY_CRITICAL);
| |||
| |||
// Log final statistics
| |||
string stats = StringFormat("Total errors: %d, Recovered: %d, Failed: %d",
| |||
m_total_errors, m_recovered_errors, m_failed_errors);
| |||
Log(stats, ERR_SEVERITY_CRITICAL);
| |||
| |||
// In real implementation, this would trigger EA shutdown
| |||
// ExpertRemove(); // Uncomment when ready for production
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Circuit breaker management |
| |||
//+------------------------------------------------------------------+
| |||
bool IsCircuitBreakerActive()
| |||
{
| |||
if(m_circuit_breaker_until == 0)
| |||
return false;
| |||
| |||
if(TimeCurrent() < m_circuit_breaker_until)
| |||
{
| |||
int remaining = (int)(m_circuit_breaker_until - TimeCurrent());
| |||
if(m_verbose && remaining % 10 == 0) // Log every 10 seconds
| |||
Log("Circuit breaker active: " + IntegerToString(remaining) + "s remaining", ERR_SEVERITY_MEDIUM);
| |||
return true;
| |||
}
| |||
| |||
// Circuit breaker expired
| |||
m_circuit_breaker_until = 0;
| |||
m_consecutive_errors = 0;
| |||
Log("Circuit breaker deactivated", ERR_SEVERITY_LOW);
| |||
return false;
| |||
}
| |||
| |||
void ActivateCircuitBreaker()
| |||
{
| |||
m_circuit_breaker_until = TimeCurrent() + m_circuit_breaker_cooldown_sec;
| |||
Log("Circuit breaker ACTIVATED for " + IntegerToString(m_circuit_breaker_cooldown_sec) + "s", ERR_SEVERITY_HIGH);
| |||
}
| |||
| |||
void DeactivateCircuitBreaker()
| |||
{
| |||
m_circuit_breaker_until = 0;
| |||
m_consecutive_errors = 0;
| |||
Log("Circuit breaker manually deactivated", ERR_SEVERITY_LOW);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| History management |
| |||
//+------------------------------------------------------------------+
| |||
void AddToHistory(SErrorRecord &record)
| |||
{
| |||
// Shift existing records
| |||
for(int i = m_history_count - 1; i > 0; i--)
| |||
{
| |||
m_error_history[i] = m_error_history[i - 1];
| |||
}
| |||
| |||
// Add new record at start
| |||
m_error_history[0] = record;
| |||
| |||
if(m_history_count < m_max_history)
| |||
m_history_count++;
| |||
}
| |||
| |||
void UpdateHistoryRecord(SErrorRecord &record)
| |||
{
| |||
// Find and update record
| |||
for(int i = 0; i < m_history_count; i++)
| |||
{
| |||
if(m_error_history[i].timestamp == record.timestamp &&
| |||
m_error_history[i].error_code == record.error_code)
| |||
{
| |||
m_error_history[i] = record;
| |||
break;
| |||
}
| |||
}
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Statistics and reporting |
| |||
//+------------------------------------------------------------------+
| |||
string GetErrorStats()
| |||
{
| |||
string stats = StringFormat(
| |||
"=== Error Recovery Statistics ===\n" +
| |||
"Total Errors: %d\n" +
| |||
"Recovered: %d (%.1f%%)\n" +
| |||
"Failed: %d (%.1f%%)\n" +
| |||
"Consecutive Errors: %d\n" +
| |||
"Circuit Breaker: %s\n" +
| |||
"History: %d/%d records",
| |||
m_total_errors,
| |||
m_recovered_errors,
| |||
(m_total_errors > 0 ? 100.0 * m_recovered_errors / m_total_errors : 0),
| |||
m_failed_errors,
| |||
(m_total_errors > 0 ? 100.0 * m_failed_errors / m_total_errors : 0),
| |||
m_consecutive_errors,
| |||
(IsCircuitBreakerActive() ? "ACTIVE" : "inactive"),
| |||
m_history_count, m_max_history
| |||
);
| |||
| |||
return stats;
| |||
}
| |||
| |||
string GetRecentErrors(int count = 10)
| |||
{
| |||
string report = "=== Recent Errors ===\n";
| |||
int show = MathMin(count, m_history_count);
| |||
| |||
for(int i = 0; i < show; i++)
| |||
{
| |||
SErrorRecord &rec = m_error_history[i];
| |||
string status = rec.recovered ? "[RECOVERED]" : "[FAILED]";
| |||
report += StringFormat("%s %s: %d - %s (%s retries)\n",
| |||
status,
| |||
TimeToString(rec.timestamp, TIME_MINUTES),
| |||
rec.error_code,
| |||
rec.message,
| |||
IntegerToString(rec.retry_count)
| |||
);
| |||
}
| |||
| |||
return report;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Utility functions |
| |||
//+------------------------------------------------------------------+
| |||
void Log(const string message, ENUM_ERROR_SEVERITY severity)
| |||
{
| |||
string prefix = m_log_prefix;
| |||
switch(severity)
| |||
{
| |||
case ERR_SEVERITY_CRITICAL: prefix += " [CRITICAL]"; break;
| |||
case ERR_SEVERITY_HIGH: prefix += " [HIGH]"; break;
| |||
case ERR_SEVERITY_MEDIUM: prefix += " [MEDIUM]"; break;
| |||
case ERR_SEVERITY_LOW: prefix += " [LOW]"; break;
| |||
}
| |||
| |||
Print(prefix + " " + message);
| |||
}
| |||
| |||
void ResetLastError()
| |||
{
| |||
// Clear any pending error
| |||
int _ = GetLastError(); // Read and clear
| |||
}
| |||
| |||
// Getters
| |||
int GetTotalErrors() const { return m_total_errors; }
| |||
int GetRecoveredErrors() const { return m_recovered_errors; }
| |||
int GetFailedErrors() const { return m_failed_errors; }
| |||
int GetConsecutiveErrors() const { return m_consecutive_errors; }
| |||
bool IsHealthy() const { return m_consecutive_errors < 3; }
| |||
};
| |||
| |||
// Static instance initialization
| |||
CErrorRecovery* CErrorRecovery::s_instance = NULL;
| |||
| |||
// Global macro for easy error handling
| |||
#define HANDLE_ERROR(code, context, message) \
| |||
CErrorRecovery::Instance().HandleError(code, context, message)
| |||
| |||
#define CHECK_ERROR(context) \
| |||
do { int _err = GetLastError(); if(_err != 0) { \
| |||
CErrorRecovery::Instance().HandleError(_err, context, ""); \
| |||
ResetLastError(); \
| |||
} } while(0)
| |||
| |||
#define CLEAR_ERROR() ResetLastError()
| |||
| |||
#endif // CERRORRECOVERY_MQH
|