628 lines
20 KiB
MQL5
628 lines
20 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| CErrorRecovery.mqh - Centralized Error Recovery System |
|
|
//| Provides consistent error handling, retry logic, and recovery |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef CERRORRECOVERY_MQH
|
|
#define CERRORRECOVERY_MQH
|
|
|
|
#include "CArchitectureConfig.mqh"
|
|
|
|
// Error severity levels
|
|
enum ENUM_ERROR_SEVERITY
|
|
{
|
|
ERR_SEVERITY_LOW = 0, // Log only, continue
|
|
ERR_SEVERITY_MEDIUM = 1, // Degrade gracefully
|
|
ERR_SEVERITY_HIGH = 2, // Block operations temporarily
|
|
ERR_SEVERITY_CRITICAL = 3 // Shutdown required
|
|
};
|
|
|
|
// Error categories for classification
|
|
enum ENUM_ERROR_CATEGORY
|
|
{
|
|
ERR_CAT_FILE_IO = 0,
|
|
ERR_CAT_NETWORK = 1,
|
|
ERR_CAT_MEMORY = 2,
|
|
ERR_CAT_TRADE = 3,
|
|
ERR_CAT_SYSTEM = 4,
|
|
ERR_CAT_UNKNOWN = 5
|
|
};
|
|
|
|
// Recovery action types
|
|
enum ENUM_RECOVERY_ACTION
|
|
{
|
|
RECOVERY_NONE = 0, // No action, just log
|
|
RECOVERY_RETRY = 1, // Retry operation
|
|
RECOVERY_FALLBACK = 2, // Use fallback mode
|
|
RECOVERY_RESET = 3, // Reset and reinitialize
|
|
RECOVERY_SHUTDOWN = 4 // Graceful shutdown
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Error Record Structure |
|
|
//+------------------------------------------------------------------+
|
|
struct SErrorRecord
|
|
{
|
|
int error_code;
|
|
ENUM_ERROR_CATEGORY category;
|
|
ENUM_ERROR_SEVERITY severity;
|
|
string context;
|
|
string message;
|
|
datetime timestamp;
|
|
int retry_count;
|
|
bool recovered;
|
|
|
|
SErrorRecord()
|
|
{
|
|
error_code = 0;
|
|
category = ERR_CAT_UNKNOWN;
|
|
severity = ERR_SEVERITY_LOW;
|
|
context = "";
|
|
message = "";
|
|
timestamp = 0;
|
|
retry_count = 0;
|
|
recovered = false;
|
|
}
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Error Recovery Manager |
|
|
//+------------------------------------------------------------------+
|
|
class CErrorRecovery
|
|
{
|
|
private:
|
|
static CErrorRecovery* s_instance;
|
|
|
|
// Error history for analysis
|
|
SErrorRecord m_error_history[];
|
|
int m_history_count;
|
|
int m_max_history;
|
|
|
|
// Retry configuration
|
|
int m_max_retries;
|
|
int m_retry_delay_ms;
|
|
|
|
// Circuit breaker state
|
|
int m_consecutive_errors;
|
|
datetime m_circuit_breaker_until;
|
|
int m_circuit_breaker_threshold;
|
|
int m_circuit_breaker_cooldown_sec;
|
|
|
|
// Category-specific thresholds
|
|
int m_category_errors[];
|
|
datetime m_category_last_error[];
|
|
|
|
// Statistics
|
|
int m_total_errors;
|
|
int m_recovered_errors;
|
|
int m_failed_errors;
|
|
|
|
// Logging
|
|
bool m_verbose;
|
|
string m_log_prefix;
|
|
|
|
public:
|
|
CErrorRecovery()
|
|
{
|
|
m_history_count = 0;
|
|
m_max_history = 100;
|
|
ArrayResize(m_error_history, m_max_history);
|
|
|
|
m_max_retries = 3;
|
|
m_retry_delay_ms = 100;
|
|
|
|
m_consecutive_errors = 0;
|
|
m_circuit_breaker_until = 0;
|
|
m_circuit_breaker_threshold = 5;
|
|
m_circuit_breaker_cooldown_sec = 60;
|
|
|
|
ArrayResize(m_category_errors, 6);
|
|
ArrayResize(m_category_last_error, 6);
|
|
for(int i = 0; i < 6; i++)
|
|
{
|
|
m_category_errors[i] = 0;
|
|
m_category_last_error[i] = 0;
|
|
}
|
|
|
|
m_total_errors = 0;
|
|
m_recovered_errors = 0;
|
|
m_failed_errors = 0;
|
|
|
|
m_verbose = false;
|
|
m_log_prefix = "[ErrorRecovery]";
|
|
}
|
|
|
|
~CErrorRecovery()
|
|
{
|
|
// Cleanup
|
|
}
|
|
|
|
static CErrorRecovery* Instance()
|
|
{
|
|
if(s_instance == NULL)
|
|
s_instance = new CErrorRecovery();
|
|
return s_instance;
|
|
}
|
|
|
|
static void Destroy()
|
|
{
|
|
if(s_instance != NULL)
|
|
{
|
|
delete s_instance;
|
|
s_instance = NULL;
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Configure recovery behavior |
|
|
//+------------------------------------------------------------------+
|
|
void SetMaxRetries(int max_retries) { m_max_retries = max_retries; }
|
|
void SetRetryDelay(int delay_ms) { m_retry_delay_ms = delay_ms; }
|
|
void SetCircuitBreakerThreshold(int threshold) { m_circuit_breaker_threshold = threshold; }
|
|
void SetCircuitBreakerCooldown(int cooldown_sec) { m_circuit_breaker_cooldown_sec = cooldown_sec; }
|
|
void SetVerbose(bool verbose) { m_verbose = verbose; }
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Main error handling entry point |
|
|
//+------------------------------------------------------------------+
|
|
bool HandleError(int error_code, const string context, const string message)
|
|
{
|
|
// Clear error to prevent contamination
|
|
ResetLastError();
|
|
|
|
// Check if in circuit breaker
|
|
if(IsCircuitBreakerActive())
|
|
{
|
|
Log("Circuit breaker active, operation blocked", ERR_SEVERITY_HIGH);
|
|
return false;
|
|
}
|
|
|
|
// Classify error
|
|
ENUM_ERROR_CATEGORY category = ClassifyError(error_code);
|
|
ENUM_ERROR_SEVERITY severity = DetermineSeverity(error_code, category);
|
|
|
|
// Create error record
|
|
SErrorRecord record;
|
|
record.error_code = error_code;
|
|
record.category = category;
|
|
record.severity = severity;
|
|
record.context = context;
|
|
record.message = message;
|
|
record.timestamp = TimeCurrent();
|
|
record.retry_count = 0;
|
|
record.recovered = false;
|
|
|
|
// Add to history
|
|
AddToHistory(record);
|
|
|
|
// Update statistics
|
|
m_total_errors++;
|
|
m_consecutive_errors++;
|
|
m_category_errors[category]++;
|
|
m_category_last_error[category] = TimeCurrent();
|
|
|
|
// Check circuit breaker
|
|
if(m_consecutive_errors >= m_circuit_breaker_threshold)
|
|
{
|
|
ActivateCircuitBreaker();
|
|
}
|
|
|
|
// Determine recovery action
|
|
ENUM_RECOVERY_ACTION action = DetermineRecoveryAction(error_code, category, severity);
|
|
|
|
// Execute recovery
|
|
bool recovered = ExecuteRecovery(action, record);
|
|
|
|
if(recovered)
|
|
{
|
|
m_recovered_errors++;
|
|
m_consecutive_errors = 0;
|
|
record.recovered = true;
|
|
UpdateHistoryRecord(record);
|
|
Log("Error recovered: " + message, ERR_SEVERITY_LOW);
|
|
}
|
|
else
|
|
{
|
|
m_failed_errors++;
|
|
Log("Error recovery failed: " + message, severity);
|
|
}
|
|
|
|
return recovered;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Classify error code to category |
|
|
//+------------------------------------------------------------------+
|
|
ENUM_ERROR_CATEGORY ClassifyError(int error_code)
|
|
{
|
|
switch(error_code)
|
|
{
|
|
// File I/O errors
|
|
case 5001: case 5002: case 5003: case 5004:
|
|
case 5005: case 5006: case 5007: case 5008:
|
|
case 5010: case 5011: case 5012: case 5013:
|
|
case 5014: case 5015: case 5016: case 5017:
|
|
case 5020:
|
|
return ERR_CAT_FILE_IO;
|
|
|
|
// Network/connection errors
|
|
case 5200: case 5201: case 5202: case 5203:
|
|
case 6750: case 6751: case 6752:
|
|
return ERR_CAT_NETWORK;
|
|
|
|
// Memory errors
|
|
case 4001: case 4002: case 4003:
|
|
return ERR_CAT_MEMORY;
|
|
|
|
// Trading errors
|
|
case 10004: case 10006: case 10007: case 10009:
|
|
case 10010: case 10013: case 10014: case 10015:
|
|
case 10016: case 10021: case 10022: case 10023:
|
|
case 10024: case 10025: case 10026: case 10027:
|
|
return ERR_CAT_TRADE;
|
|
|
|
// System errors
|
|
case 1: case 2: case 3: case 4:
|
|
return ERR_CAT_SYSTEM;
|
|
|
|
default:
|
|
return ERR_CAT_UNKNOWN;
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Determine error severity |
|
|
//+------------------------------------------------------------------+
|
|
ENUM_ERROR_SEVERITY DetermineSeverity(int error_code, ENUM_ERROR_CATEGORY category)
|
|
{
|
|
// Critical errors
|
|
switch(error_code)
|
|
{
|
|
case 4001: // ERR_WRONG_FUNCTION_POINTER
|
|
case 4002: // ERR_ARRAY_INDEX_OUT_OF_RANGE
|
|
case 2: // ERR_COMMON_ERROR
|
|
case 10026: // ERR_TRADE_MODIFY_DENIED
|
|
return ERR_SEVERITY_CRITICAL;
|
|
|
|
// High severity
|
|
case 10007: // ERR_TRADE_NOT_ALLOWED
|
|
case 10014: // ERR_TRADE_INVALID_STOPS
|
|
case 5001: // ERR_FILE_TOO_LARGE
|
|
case 5200: // ERR_INTERNET_CONNECTION_CLOSED
|
|
return ERR_SEVERITY_HIGH;
|
|
|
|
// Medium severity
|
|
case 10004: // ERR_REQUOTE
|
|
case 10006: // ERR_NO_PRICES
|
|
case 5004: // ERR_FILE_NOT_OPEN
|
|
case 6750: // ERR_SOCKET_INIT_FAILED
|
|
return ERR_SEVERITY_MEDIUM;
|
|
|
|
// Low severity (transient)
|
|
case 10009: // ERR_TRADE_ERROR
|
|
case 10010: // ERR_TRADE_TIMEOUT
|
|
case 5003: // ERR_FILE_INVALID_HANDLE
|
|
return ERR_SEVERITY_LOW;
|
|
}
|
|
|
|
// Category-based fallback
|
|
if(category == ERR_CAT_MEMORY)
|
|
return ERR_SEVERITY_HIGH;
|
|
if(category == ERR_CAT_SYSTEM)
|
|
return ERR_SEVERITY_HIGH;
|
|
|
|
return ERR_SEVERITY_MEDIUM;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Determine recovery action |
|
|
//+------------------------------------------------------------------+
|
|
ENUM_RECOVERY_ACTION DetermineRecoveryAction(int error_code, ENUM_ERROR_CATEGORY category, ENUM_ERROR_SEVERITY severity)
|
|
{
|
|
// Never retry critical errors
|
|
if(severity == ERR_SEVERITY_CRITICAL)
|
|
return RECOVERY_SHUTDOWN;
|
|
|
|
// Network errors: retry with fallback
|
|
if(category == ERR_CAT_NETWORK)
|
|
return RECOVERY_RETRY;
|
|
|
|
// File I/O errors: retry then fallback
|
|
if(category == ERR_CAT_FILE_IO)
|
|
return RECOVERY_RETRY;
|
|
|
|
// Trading errors: specific handling
|
|
if(category == ERR_CAT_TRADE)
|
|
{
|
|
switch(error_code)
|
|
{
|
|
case 10004: // Requote - retry immediately
|
|
case 10006: // No prices - retry later
|
|
return RECOVERY_RETRY;
|
|
|
|
case 10007: // Not allowed - check permissions then fallback
|
|
return RECOVERY_FALLBACK;
|
|
|
|
default:
|
|
return RECOVERY_NONE;
|
|
}
|
|
}
|
|
|
|
// Default: no recovery action
|
|
return RECOVERY_NONE;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Execute recovery action |
|
|
//+------------------------------------------------------------------+
|
|
bool ExecuteRecovery(ENUM_RECOVERY_ACTION action, SErrorRecord &record)
|
|
{
|
|
switch(action)
|
|
{
|
|
case RECOVERY_NONE:
|
|
return true; // Nothing to do
|
|
|
|
case RECOVERY_RETRY:
|
|
return ExecuteRetry(record);
|
|
|
|
case RECOVERY_FALLBACK:
|
|
return ExecuteFallback(record);
|
|
|
|
case RECOVERY_RESET:
|
|
return ExecuteReset(record);
|
|
|
|
case RECOVERY_SHUTDOWN:
|
|
ExecuteShutdown(record);
|
|
return false; // Cannot recover from shutdown
|
|
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Retry logic with exponential backoff |
|
|
//+------------------------------------------------------------------+
|
|
bool ExecuteRetry(SErrorRecord &record)
|
|
{
|
|
if(record.retry_count >= m_max_retries)
|
|
{
|
|
Log("Max retries exceeded for error " + IntegerToString(record.error_code), ERR_SEVERITY_MEDIUM);
|
|
return false;
|
|
}
|
|
|
|
record.retry_count++;
|
|
|
|
// Calculate delay with exponential backoff
|
|
int delay = m_retry_delay_ms * (1 << (record.retry_count - 1));
|
|
if(delay > 5000) delay = 5000; // Cap at 5 seconds
|
|
|
|
Log("Retry " + IntegerToString(record.retry_count) + "/" + IntegerToString(m_max_retries) +
|
|
" after " + IntegerToString(delay) + "ms for " + record.context, ERR_SEVERITY_LOW);
|
|
|
|
// Use Sleep only in non-tick contexts
|
|
// In OnTick, this should be handled asynchronously
|
|
// For now, we return false to indicate retry needed
|
|
return false; // Caller must handle actual retry
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Fallback mode execution |
|
|
//+------------------------------------------------------------------+
|
|
bool ExecuteFallback(SErrorRecord &record)
|
|
{
|
|
Log("Activating fallback mode for " + record.context, ERR_SEVERITY_MEDIUM);
|
|
|
|
// Fallback behaviors by category
|
|
switch(record.category)
|
|
{
|
|
case ERR_CAT_FILE_IO:
|
|
// Switch to memory buffer mode
|
|
return true;
|
|
|
|
case ERR_CAT_NETWORK:
|
|
// Use cached data
|
|
return true;
|
|
|
|
case ERR_CAT_TRADE:
|
|
// Switch to shadow/simulation mode
|
|
return true;
|
|
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Reset and reinitialize |
|
|
//+------------------------------------------------------------------+
|
|
bool ExecuteReset(SErrorRecord &record)
|
|
{
|
|
Log("Executing system reset for " + record.context, ERR_SEVERITY_HIGH);
|
|
|
|
// Clear error state
|
|
ResetLastError();
|
|
|
|
// Reset counters
|
|
m_consecutive_errors = 0;
|
|
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Graceful shutdown |
|
|
//+------------------------------------------------------------------+
|
|
void ExecuteShutdown(SErrorRecord &record)
|
|
{
|
|
Log("CRITICAL: Executing graceful shutdown - " + record.message, ERR_SEVERITY_CRITICAL);
|
|
|
|
// Log final statistics
|
|
string stats = StringFormat("Total errors: %d, Recovered: %d, Failed: %d",
|
|
m_total_errors, m_recovered_errors, m_failed_errors);
|
|
Log(stats, ERR_SEVERITY_CRITICAL);
|
|
|
|
// In real implementation, this would trigger EA shutdown
|
|
// ExpertRemove(); // Uncomment when ready for production
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Circuit breaker management |
|
|
//+------------------------------------------------------------------+
|
|
bool IsCircuitBreakerActive()
|
|
{
|
|
if(m_circuit_breaker_until == 0)
|
|
return false;
|
|
|
|
if(TimeCurrent() < m_circuit_breaker_until)
|
|
{
|
|
int remaining = (int)(m_circuit_breaker_until - TimeCurrent());
|
|
if(m_verbose && remaining % 10 == 0) // Log every 10 seconds
|
|
Log("Circuit breaker active: " + IntegerToString(remaining) + "s remaining", ERR_SEVERITY_MEDIUM);
|
|
return true;
|
|
}
|
|
|
|
// Circuit breaker expired
|
|
m_circuit_breaker_until = 0;
|
|
m_consecutive_errors = 0;
|
|
Log("Circuit breaker deactivated", ERR_SEVERITY_LOW);
|
|
return false;
|
|
}
|
|
|
|
void ActivateCircuitBreaker()
|
|
{
|
|
m_circuit_breaker_until = TimeCurrent() + m_circuit_breaker_cooldown_sec;
|
|
Log("Circuit breaker ACTIVATED for " + IntegerToString(m_circuit_breaker_cooldown_sec) + "s", ERR_SEVERITY_HIGH);
|
|
}
|
|
|
|
void DeactivateCircuitBreaker()
|
|
{
|
|
m_circuit_breaker_until = 0;
|
|
m_consecutive_errors = 0;
|
|
Log("Circuit breaker manually deactivated", ERR_SEVERITY_LOW);
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| History management |
|
|
//+------------------------------------------------------------------+
|
|
void AddToHistory(SErrorRecord &record)
|
|
{
|
|
// Shift existing records
|
|
for(int i = m_history_count - 1; i > 0; i--)
|
|
{
|
|
m_error_history[i] = m_error_history[i - 1];
|
|
}
|
|
|
|
// Add new record at start
|
|
m_error_history[0] = record;
|
|
|
|
if(m_history_count < m_max_history)
|
|
m_history_count++;
|
|
}
|
|
|
|
void UpdateHistoryRecord(SErrorRecord &record)
|
|
{
|
|
// Find and update record
|
|
for(int i = 0; i < m_history_count; i++)
|
|
{
|
|
if(m_error_history[i].timestamp == record.timestamp &&
|
|
m_error_history[i].error_code == record.error_code)
|
|
{
|
|
m_error_history[i] = record;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Statistics and reporting |
|
|
//+------------------------------------------------------------------+
|
|
string GetErrorStats()
|
|
{
|
|
string stats = StringFormat(
|
|
"=== Error Recovery Statistics ===\n" +
|
|
"Total Errors: %d\n" +
|
|
"Recovered: %d (%.1f%%)\n" +
|
|
"Failed: %d (%.1f%%)\n" +
|
|
"Consecutive Errors: %d\n" +
|
|
"Circuit Breaker: %s\n" +
|
|
"History: %d/%d records",
|
|
m_total_errors,
|
|
m_recovered_errors,
|
|
(m_total_errors > 0 ? 100.0 * m_recovered_errors / m_total_errors : 0),
|
|
m_failed_errors,
|
|
(m_total_errors > 0 ? 100.0 * m_failed_errors / m_total_errors : 0),
|
|
m_consecutive_errors,
|
|
(IsCircuitBreakerActive() ? "ACTIVE" : "inactive"),
|
|
m_history_count, m_max_history
|
|
);
|
|
|
|
return stats;
|
|
}
|
|
|
|
string GetRecentErrors(int count = 10)
|
|
{
|
|
string report = "=== Recent Errors ===\n";
|
|
int show = MathMin(count, m_history_count);
|
|
|
|
for(int i = 0; i < show; i++)
|
|
{
|
|
SErrorRecord &rec = m_error_history[i];
|
|
string status = rec.recovered ? "[RECOVERED]" : "[FAILED]";
|
|
report += StringFormat("%s %s: %d - %s (%s retries)\n",
|
|
status,
|
|
TimeToString(rec.timestamp, TIME_MINUTES),
|
|
rec.error_code,
|
|
rec.message,
|
|
IntegerToString(rec.retry_count)
|
|
);
|
|
}
|
|
|
|
return report;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Utility functions |
|
|
//+------------------------------------------------------------------+
|
|
void Log(const string message, ENUM_ERROR_SEVERITY severity)
|
|
{
|
|
string prefix = m_log_prefix;
|
|
switch(severity)
|
|
{
|
|
case ERR_SEVERITY_CRITICAL: prefix += " [CRITICAL]"; break;
|
|
case ERR_SEVERITY_HIGH: prefix += " [HIGH]"; break;
|
|
case ERR_SEVERITY_MEDIUM: prefix += " [MEDIUM]"; break;
|
|
case ERR_SEVERITY_LOW: prefix += " [LOW]"; break;
|
|
}
|
|
|
|
Print(prefix + " " + message);
|
|
}
|
|
|
|
void ResetLastError()
|
|
{
|
|
// Clear any pending error
|
|
int _ = GetLastError(); // Read and clear
|
|
}
|
|
|
|
// Getters
|
|
int GetTotalErrors() const { return m_total_errors; }
|
|
int GetRecoveredErrors() const { return m_recovered_errors; }
|
|
int GetFailedErrors() const { return m_failed_errors; }
|
|
int GetConsecutiveErrors() const { return m_consecutive_errors; }
|
|
bool IsHealthy() const { return m_consecutive_errors < 3; }
|
|
};
|
|
|
|
// Static instance initialization
|
|
CErrorRecovery* CErrorRecovery::s_instance = NULL;
|
|
|
|
// Global macro for easy error handling
|
|
#define HANDLE_ERROR(code, context, message) \
|
|
CErrorRecovery::Instance().HandleError(code, context, message)
|
|
|
|
#define CHECK_ERROR(context) \
|
|
do { int _err = GetLastError(); if(_err != 0) { \
|
|
CErrorRecovery::Instance().HandleError(_err, context, ""); \
|
|
ResetLastError(); \
|
|
} } while(0)
|
|
|
|
#define CLEAR_ERROR() ResetLastError()
|
|
|
|
#endif // CERRORRECOVERY_MQH
|