| //+------------------------------------------------------------------+
|
| //| CUnifiedFileIO.mqh - Thread-Safe File Operations with Retry Logic |
|
| //| Provides robust file I/O with automatic retry, buffering, and |
|
| //| atomic write operations |
|
| //+------------------------------------------------------------------+
|
| #ifndef CUNIFIEDFILEIO_MQH
|
| #define CUNIFIEDFILEIO_MQH
|
|
|
| #include "CErrorRecovery.mqh"
|
|
|
| // File operation types
|
| enum ENUM_FILE_OPERATION
|
| {
|
| FILE_OP_READ = 0,
|
| FILE_OP_WRITE = 1,
|
| FILE_OP_APPEND = 2,
|
| FILE_OP_DELETE = 3
|
| };
|
|
|
| // Buffer strategy
|
| enum ENUM_BUFFER_STRATEGY
|
| {
|
| BUFFER_NONE = 0, // Direct I/O
|
| BUFFER_MEMORY = 1, // Memory buffer with async flush
|
| BUFFER_DISK = 2 // Disk buffer for critical data
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| File Buffer Entry |
|
| //+------------------------------------------------------------------+
|
| struct SFileBufferEntry
|
| {
|
| string data;
|
| datetime timestamp;
|
| int priority;
|
| int retry_count;
|
|
|
| SFileBufferEntry()
|
| {
|
| data = "";
|
| timestamp = 0;
|
| priority = 0;
|
| retry_count = 0;
|
| }
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Unified File I/O Manager |
|
| //+------------------------------------------------------------------+
|
| class CUnifiedFileIO
|
| {
|
| private:
|
| static CUnifiedFileIO* s_instance;
|
|
|
| // Memory buffer
|
| SFileBufferEntry m_buffer[];
|
| int m_buffer_count;
|
| int m_buffer_max;
|
| int m_buffer_critical_threshold;
|
|
|
| // Statistics
|
| int m_total_operations;
|
| int m_successful_operations;
|
| int m_failed_operations;
|
| int m_buffered_operations;
|
|
|
| // Configuration
|
| int m_max_retries;
|
| int m_retry_delay_ms;
|
| bool m_use_atomic_writes;
|
| bool m_verify_writes;
|
| int m_file_share_flags;
|
|
|
| // Error recovery
|
| CErrorRecovery* m_error_recovery;
|
|
|
| // Folder creation tracking
|
| string m_created_folders[];
|
| int m_created_count;
|
|
|
| public:
|
| CUnifiedFileIO()
|
| {
|
| m_buffer_count = 0;
|
| m_buffer_max = 1000;
|
| m_buffer_critical_threshold = 800;
|
| ArrayResize(m_buffer, m_buffer_max);
|
|
|
| m_total_operations = 0;
|
| m_successful_operations = 0;
|
| m_failed_operations = 0;
|
| m_buffered_operations = 0;
|
|
|
| m_max_retries = 3;
|
| m_retry_delay_ms = 50;
|
| m_use_atomic_writes = true;
|
| m_verify_writes = true;
|
| m_file_share_flags = FILE_SHARE_READ | FILE_SHARE_WRITE;
|
|
|
| m_error_recovery = CErrorRecovery::Instance();
|
|
|
| m_created_count = 0;
|
| ArrayResize(m_created_folders, 100);
|
| }
|
|
|
| ~CUnifiedFileIO()
|
| {
|
| // Flush any remaining buffered data
|
| FlushBuffer(true); // Force flush
|
| }
|
|
|
| static CUnifiedFileIO* Instance()
|
| {
|
| if(s_instance == NULL)
|
| s_instance = new CUnifiedFileIO();
|
| return s_instance;
|
| }
|
|
|
| static void Destroy()
|
| {
|
| if(s_instance != NULL)
|
| {
|
| delete s_instance;
|
| s_instance = NULL;
|
| }
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Configuration |
|
| //+------------------------------------------------------------------+
|
| void SetMaxRetries(int max_retries) { m_max_retries = max_retries; }
|
| void SetRetryDelay(int delay_ms) { m_retry_delay_ms = delay_ms; }
|
| void SetBufferSize(int size) { m_buffer_max = size; ArrayResize(m_buffer, size); }
|
| void SetAtomicWrites(bool atomic) { m_use_atomic_writes = atomic; }
|
| void SetVerifyWrites(bool verify) { m_verify_writes = verify; }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Safe file read with retry logic |
|
| //+------------------------------------------------------------------+
|
| bool ReadFile(const string filename, string &content, bool use_common = true)
|
| {
|
| m_total_operations++;
|
|
|
| // Clear any pending errors
|
| ClearError();
|
|
|
| // Ensure folder exists
|
| EnsureFolderExists(filename, use_common);
|
|
|
| int flags = FILE_READ | FILE_TXT | FILE_ANSI;
|
| if(use_common) flags |= FILE_COMMON;
|
| flags |= m_file_share_flags;
|
|
|
| int handle = INVALID_HANDLE;
|
|
|
| // Retry loop
|
| for(int attempt = 1; attempt <= m_max_retries; attempt++)
|
| {
|
| handle = FileOpen(filename, flags);
|
|
|
| if(handle != INVALID_HANDLE)
|
| break;
|
|
|
| int error_code = GetLastError();
|
| if(error_code != ERR_FILE_NOT_FOUND && attempt < m_max_retries)
|
| {
|
| Log("File read attempt " + IntegerToString(attempt) + "/" + IntegerToString(m_max_retries) +
|
| " failed for " + filename + ": " + IntegerToString(error_code));
|
| Sleep(m_retry_delay_ms * attempt);
|
| ClearError();
|
| }
|
| else
|
| {
|
| break;
|
| }
|
| }
|
|
|
| if(handle == INVALID_HANDLE)
|
| {
|
| m_failed_operations++;
|
| int err = GetLastError();
|
| if(err != 0)
|
| m_error_recovery.HandleError(err, "CUnifiedFileIO::ReadFile", filename);
|
| return false;
|
| }
|
|
|
| // Read content
|
| content = "";
|
| while(!FileIsEnding(handle))
|
| {
|
| content += FileReadString(handle) + "\n";
|
| }
|
|
|
| FileClose(handle);
|
|
|
| m_successful_operations++;
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Safe file write with atomic operation option |
|
| //+------------------------------------------------------------------+
|
| bool WriteFile(const string filename, const string content, bool use_common = true, bool append = false)
|
| {
|
| m_total_operations++;
|
|
|
| // Clear any pending errors
|
| ClearError();
|
|
|
| // Ensure folder exists
|
| EnsureFolderExists(filename, use_common);
|
|
|
| if(m_use_atomic_writes && !append)
|
| {
|
| return WriteFileAtomic(filename, content, use_common);
|
| }
|
|
|
| int flags = FILE_WRITE | FILE_TXT | FILE_ANSI;
|
| if(use_common) flags |= FILE_COMMON;
|
| if(append) flags |= FILE_READ; // Required for seek
|
| flags |= m_file_share_flags;
|
|
|
| int handle = INVALID_HANDLE;
|
|
|
| // Retry loop
|
| for(int attempt = 1; attempt <= m_max_retries; attempt++)
|
| {
|
| if(append)
|
| {
|
| handle = FileOpen(filename, flags, ',');
|
| if(handle != INVALID_HANDLE)
|
| FileSeek(handle, 0, SEEK_END);
|
| }
|
| else
|
| {
|
| handle = FileOpen(filename, flags);
|
| }
|
|
|
| if(handle != INVALID_HANDLE)
|
| break;
|
|
|
| int error_code = GetLastError();
|
| if(attempt < m_max_retries)
|
| {
|
| Log("File write attempt " + IntegerToString(attempt) + "/" + IntegerToString(m_max_retries) +
|
| " failed for " + filename + ": " + IntegerToString(error_code));
|
| Sleep(m_retry_delay_ms * attempt);
|
| ClearError();
|
| }
|
| else
|
| {
|
| break;
|
| }
|
| }
|
|
|
| if(handle == INVALID_HANDLE)
|
| {
|
| m_failed_operations++;
|
| int err = GetLastError();
|
| if(err != 0)
|
| m_error_recovery.HandleError(err, "CUnifiedFileIO::WriteFile", filename);
|
|
|
| // Buffer to memory as fallback
|
| BufferWrite(filename, content);
|
| return false;
|
| }
|
|
|
| // Write content
|
| bool write_ok = (FileWriteString(handle, content) > 0);
|
| FileFlush(handle);
|
|
|
| // Verify write if requested
|
| if(m_verify_writes && write_ok && !append)
|
| {
|
| long size = FileSize(handle);
|
| if(size <= 0)
|
| {
|
| Log("Write verification failed for " + filename);
|
| write_ok = false;
|
| }
|
| }
|
|
|
| FileClose(handle);
|
|
|
| if(write_ok)
|
| {
|
| m_successful_operations++;
|
| return true;
|
| }
|
| else
|
| {
|
| m_failed_operations++;
|
| BufferWrite(filename, content);
|
| return false;
|
| }
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Atomic file write (write to temp, then rename) |
|
| //+------------------------------------------------------------------+
|
| bool WriteFileAtomic(const string filename, const string content, bool use_common = true)
|
| {
|
| string temp_filename = filename + ".tmp";
|
|
|
| // Write to temp file
|
| int flags = FILE_WRITE | FILE_TXT | FILE_ANSI;
|
| if(use_common) flags |= FILE_COMMON;
|
| flags |= m_file_share_flags;
|
|
|
| int handle = FileOpen(temp_filename, flags);
|
| if(handle == INVALID_HANDLE)
|
| {
|
| // Fallback to non-atomic write
|
| return WriteFile(filename, content, use_common, false);
|
| }
|
|
|
| bool write_ok = (FileWriteString(handle, content) > 0);
|
| FileFlush(handle);
|
| long temp_size = FileSize(handle);
|
| FileClose(handle);
|
|
|
| if(!write_ok || temp_size <= 0)
|
| {
|
| // Clean up temp file
|
| FileDelete(temp_filename, use_common ? FILE_COMMON : 0);
|
| return false;
|
| }
|
|
|
| // Delete old file if exists
|
| if(FileIsExist(filename, use_common ? FILE_COMMON : 0))
|
| {
|
| FileDelete(filename, use_common ? FILE_COMMON : 0);
|
| }
|
|
|
| // Rename temp to final
|
| // Note: MQL5 doesn't have direct FileRename, so we copy and delete
|
| bool renamed = FileCopy(temp_filename, use_common ? FILE_COMMON : 0,
|
| filename, use_common ? FILE_COMMON : 0,
|
| FILE_REWRITE);
|
|
|
| if(renamed)
|
| {
|
| FileDelete(temp_filename, use_common ? FILE_COMMON : 0);
|
| m_successful_operations++;
|
| return true;
|
| }
|
| else
|
| {
|
| // Cleanup
|
| FileDelete(temp_filename, use_common ? FILE_COMMON : 0);
|
| m_failed_operations++;
|
| BufferWrite(filename, content);
|
| return false;
|
| }
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Buffer write for failed operations |
|
| //+------------------------------------------------------------------+
|
| void BufferWrite(const string filename, const string content)
|
| {
|
| if(m_buffer_count >= m_buffer_max)
|
| {
|
| // Remove oldest entry
|
| for(int i = 0; i < m_buffer_count - 1; i++)
|
| {
|
| m_buffer[i] = m_buffer[i + 1];
|
| }
|
| m_buffer_count--;
|
| }
|
|
|
| SFileBufferEntry entry;
|
| entry.data = filename + "|" + content;
|
| entry.timestamp = TimeCurrent();
|
| entry.priority = 1;
|
| entry.retry_count = 0;
|
|
|
| m_buffer[m_buffer_count] = entry;
|
| m_buffer_count++;
|
| m_buffered_operations++;
|
|
|
| if(m_buffer_count >= m_buffer_critical_threshold)
|
| {
|
| Log("WARNING: File buffer at " + IntegerToString(m_buffer_count) + "/" + IntegerToString(m_buffer_max));
|
| }
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Flush buffer to disk |
|
| //+------------------------------------------------------------------+
|
| int FlushBuffer(bool force = false)
|
| {
|
| if(m_buffer_count == 0) return 0;
|
|
|
| int flushed = 0;
|
| int max_flush = force ? m_buffer_count : MathMin(100, m_buffer_count);
|
|
|
| for(int i = 0; i < max_flush; i++)
|
| {
|
| SFileBufferEntry &entry = m_buffer[i];
|
|
|
| // Parse filename and content
|
| int sep_pos = StringFind(entry.data, "|");
|
| if(sep_pos < 0) continue;
|
|
|
| string filename = StringSubstr(entry.data, 0, sep_pos);
|
| string content = StringSubstr(entry.data, sep_pos + 1);
|
|
|
| if(WriteFile(filename, content, true, true)) // Append mode
|
| {
|
| flushed++;
|
| }
|
| else
|
| {
|
| entry.retry_count++;
|
| if(entry.retry_count >= m_max_retries)
|
| {
|
| Log("Failed to flush buffer entry after " + IntegerToString(m_max_retries) + " retries");
|
| // Keep in buffer for next attempt
|
| }
|
| }
|
| }
|
|
|
| // Remove flushed entries
|
| int remaining = 0;
|
| for(int i = 0; i < m_buffer_count; i++)
|
| {
|
| if(i < flushed)
|
| continue; // Skip flushed entries
|
|
|
| m_buffer[remaining] = m_buffer[i];
|
| remaining++;
|
| }
|
|
|
| m_buffer_count = remaining;
|
| ArrayResize(m_buffer, m_buffer_max); // Ensure capacity
|
|
|
| if(flushed > 0)
|
| {
|
| Log("Flushed " + IntegerToString(flushed) + " buffer entries, " + IntegerToString(m_buffer_count) + " remaining");
|
| }
|
|
|
| return flushed;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Ensure folder exists |
|
| //+------------------------------------------------------------------+
|
| void EnsureFolderExists(const string path, bool use_common)
|
| {
|
| // Extract folder path
|
| int last_sep = -1;
|
| for(int i = StringLen(path) - 1; i >= 0; i--)
|
| {
|
| string c = StringSubstr(path, i, 1);
|
| if(c == "\\" || c == "/")
|
| {
|
| last_sep = i;
|
| break;
|
| }
|
| }
|
|
|
| if(last_sep < 0) return; // No folder component
|
|
|
| string folder = StringSubstr(path, 0, last_sep);
|
| if(folder == "") return;
|
|
|
| // Check if already created
|
| for(int i = 0; i < m_created_count; i++)
|
| {
|
| if(m_created_folders[i] == folder)
|
| return;
|
| }
|
|
|
| // Create folder
|
| int flags = use_common ? FILE_COMMON : 0;
|
| bool created = FolderCreate(folder, flags);
|
|
|
| if(created || GetLastError() == 0)
|
| {
|
| // Track created folder
|
| if(m_created_count < ArraySize(m_created_folders))
|
| {
|
| m_created_folders[m_created_count] = folder;
|
| m_created_count++;
|
| }
|
| }
|
|
|
| ClearError();
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Statistics and reporting |
|
| //+------------------------------------------------------------------+
|
| string GetStatistics()
|
| {
|
| string stats = StringFormat(
|
| "=== Unified File I/O Statistics ===\n" +
|
| "Total Operations: %d\n" +
|
| "Successful: %d (%.1f%%)\n" +
|
| "Failed: %d (%.1f%%)\n" +
|
| "Buffered: %d\n" +
|
| "Buffer: %d/%d entries",
|
| m_total_operations,
|
| m_successful_operations,
|
| (m_total_operations > 0 ? 100.0 * m_successful_operations / m_total_operations : 0),
|
| m_failed_operations,
|
| (m_total_operations > 0 ? 100.0 * m_failed_operations / m_total_operations : 0),
|
| m_buffered_operations,
|
| m_buffer_count,
|
| m_buffer_max
|
| );
|
|
|
| return stats;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Utility functions |
|
| //+------------------------------------------------------------------+
|
| void Log(const string message)
|
| {
|
| Print("[UnifiedFileIO] " + message);
|
| }
|
|
|
| void ClearError()
|
| {
|
| int _ = GetLastError();
|
| }
|
|
|
| // Getters
|
| int GetBufferedCount() const { return m_buffer_count; }
|
| bool IsBufferCritical() const { return m_buffer_count >= m_buffer_critical_threshold; }
|
| int GetBufferUtilizationPercent() const { return (m_buffer_count * 100) / m_buffer_max; }
|
| };
|
|
|
| // Static instance initialization
|
| CUnifiedFileIO* CUnifiedFileIO::s_instance = NULL;
|
|
|
| // Convenience macros
|
| #define SAFE_FILE_READ(filename, content, use_common) \
|
| CUnifiedFileIO::Instance().ReadFile(filename, content, use_common)
|
|
|
| #define SAFE_FILE_WRITE(filename, content, use_common) \
|
| CUnifiedFileIO::Instance().WriteFile(filename, content, use_common, false)
|
|
|
| #define SAFE_FILE_APPEND(filename, content, use_common) \
|
| CUnifiedFileIO::Instance().WriteFile(filename, content, use_common, true)
|
|
|
| #define FLUSH_FILE_BUFFER(force) \
|
| CUnifiedFileIO::Instance().FlushBuffer(force)
|
|
|
| #endif // CUNIFIEDFILEIO_MQH
|
