Warrior_EA/Database/DatabaseOperationsManager.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//+------------------------------------------------------------------+
#include "..\System\PrintVerbose.mqh"
#define DB_RETRY_ATTEMPTS   3
#define DB_RETRY_DELAY_MS   20
//--- ERR_DATABASE_NO_MORE_DATA: what GetLastError() reports after DatabaseRead() steps a statement to
//--- completion with nothing to read back (SQLite's DONE). For INSERT/UPDATE that is the SUCCESS
//--- outcome, not a failure - see PrepareAndExecuteBound().
#define DB_NO_MORE_DATA     5126
class CDatabaseOperationsManager
  {
private:
   bool              m_verboseMode;
   int               m_databaseHandle;
   // Table/column names can't be bound as SQL parameters, so they must be
   // restricted to a safe character set before being concatenated into a query.
   bool              IsValidIdentifier(const string name)
     {
      int len = StringLen(name);
      if(len == 0)
         return false;
      for(int i = 0; i < len; i++)
        {
         ushort c = StringGetCharacter(name, i);
         bool isAlnum = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
         if(!isAlnum)
            return false;
        }
      return true;
     }
   // Prepare and execute a query that has no bound parameters (DDL, or already-safe DML)
   bool              PrepareAndExecuteQuery(string query)
     {
      if(m_databaseHandle == INVALID_HANDLE)
         return false;
      int attempts = 0;
      while(attempts < DB_RETRY_ATTEMPTS)
        {
         if(DatabaseExecute(m_databaseHandle, query))
            return true;
         Print("DatabaseExecute error: Attempt " + IntegerToString(attempts + 1) + " failed for query: " + query);
         Sleep(DB_RETRY_DELAY_MS);
         attempts++;
        }
      return false;
     }
   // Prepare a statement, bind parameters in order, execute it once, and finalize
   bool              PrepareAndExecuteBound(string query, const string &params[])
     {
      if(m_databaseHandle == INVALID_HANDLE)
         return false;
      int handle = DatabasePrepare(m_databaseHandle, query);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare bound query: " + query);
         return false;
        }
      for(int i = 0; i < ArraySize(params); i++)
        {
         if(!DatabaseBind(handle, i, params[i]))
           {
            Print("Failed to bind parameter " + IntegerToString(i) + " for query: " + query);
            DatabaseFinalize(handle);
            return false;
           }
        }
      // INSERT/UPDATE have no result set, so DatabaseRead() returns false on
      // success too (nothing to read) - the real signal is the error code. A
      // completed DML step can surface as either 0 or DB_NO_MORE_DATA (5126,
      // SQLite's DONE): the statement RAN, there is just nothing to read back.
      // The tester agent reports 5126 where the live terminal reports 0 for
      // the same completed step; treating it as failure printed a phantom
      // "Failed to insert/update" pair for EVERY row a backtest journaled
      // (~11.7k lines per run) while all of the data landed correctly - and
      // that spam would bury any real error. Genuine failures (busy, locked,
      // constraint, misuse) surface as other codes and still fail here.
      ResetLastError();
      DatabaseRead(handle);
      int error = GetLastError();
      DatabaseFinalize(handle);
      if(error != 0 && error != DB_NO_MORE_DATA)
        {
         Print("Failed to execute bound query (error " + IntegerToString(error) + "): " + query);
         return false;
        }
      return true;
     }
public:
   //constructor
                     CDatabaseOperationsManager(bool verbose = false)
     {
      m_verboseMode = verbose;
      m_databaseHandle = INVALID_HANDLE;
     }
   void              SetDatabaseHandle(int dbHandle)
     {
      m_databaseHandle = dbHandle;
     }
   // Create a table in the database
   bool              CreateTable(string tableName, string tableSchema)
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to create table with invalid identifier: " + tableName);
         return false;
        }
      if(!DatabaseTableExists(m_databaseHandle, tableName))
        {
         string createTableQuery = "CREATE TABLE " + tableName + " (" + tableSchema + ")";
         if(PrepareAndExecuteQuery(createTableQuery))
            return true;
         Print("Failed to create table " + tableName);
         return false;
        }
      return true; // Table already exists
     }
   bool              DeleteTable(string tableName)
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to delete table with invalid identifier: " + tableName);
         return false;
        }
      string deleteTableQuery = "DROP TABLE IF EXISTS " + tableName;
      if(PrepareAndExecuteQuery(deleteTableQuery))
        {
         PrintVerbose("Table " + tableName + " deleted successfully.");
         return true;
        }
      Print("Failed to delete table " + tableName);
      return false;
     }
   // Insert a trade record into the database. Column names are our own fixed
   // schema field names (never user data); values are bound as parameters.
   bool              InsertTradeRecord(string tableName, const string &columns[], const string &values[])
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to insert into table with invalid identifier: " + tableName);
         return false;
        }
      string insertQuery = "INSERT INTO " + tableName + " (";
      string placeholders = "";
      for(int i = 0; i < ArraySize(columns); i++)
        {
         insertQuery += columns[i];
         placeholders += "?";
         if(i < ArraySize(columns) - 1)
           {
            insertQuery += ", ";
            placeholders += ", ";
           }
        }
      insertQuery += ") VALUES (" + placeholders + ")";
      if(PrepareAndExecuteBound(insertQuery, values))
         return true;
      Print("Failed to insert trade record into " + tableName);
      return false;
     }
   struct RecordCount
     {
      int            count;
     };
   // Fetch the number of records in the specified table
   bool              FetchRecordCount(string tableName, int &count)
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to query table with invalid identifier: " + tableName);
         return false;
        }
      RecordCount countStruct;
      string countQuery = "SELECT COUNT(*) as count FROM " + tableName;
      int handle = DatabasePrepare(m_databaseHandle, countQuery);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare COUNT query for " + tableName);
         return false;
        }
      bool ok = DatabaseReadBind(handle, countStruct);
      if(ok)
         count = countStruct.count;
      DatabaseFinalize(handle);
      return ok;
     }
   //--- Targeted lookups. These answer ProcessSignal/UpdateSignalsWeights' questions inside SQLite
   //--- and return one row or one pair of numbers, so per-signal cost stays flat no matter how large
   //--- a table grows. They exist because the original design pulled ENTIRE tables into MQL struct
   //--- arrays for every question - the real constraint behind the historical 1000-row cap: SQLite
   //--- has no such limit, but materializing thousands of string-bearing structs per signal does.
   struct OpenTradeRow
     {
      double         entryPrice;
     };
   // Entry price of the still-open ("NA") trade for pattern+direction, if one exists
   bool              FetchOpenTradeEntry(string tableName, string pattern, string direction,
                                         double &entryPrice, bool &found)
     {
      found = false;
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to query table with invalid identifier: " + tableName);
         return false;
        }
      string q = "SELECT entryPrice FROM " + tableName +
                 " WHERE pattern=? AND direction=? AND result='NA' LIMIT 1";
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare open-trade query for " + tableName);
         return false;
        }
      if(!DatabaseBind(handle, 0, pattern) || !DatabaseBind(handle, 1, direction))
        {
         Print("Failed to bind open-trade query for " + tableName);
         DatabaseFinalize(handle);
         return false;
        }
      OpenTradeRow row;
      if(DatabaseReadBind(handle, row))
        {
         entryPrice = row.entryPrice;
         found = true;
        }
      DatabaseFinalize(handle);
      return true;
     }
   struct TimeKeyRow
     {
      long           k;
     };
   // Time key (yyyymmddhhmm as a number) of the table's NEWEST row. Rows are inserted in
   // chronological order, so max ROWID carries the latest timestamp. `quiet` suppresses the
   // prepare-failure print so existence probes over possibly-absent tables don't spam the log.
   bool              FetchNewestTimeKey(string tableName, long &key, bool &found, const bool quiet = false)
     {
      found = false;
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to query table with invalid identifier: " + tableName);
         return false;
        }
      string q = "SELECT (((year*100+month)*100+day)*100+hour)*100+minutes AS k FROM " +
                 tableName + " ORDER BY ROWID DESC LIMIT 1";
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         if(!quiet)
            Print("Failed to prepare newest-time query for " + tableName);
         return false;
        }
      TimeKeyRow row;
      if(DatabaseReadBind(handle, row))
        {
         key = row.k;
         found = true;
        }
      DatabaseFinalize(handle);
      return true;
     }
   struct WinLossRow
     {
      int            wins;
      int            losses;
     };
   // Closed-trade outcome counts with open time strictly before beforeKey (yyyymmddhhmm). The bound
   // is what keeps a resumed/mixed database's not-yet-current rows out of the weights in the tester.
   bool              FetchWinLossCounts(string tableName, long beforeKey, int &wins, int &losses)
     {
      wins = 0;
      losses = 0;
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to query table with invalid identifier: " + tableName);
         return false;
        }
      string q = "SELECT COALESCE(SUM(result='Profit'),0), COALESCE(SUM(result='Loss'),0) FROM " +
                 tableName + " WHERE result!='NA' AND (((year*100+month)*100+day)*100+hour)*100+minutes < " +
                 IntegerToString(beforeKey);
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare win-loss query for " + tableName);
         return false;
        }
      WinLossRow row;
      bool ok = DatabaseReadBind(handle, row);
      if(ok)
        {
         wins = row.wins;
         losses = row.losses;
        }
      DatabaseFinalize(handle);
      return ok;
     }
   // Fetch trade records from specified table and read into a structure, then populate the dynamic array with the data
   template <typename T>
   bool              FetchTradeRecords(string tableName, T &tradeRecordStructure, T &tradeRecords[])
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to query table with invalid identifier: " + tableName);
         return false;
        }
      int recordCount;
      if(!FetchRecordCount(tableName, recordCount))
         return false;
      ArrayResize(tradeRecords, recordCount);
      string selectQuery = "SELECT * FROM " + tableName;
      int handle = DatabasePrepare(m_databaseHandle, selectQuery);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare SELECT query for " + tableName);
         return false;
        }
      int i = 0;
      for(; i < recordCount && DatabaseReadBind(handle, tradeRecordStructure); i++)
         tradeRecords[i] = tradeRecordStructure;
      DatabaseFinalize(handle);
      if(i < recordCount)
        {
         // The read loop can exit early (a concurrent delete on the shared COMMON db between the
         // COUNT and SELECT above, or a transient read failure) - trim the array to what was
         // actually read instead of leaving trailing zero/empty-string structs that downstream
         // reporting would otherwise silently count as real (empty-symbol, 0-profit) trades.
         Print(__FUNCTION__ + ": only read " + IntegerToString(i) + " of " + IntegerToString(recordCount) +
               " expected rows from " + tableName + " - trimming to actual rows read.");
         ArrayResize(tradeRecords, i);
        }
      return true;
     }
   // Update the exit price/result of the still-open ("NA") trade matching pattern+direction
   bool              UpdateTradeRecord(string tableName, const string &columns[], const string &values[], string pattern, string direction)
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to update table with invalid identifier: " + tableName);
         return false;
        }
      string updateQuery = "UPDATE " + tableName + " SET ";
      for(int i = 0; i < ArraySize(columns); i++)
        {
         updateQuery += columns[i] + "=?";
         if(i < ArraySize(columns) - 1)
            updateQuery += ", ";
        }
      updateQuery += " WHERE pattern=? AND direction=? AND result='NA'";
      string params[];
      ArrayResize(params, ArraySize(values) + 2);
      for(int i = 0; i < ArraySize(values); i++)
         params[i] = values[i];
      params[ArraySize(values)] = pattern;
      params[ArraySize(values) + 1] = direction;
      if(PrepareAndExecuteBound(updateQuery, params))
         return true;
      Print("Failed to update trade records in " + tableName);
      return false;
     }
   // Delete the oldest entry from a specified table
   bool              DeleteOldestEntry(string tableName)
     {
      if(!IsValidIdentifier(tableName))
        {
         Print("Refusing to delete from table with invalid identifier: " + tableName);
         return false;
        }
      string deleteQuery = "DELETE FROM " + tableName + " WHERE ROWID = (SELECT MIN(ROWID) FROM " + tableName + ")";
      if(PrepareAndExecuteQuery(deleteQuery))
        {
         PrintVerbose("Oldest entry deleted from " + tableName + " successfully.");
         return true;
        }
      Print("Failed to delete oldest entry from " + tableName);
      return false;
     }
  };
//+------------------------------------------------------------------+