Warrior_EA/Database/DatabaseOperationsManager.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//+------------------------------------------------------------------+
#include "..\System\PrintVerbose.mqh"
#define DB_RETRY_ATTEMPTS   3
#define DB_RETRY_DELAY_MS   20
//--- ERR_DATABASE_NO_MORE_DATA: what GetLastError() reports after DatabaseRead() steps a statement to
//--- completion with nothing to read back (SQLite's DONE). For INSERT/UPDATE that is the SUCCESS
//--- outcome, not a failure - see PrepareAndExecuteBound().
#define DB_NO_MORE_DATA     5126
class CDatabaseOperationsManager
  {
private:
   bool              m_verboseMode;
   int               m_databaseHandle;
   // Table/column names can't be bound as SQL parameters, so they must be
   // restricted to a safe character set before being concatenated into a query.
   bool              IsValidIdentifier(const string name)
     {
      int len = StringLen(name);
      if(len == 0)
         return false;
      for(int i = 0; i < len; i++)
        {
         ushort c = StringGetCharacter(name, i);
         bool isAlnum = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
         if(!isAlnum)
            return false;
        }
      return true;
     }
   // Shared guard for every public method below: refuse an identifier that failed IsValidIdentifier(),
   // printing which operation was refused (the verb) and which identifier failed.
   bool              RequireValidIdentifier(const string tableName, const string verb)
     {
      if(IsValidIdentifier(tableName))
         return true;
      Print("Refusing to " + verb + " table with invalid identifier: " + tableName);
      return false;
     }
   // Join array elements into one comma-separated string, appending `suffix` to each element
   // first (e.g. suffix "=?" turns column names into an UPDATE ... SET list). Shared by every
   // column/placeholder list built below so the "comma unless this is the last element" shape
   // exists exactly once.
   //--- A QUOTED IDENTIFIER. Pattern tables are named after the filter id, and "2WD_Pattern_0_Sell"
   //--- starts with a digit, which SQLite reads as a malformed number unless the name is quoted.
   //--- Measured 2026-09-08 on the first tester pass with the database on: every Second Wind table
   //--- failed to create, three retries each, and the rows were dropped.
   string            Q(const string name)
     {
      return "\"" + name + "\"";
     }
   string            JoinWithCommas(const string &items[], const string suffix = "")
     {
      string result = "";
      int n = ArraySize(items);
      for(int i = 0; i < n; i++)
        {
         result += items[i] + suffix;
         if(i < n - 1)
            result += ", ";
        }
      return result;
     }
   // Build a "?, ?, ..." bound-parameter placeholder list of the given length.
   string            PlaceholderList(int count)
     {
      string items[];
      ArrayResize(items, count);
      for(int i = 0; i < count; i++)
         items[i] = "?";
      return JoinWithCommas(items);
     }
   // Prepare and execute a query that has no bound parameters (DDL, or already-safe DML)
   bool              PrepareAndExecuteQuery(string query)
     {
      if(m_databaseHandle == INVALID_HANDLE)
         return false;
      int attempts = 0;
      while(attempts < DB_RETRY_ATTEMPTS)
        {
         if(DatabaseExecute(m_databaseHandle, query))
            return true;
         Print("DatabaseExecute error: Attempt " + IntegerToString(attempts + 1) + " failed for query: " + query);
         Sleep(DB_RETRY_DELAY_MS);
         attempts++;
        }
      return false;
     }
   // Is this GetLastError() code the kind a retry can actually fix - the DB momentarily busy/locked
   // by another chart on the shared COMMON database (DatabaseConnectionManager.mqh opens it
   // DATABASE_OPEN_COMMON) - versus a genuine defect (constraint, misuse, bad SQL) that will fail
   // identically on attempt 2? Only PrepareAndExecuteQuery had this retry before; every trade-journal
   // row and pattern-weight update went through PrepareAndExecuteBound with none.
   bool              IsRetryableDbError(const int error)
     {
      return (error == ERR_DATABASE_BUSY || error == ERR_DATABASE_LOCKED || error == ERR_DATABASE_CONNECT);
     }
   // Prepare a statement, bind parameters in order, execute it once, and finalize. Retries the WHOLE
   // prepare/bind/step sequence (not just the step) on a busy/locked error, same DB_RETRY_ATTEMPTS/
   // DB_RETRY_DELAY_MS as PrepareAndExecuteQuery - a stale prepared handle from a failed attempt is
   // never reused.
   bool              PrepareAndExecuteBound(string query, const string &params[])
     {
      if(m_databaseHandle == INVALID_HANDLE)
         return false;
      int attempts = 0;
      while(true)
        {
         int handle = DatabasePrepare(m_databaseHandle, query);
         if(handle == INVALID_HANDLE)
           {
            int prepErr = GetLastError();
            if(IsRetryableDbError(prepErr) && attempts + 1 < DB_RETRY_ATTEMPTS)
              {
               attempts++;
               Sleep(DB_RETRY_DELAY_MS);
               continue;
              }
            Print("Failed to prepare bound query: " + query);
            return false;
           }
         bool bindFailed = false;
         for(int i = 0; i < ArraySize(params); i++)
           {
            if(!DatabaseBind(handle, i, params[i]))
              {
               Print("Failed to bind parameter " + IntegerToString(i) + " for query: " + query);
               DatabaseFinalize(handle);
               bindFailed = true;
               break;
              }
           }
         if(bindFailed)
            return false;
         // INSERT/UPDATE have no result set, so DatabaseRead() returns false on
         // success too (nothing to read) - the real signal is the error code. A
         // completed DML step can surface as either 0 or DB_NO_MORE_DATA (5126,
         // SQLite's DONE): the statement RAN, there is just nothing to read back.
         // The tester agent reports 5126 where the live terminal reports 0 for
         // the same completed step; treating it as failure printed a phantom
         // "Failed to insert/update" pair for EVERY row a backtest journaled
         // (~11.7k lines per run) while all of the data landed correctly - and
         // that spam would bury any real error. Genuine failures (constraint,
         // misuse) surface as other codes and still fail here; busy/locked
         // retries the whole statement instead.
         ResetLastError();
         DatabaseRead(handle);
         int error = GetLastError();
         DatabaseFinalize(handle);
         if(error != 0 && error != DB_NO_MORE_DATA)
           {
            if(IsRetryableDbError(error) && attempts + 1 < DB_RETRY_ATTEMPTS)
              {
               attempts++;
               Sleep(DB_RETRY_DELAY_MS);
               continue;
              }
            Print("Failed to execute bound query (error " + IntegerToString(error) + "): " + query);
            return false;
           }
         return true;
        }
      return false; // unreachable - every path through the loop above returns or continues
     }
public:
   //constructor
                     CDatabaseOperationsManager(bool verbose = false)
     {
      m_verboseMode = verbose;
      m_databaseHandle = INVALID_HANDLE;
     }
   void              SetDatabaseHandle(int dbHandle)
     {
      m_databaseHandle = dbHandle;
     }
   // Create a table in the database
   bool              CreateTable(string tableName, string tableSchema)
     {
      if(!RequireValidIdentifier(tableName, "create"))
         return false;
      if(!DatabaseTableExists(m_databaseHandle, tableName))
        {
         string createTableQuery = "CREATE TABLE " + Q(tableName) + " (" + tableSchema + ")";
         if(PrepareAndExecuteQuery(createTableQuery))
            return true;
         Print("Failed to create table " + tableName);
         return false;
        }
      return true; // Table already exists
     }
   //--- SCHEMA EVOLUTION WITHOUT A VERSION WIPE (2026-09-06). CreateTable() is IF-NOT-EXISTS, so a
   //--- column added to a schema string never reaches a table that already exists - and the only
   //--- other path, bumping dbVersion, CLEANS THE WHOLE DATABASE DIRECTORY, journal included. Neither
   //--- is acceptable for adding a column to the trade journal, which is exactly the table whose
   //--- history is the point. SQLite's ALTER TABLE ADD COLUMN appends at the END, which is also the
   //--- only place a new column can go without breaking DatabaseReadBind()'s positional mapping.
   //--- Existence is probed with a zero-row SELECT rather than PRAGMA table_info so no result-set
   //--- struct is needed; a failed prepare means "no such column", nothing else.
   bool              ColumnExists(string tableName, string columnName)
     {
      if(!RequireValidIdentifier(tableName, "inspect") || !RequireValidIdentifier(columnName, "inspect column of"))
         return false;
      if(m_databaseHandle == INVALID_HANDLE)
         return false;
      ResetLastError();
      int handle = DatabasePrepare(m_databaseHandle, "SELECT " + columnName + " FROM " + Q(tableName) + " LIMIT 0");
      if(handle == INVALID_HANDLE)
         return false;
      DatabaseFinalize(handle);
      return true;
     }
   //--- columnType is one of our own literals (TEXT / REAL / INTEGER), never user data.
   bool              AddColumn(string tableName, string columnName, string columnType)
     {
      if(!RequireValidIdentifier(tableName, "alter") || !RequireValidIdentifier(columnName, "add column to"))
         return false;
      if(PrepareAndExecuteQuery("ALTER TABLE " + Q(tableName) + " ADD COLUMN " + columnName + " " + columnType))
         return true;
      Print("Failed to add column " + columnName + " to " + tableName);
      return false;
     }
   bool              EnsureColumn(string tableName, string columnName, string columnType)
     {
      if(ColumnExists(tableName, columnName))
         return true;
      return AddColumn(tableName, columnName, columnType);
     }
   bool              DeleteTable(string tableName)
     {
      if(!RequireValidIdentifier(tableName, "delete"))
         return false;
      string deleteTableQuery = "DROP TABLE IF EXISTS " + Q(tableName);
      if(PrepareAndExecuteQuery(deleteTableQuery))
        {
         PrintVerbose("Table " + tableName + " deleted successfully.");
         return true;
        }
      Print("Failed to delete table " + tableName);
      return false;
     }
   // Insert a trade record into the database. Column names are our own fixed
   // schema field names (never user data); values are bound as parameters.
   //--- a statement with no result set and no parameters (CREATE INDEX ...), through the same retry
   bool              Execute(const string query)
     {
      return PrepareAndExecuteQuery(query);
     }
   //--- DELETE the rows matching a bound WHERE clause. Built for the journal's replace-on-key: a
   //--- tester re-run over the same history must not double-count the trades it already recorded.
   bool              DeleteWhere(string tableName, const string whereClause, const string &params[])
     {
      if(!RequireValidIdentifier(tableName, "delete from"))
         return false;
      return PrepareAndExecuteBound("DELETE FROM " + Q(tableName) + " WHERE " + whereClause, params);
     }
   bool              InsertTradeRecord(string tableName, const string &columns[], const string &values[])
     {
      if(!RequireValidIdentifier(tableName, "insert into"))
         return false;
      string insertQuery = "INSERT INTO " + Q(tableName) + " (" + JoinWithCommas(columns) +
                            ") VALUES (" + PlaceholderList(ArraySize(columns)) + ")";
      if(PrepareAndExecuteBound(insertQuery, values))
         return true;
      Print("Failed to insert trade record into " + tableName);
      return false;
     }
   struct RecordCount
     {
      int            count;
     };
   // Fetch the number of records in the specified table
   bool              FetchRecordCount(string tableName, int &count)
     {
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      RecordCount countStruct;
      string countQuery = "SELECT COUNT(*) as count FROM " + Q(tableName);
      int handle = DatabasePrepare(m_databaseHandle, countQuery);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare COUNT query for " + tableName);
         return false;
        }
      bool ok = DatabaseReadBind(handle, countStruct);
      if(ok)
         count = countStruct.count;
      DatabaseFinalize(handle);
      return ok;
     }
   //--- Targeted lookups. These answer ProcessSignal/UpdateSignalsWeights' questions inside SQLite
   //--- and return one row or one pair of numbers, so per-signal cost stays flat no matter how large
   //--- a table grows. They exist because the original design pulled ENTIRE tables into MQL struct
   //--- arrays for every question - the real constraint behind the historical 1000-row cap: SQLite
   //--- has no such limit, but materializing thousands of string-bearing structs per signal does.
   struct OpenTradeRow
     {
      double         entryPrice;
     };
   // Entry price of the still-open ("NA") trade for pattern+direction, if one exists
   bool              FetchOpenTradeEntry(string tableName, string pattern, string direction,
                                         double &entryPrice, bool &found)
     {
      found = false;
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      string q = "SELECT entryPrice FROM " + Q(tableName) +
                 " WHERE pattern=? AND direction=? AND result='NA' LIMIT 1";
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare open-trade query for " + tableName);
         return false;
        }
      if(!DatabaseBind(handle, 0, pattern) || !DatabaseBind(handle, 1, direction))
        {
         Print("Failed to bind open-trade query for " + tableName);
         DatabaseFinalize(handle);
         return false;
        }
      OpenTradeRow row;
      if(DatabaseReadBind(handle, row))
        {
         entryPrice = row.entryPrice;
         found = true;
        }
      DatabaseFinalize(handle);
      return true;
     }
   struct TimeKeyRow
     {
      long           k;
     };
   // Time key (yyyymmddhhmm as a number) of the table's NEWEST row. Rows are inserted in
   // chronological order, so max ROWID carries the latest timestamp. `quiet` suppresses the
   // prepare-failure print so existence probes over possibly-absent tables don't spam the log.
   bool              FetchNewestTimeKey(string tableName, long &key, bool &found, const bool quiet = false)
     {
      found = false;
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      string q = "SELECT (((year*100+month)*100+day)*100+hour)*100+minutes AS k FROM " +
                 Q(tableName) + " ORDER BY ROWID DESC LIMIT 1";
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         if(!quiet)
            Print("Failed to prepare newest-time query for " + tableName);
         return false;
        }
      TimeKeyRow row;
      if(DatabaseReadBind(handle, row))
        {
         key = row.k;
         found = true;
        }
      DatabaseFinalize(handle);
      return true;
     }
   struct ExpectancyRow
     {
      int            n;
      double         sumR;
     };
   //--- MEAN R PER FIRING, not win rate. A pattern that is right 30% of the time and wins three
   //--- times what it loses beats one that is right 70% and does the opposite, and a win-rate
   //--- ranking cannot tell them apart - it scores the first 30 and the second 70, exactly
   //--- backwards. Same beforeKey bound as the counts below, for the same reason.
   //--- A TABLE THAT DOES NOT EXIST IS NOT AN ERROR HERE, it is the normal state of a pattern that
   //--- has not fired yet. Without this guard DatabasePrepare() fails and the TERMINAL logs
   //--- "database error, no such table" itself - once per absent table, per module, per daily
   //--- rerank. Over a 16-year pass that is tens of thousands of lines that mean nothing, burying
   //--- the ones that do. It also skips the wasted prepare.
   bool              TableExists(const string tableName)
     {
      return ::DatabaseTableExists(m_databaseHandle, tableName);
     }
   bool              FetchExpectancy(string tableName, long beforeKey, int &n, double &meanR,
                                  const int regime = -1)
     {
      n = 0;
      meanR = 0.0;
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      if(!TableExists(tableName))
         return false;
      //--- `regime` < 0 means "every regime" - the unconditional record. Otherwise the count is
      //--- restricted to firings that happened under the SAME conditions as now, which is the
      //--- whole point: a trend-following pattern is good in a trend and bad in a range, and
      //--- averaging the two produces the coin flip that was measured across every module.
      string q = "SELECT COUNT(rMultiple), COALESCE(SUM(rMultiple),0) FROM " + Q(tableName) +
                 " WHERE rMultiple IS NOT NULL AND "
                 "(((year*100+month)*100+day)*100+hour)*100+minutes < " + IntegerToString(beforeKey);
      if(regime >= 0)
         q += " AND regime = " + IntegerToString(regime);
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
         return false;
      ExpectancyRow row;
      bool ok = DatabaseReadBind(handle, row);
      if(ok && row.n > 0)
        {
         n = row.n;
         meanR = row.sumR / row.n;
        }
      DatabaseFinalize(handle);
      return ok;
     }
   struct WinLossRow
     {
      int            wins;
      int            losses;
     };
   // Closed-trade outcome counts with open time strictly before beforeKey (yyyymmddhhmm). The bound
   // is what keeps a resumed/mixed database's not-yet-current rows out of the weights in the tester.
   bool              FetchWinLossCounts(string tableName, long beforeKey, int &wins, int &losses)
     {
      wins = 0;
      losses = 0;
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      if(!TableExists(tableName))
         return false;                 // see TableExists - an absent table is a pattern, not a fault
      string q = "SELECT COALESCE(SUM(result='Profit'),0), COALESCE(SUM(result='Loss'),0) FROM " +
                 tableName + " WHERE result!='NA' AND (((year*100+month)*100+day)*100+hour)*100+minutes < " +
                 IntegerToString(beforeKey);
      int handle = DatabasePrepare(m_databaseHandle, q);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare win-loss query for " + tableName);
         return false;
        }
      WinLossRow row;
      bool ok = DatabaseReadBind(handle, row);
      if(ok)
        {
         wins = row.wins;
         losses = row.losses;
        }
      DatabaseFinalize(handle);
      return ok;
     }
   // Fetch trade records from specified table and read into a structure, then populate the dynamic array with the data
   template <typename T>
   bool              FetchTradeRecords(string tableName, T &tradeRecordStructure, T &tradeRecords[])
     {
      if(!RequireValidIdentifier(tableName, "query"))
         return false;
      int recordCount;
      if(!FetchRecordCount(tableName, recordCount))
         return false;
      ArrayResize(tradeRecords, recordCount);
      string selectQuery = "SELECT * FROM " + Q(tableName);
      int handle = DatabasePrepare(m_databaseHandle, selectQuery);
      if(handle == INVALID_HANDLE)
        {
         Print("Failed to prepare SELECT query for " + tableName);
         return false;
        }
      int i = 0;
      for(; i < recordCount && DatabaseReadBind(handle, tradeRecordStructure); i++)
         tradeRecords[i] = tradeRecordStructure;
      DatabaseFinalize(handle);
      if(i < recordCount)
        {
         // The read loop can exit early (a concurrent delete on the shared COMMON db between the
         // COUNT and SELECT above, or a transient read failure) - trim the array to what was
         // actually read instead of leaving trailing zero/empty-string structs that downstream
         // reporting would otherwise silently count as real (empty-symbol, 0-profit) trades.
         Print(__FUNCTION__ + ": only read " + IntegerToString(i) + " of " + IntegerToString(recordCount) +
               " expected rows from " + tableName + " - trimming to actual rows read.");
         ArrayResize(tradeRecords, i);
        }
      return true;
     }
   // Update the exit price/result of the still-open ("NA") trade matching pattern+direction
   bool              UpdateTradeRecord(string tableName, const string &columns[], const string &values[], string pattern, string direction)
     {
      if(!RequireValidIdentifier(tableName, "update"))
         return false;
      string updateQuery = "UPDATE " + Q(tableName) + " SET " + JoinWithCommas(columns, "=?") +
                            " WHERE pattern=? AND direction=? AND result='NA'";
      string params[];
      ArrayResize(params, ArraySize(values) + 2);
      for(int i = 0; i < ArraySize(values); i++)
         params[i] = values[i];
      params[ArraySize(values)] = pattern;
      params[ArraySize(values) + 1] = direction;
      if(PrepareAndExecuteBound(updateQuery, params))
         return true;
      Print("Failed to update trade records in " + tableName);
      return false;
     }
   // Delete the oldest entry from a specified table
   bool              DeleteOldestEntry(string tableName)
     {
      if(!RequireValidIdentifier(tableName, "delete from"))
         return false;
      string deleteQuery = "DELETE FROM " + Q(tableName) + " WHERE ROWID = (SELECT MIN(ROWID) FROM " + Q(tableName) + ")";
      if(PrepareAndExecuteQuery(deleteQuery))
        {
         PrintVerbose("Oldest entry deleted from " + tableName + " successfully.");
         return true;
        }
      Print("Failed to delete oldest entry from " + tableName);
      return false;
     }
  };
//+------------------------------------------------------------------+