mql5/Experts/Advisors/DualEA/Include/KnowledgeBase.mqh

// KnowledgeBase.mqh
// Shared functions for reading from and writing to the knowledge base.

#property copyright "2025, Windsurf Engineering"
#property link      "https://www.windsurf.ai"

#include <Files\File.mqh>
// Centralized default strategy names
#include "Strategies\Registry.mqh"
// Optional cancellation flag (default false). Host programs may toggle this.
// Guard the definition to avoid duplicate symbols if this header is included multiple times in a unit.
#ifndef KB_CANCEL_FLAG_DEFINED
#define KB_CANCEL_FLAG_DEFINED 1
static bool g_insights_cancel_requested = false;
#endif

// --- Defines the structure for a single trade record
struct TradeRecord
  {
   datetime          timestamp;        // Trade execution timestamp
   string            symbol;           // Trading symbol
   ENUM_ORDER_TYPE   type;             // Order type (e.g., ORDER_TYPE_BUY, ORDER_TYPE_SELL)
   double            entry_price;      // Price at which the trade was opened
   double            stop_loss;        // Stop loss level
   double            take_profit;      // Take profit level
   double            close_price;      // Price at which the trade was closed
   double            profit;           // Profit or loss from the trade
   string            strategy_id;      // ID of the strategy that generated the trade
  };
// Uncomment to enable verbose parsing diagnostics
//#define INSIGHTS_DEBUG

// --- Class to manage knowledge base operations
class CKnowledgeBase
  {
private:
   string            m_file_path;      // Path to the knowledge base file
   int               m_file_handle;    // File handle
   string            m_csv_delimiter;  // CSV delimiter
   string            m_lock_path;      // Path to lock file in Common Files
   int               m_lock_handle;    // Lock file handle

public:
                     CKnowledgeBase(string file_name="DualEA\\knowledge_base.csv", string delimiter=",");
                     ~CKnowledgeBase();

   // --- Methods for data handling
   bool              WriteRecord(const TradeRecord &record);
   bool              LogTrade(const string strategy_name, const int retcode, const ulong deal, const ulong order_id);
   // Convenience logging used by PaperEA_v2
   bool              LogTradeExecution(const string symbol, const string strategy, const datetime exec_time,
                                       const double price, const double volume, const int order_type)
                     {
                        // Append a single trade execution row into Common Files under DualEA/trades
                        FolderCreate("DualEA\\trades", FILE_COMMON);
                        string filename = StringFormat("DualEA\\trades\\%s_trades_%s.csv", symbol, TimeToString(exec_time, TIME_DATE));
                        int h = FileOpen(filename, FILE_READ|FILE_WRITE|FILE_COMMON|FILE_CSV|FILE_ANSI, ',');
                        if(h == INVALID_HANDLE)
                        {
                           PrintFormat("KnowledgeBase: failed to open trades file: %s (err=%d)", filename, GetLastError());
                           return false;
                        }
                        if(FileSize(h) == 0)
                        {
                           FileWriteString(h, "timestamp,strategy,price,volume,type\n");
                        }
                        FileSeek(h, 0, SEEK_END);
                        FileWrite(h, TimeToString(exec_time), strategy, DoubleToString(price, 5),
                                  DoubleToString(volume, 2), IntegerToString(order_type));
                        FileClose(h);
                        return true;
                     }

private:
   bool              OpenFile(int flags=FILE_WRITE|FILE_READ|FILE_CSV);
   void              CloseFile();
   bool              AcquireLock(const int timeout_ms=3000);
   void              ReleaseLock();
  };

// --- Class to manage features.csv logging (long format: one feature per row)
class CFeaturesKB
  {
 private:
    string m_file_path;
    int    m_file_handle;
    string m_csv_delim;
    void   EnsureHeader()
      {
       // Ensure folder exists and header is present
       FolderCreate("DualEA", FILE_COMMON);
       int h = FileOpen(m_file_path, FILE_READ|FILE_CSV|FILE_COMMON);
       if(h==INVALID_HANDLE)
         {
          h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
          if(h!=INVALID_HANDLE)
            {
             FileWriteString(h, "timestamp,symbol,strategy,feature,value\n");
             FileClose(h);
            }
         }
       else
         {
          if(FileSize(h)==0)
            {
             FileClose(h);
             h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
             if(h!=INVALID_HANDLE)
                FileWriteString(h, "timestamp,symbol,strategy,feature,value\n");
            }
          if(h!=INVALID_HANDLE) FileClose(h);
         }
       string full = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_file_path;
       PrintFormat("KnowledgeBase: writing features CSV to Common Files: %s", full);
      }
 public:
                  CFeaturesKB(string file_name="DualEA\\features.csv", string delimiter=",")
                    {
                     m_file_path = file_name;
                     m_csv_delim = delimiter;
                     m_file_handle = INVALID_HANDLE;
                     EnsureHeader();
                    }
                  ~CFeaturesKB() {}
    bool          WriteKV(const datetime ts, const string symbol, const string strategy, const string feature, const double value)
                    {
                     int h = INVALID_HANDLE;
                     // File size rotation threshold (100MB)
                     int rotate_threshold = 100*1024*1024;
                     // Check if file exists and exceeds threshold
              int hsize = FileOpen(m_file_path, FILE_READ|FILE_COMMON);
              if(hsize != INVALID_HANDLE)
              {
                ulong sz = FileSize(hsize);
                FileClose(hsize);
                if(sz > (ulong)rotate_threshold)
                {
                    string tsuffix = TimeToString(TimeCurrent(), TIME_DATE|TIME_MINUTES|TIME_SECONDS);
                  StringReplace(tsuffix, ":", ""); StringReplace(tsuffix, ".", ""); StringReplace(tsuffix, " ", "_");
                    string rotated = m_file_path + "." + tsuffix + ".bak";
                  // Try to rename (no compression in MQL5)
                  if(FileIsExist(rotated, FILE_COMMON)) FileDelete(rotated, FILE_COMMON);
                 // Signature: FileMove(src_name, src_common_flag, dst_name, dst_common_flag)
                 if(FileMove(m_file_path, FILE_COMMON, rotated, FILE_COMMON))
                    PrintFormat("[ROTATE] features.csv rotated to %s", rotated);
                 else
                    PrintFormat("[ROTATE] features.csv rotation failed: %s", rotated);
                  EnsureHeader();
                }
              }
                     // Retry open to mitigate transient locks from readers/writers
                     for(int attempt=0; attempt<10 && h==INVALID_HANDLE; ++attempt)
                       {
                        h = FileOpen(m_file_path, FILE_READ|FILE_WRITE|FILE_CSV|FILE_COMMON|FILE_UNICODE, ',');
                        if(h==INVALID_HANDLE) Sleep(25);
                       }
                     if(h==INVALID_HANDLE)
                       {
                        PrintFormat("Error opening features file '%s'. Error: %d", m_file_path, GetLastError());
                        return false;
                       }
              if(FileSize(h)==0)
              {
                // Write header using FileWrite for proper CSV formatting
                FileWrite(h, "timestamp", "symbol", "strategy", "feature", "value");
              }
              FileSeek(h, 0, SEEK_END);
              // Write data fields using FileWrite (proper CSV encoding)
              FileWrite(h, TimeToString(ts, TIME_DATE|TIME_MINUTES|TIME_SECONDS), symbol, strategy, feature, DoubleToString(value, 8));
              FileClose(h);
              return true;
                    }

    // Export features for ML training in a compact CSV form per strategy/symbol/date
    bool          ExportFeatures(const string symbol, const string strategy, const datetime timestamp, const string &features[])
                    {
                     if(ArraySize(features) == 0) return false;
                     // Ensure base directory exists
                     FolderCreate("DualEA\\features", FILE_COMMON);
                     string filename = StringFormat("DualEA\\features\\%s_%s_%s.csv", symbol, strategy, TimeToString(timestamp, TIME_DATE));
                     int h = FileOpen(filename, FILE_WRITE|FILE_COMMON|FILE_CSV|FILE_ANSI, ',');
                     if(h == INVALID_HANDLE)
                       {
                        PrintFormat("KnowledgeBase: failed to create features file: %s (err=%d)", filename, GetLastError());
                        return false;
                       }
                     // Write header row
                     FileWrite(h, "timestamp", TimeToString(timestamp));
                     // Write provided feature key:value pairs
                     for(int i = 0; i < ArraySize(features); ++i)
                       {
                        string parts[]; int cnt = StringSplit(features[i], ':', parts);
                        if(cnt == 2) { FileWrite(h, parts[0], parts[1]); }
                        else { FileWrite(h, StringFormat("feature_%d", i), features[i]); }
                       }
                     FileClose(h);
                     return true;
                    }

    // Append a single trade execution record to a consolidated per-symbol CSV in Common Files
    bool          LogTradeExecution(const string symbol, const string strategy, const datetime exec_time,
                                    const double price, const double volume, const int order_type)
                    {
                     FolderCreate("DualEA\\trades", FILE_COMMON);
                     // Use consolidated file per symbol (not per day)
                     string filename = StringFormat("DualEA\\trades\\%s_trades.csv", symbol);
                     // Open as CSV with FILE_UNICODE for proper UTF-8 encoding
                     int h = FileOpen(filename, FILE_READ|FILE_WRITE|FILE_CSV|FILE_COMMON|FILE_UNICODE, ',');
                     if(h == INVALID_HANDLE)
                       {
                        PrintFormat("KnowledgeBase: failed to open trades file: %s (err=%d)", filename, GetLastError());
                        return false;
                       }
                     // Check if file is new (size 0) - write header if so using FileWrite
                     if(FileSize(h) == 0)
                     {
                        // Write header row using FileWrite for proper CSV format
                        FileWrite(h, "timestamp", "strategy", "price", "volume", "type");
                     }
                     // Seek to end to append
                     FileSeek(h, 0, SEEK_END);
                     // Write trade data using FileWrite (handles CSV encoding properly)
                     FileWrite(h, 
                               TimeToString(exec_time, TIME_DATE|TIME_MINUTES|TIME_SECONDS),
                               strategy,
                               DoubleToString(price, 5),
                               DoubleToString(volume, 2),
                               IntegerToString(order_type));
                     FileClose(h);
                     return true;
                    }
  };

// --- Insights builder: reads KB and emits insights.json (basic counts scaffold)
class CInsightsBuilder
  {
 private:
    string m_kb_path;
    string m_features_path;
    string m_out_path;
    string m_delim;
    int    m_timeout_ms; // 0=disabled
    bool   m_verbose;    // progress logging
    // --- helpers
    string Trim(string s)
      {
       StringTrimLeft(s);
        StringTrimRight(s);
        return s;
      }
    string StripQuotes(string s)
      {
       s = Trim(s);
       // strip BOM if present (U+FEFF or UTF-8 BOM 0xEF 0xBB 0xBF)
       if(StringLen(s)>0)
         {
          int c0 = StringGetCharacter(s,0);
          // U+FEFF
          if(c0==65279)
             s = StringSubstr(s,1);
          else if(StringLen(s)>=3)
            {
             int b0 = c0;
             int b1 = StringGetCharacter(s,1);
             int b2 = StringGetCharacter(s,2);
             if(b0==239 && b1==187 && b2==191)
                s = StringSubstr(s,3);
            }
         }
       // remove surrounding quotes
       if(StringLen(s)>=2 && StringSubstr(s,0,1)=="\"" && StringSubstr(s,StringLen(s)-1,1)=="\"")
          s = StringSubstr(s,1,StringLen(s)-2);
       // remove stray quotes
       StringReplace(s, "\"", "");
       // remove CR/LF and NBSP
       StringReplace(s, "\r", "");
        StringReplace(s, "\n", "");
        // remove non-breaking space (char code 160)
        for(int i=0;i<StringLen(s);++i){ if(StringGetCharacter(s,i)==160){ s = StringSubstr(s,0,i)+StringSubstr(s,i+1); i--; } }
       return s;
      }
    string NormalizeToken(string s)
      {
       s = StripQuotes(s);
       // StringToLower modifies in place; do not assign its return value
       StringToLower(s);
       string out="";
       for(int i=0;i<StringLen(s);++i)
         {
          int ch = StringGetCharacter(s,i);
          bool ok = (ch>='a' && ch<='z') || (ch>='0' && ch<='9') || (ch=='_');
          if(ok)
             out = out + StringSubstr(s,i,1);
         }
       return out;
      }
    int    FindIndexString(string &arr[], const string key)
      {
       for(int i=0;i<ArraySize(arr);++i)
         if(arr[i]==key) return i;
       return -1;
      }
    void   PushUnique(string &arr[], const string key)
      {
       if(FindIndexString(arr,key)<0)
         {
          int n = ArraySize(arr);
          ArrayResize(arr, n+1);
          arr[n] = key;
         }
      }
    double SafeDiv(const double a, const double b)
      {
       if(b==0.0) return 0.0;
       return a/b;
      }
    double Median(double &values[])
      {
       int n = ArraySize(values);
       if(n==0) return 0.0;
       // Work on a copy to avoid side effects
       double tmp[];
       ArrayResize(tmp, n);
       for(int i=0;i<n;++i) tmp[i]=values[i];
       ArraySort(tmp);
       if((n % 2)==1)
          return tmp[n/2];
       return 0.5*(tmp[n/2-1] + tmp[n/2]);
      }
    double MaxDrawdownR(double &r_series[])
      {
       // Computes max peak-to-trough drawdown on cumulative R
       int n = ArraySize(r_series);
       if(n==0) return 0.0;
       double cum=0.0, peak=0.0, maxdd=0.0;
       for(int i=0;i<n;++i)
         {
          cum += r_series[i];
          if(cum>peak) peak=cum;
          double dd = peak - cum; // in R
          if(dd>maxdd) maxdd=dd;
         }
       return maxdd;
      }
    // --- Deadline helper: returns true if timeout exceeded or external cancel requested
    bool   DeadlineExceeded(const ulong t0)
      {
       if(m_timeout_ms>0)
         {
          ulong now = GetTickCount();
          if((long)(now - t0) >= m_timeout_ms)
             return true;
         }
       if(g_insights_cancel_requested)
          return true;
       return false;
      }
  public:
                  CInsightsBuilder(string kb_path="DualEA\\knowledge_base.csv", string features_path="DualEA\\features.csv", string out_path="DualEA\\insights.json", string delim=",")
                    {
                     m_kb_path = kb_path;
                     m_features_path = features_path;
                     m_out_path = out_path;
                     m_delim = delim;
                     m_timeout_ms = 0;
                     m_verbose = false;
                    }
  void          SetTimeoutMs(const int ms){ m_timeout_ms = ms; }
  void          SetVerbose(const bool v){ m_verbose = v; }
    bool          Build()
                    {
                    // Build insights from features.csv using r_multiple rows
                    // Establish deadline (if any)
                    ulong t0 = GetTickCount();
                    // Open features.csv with explicit comma delimiter
                    int hf = FileOpen(m_features_path, FILE_READ|FILE_CSV|FILE_COMMON, (ushort)',');
                    if(hf==INVALID_HANDLE)
                      {
                       PrintFormat("InsightsBuilder: cannot open features '%s'. Err=%d", m_features_path, GetLastError());
                       return false;
                      }

                    // Storage of r-multiples with metadata (plus optional timeframe)
                    datetime ts_list[]; string sym_list[]; string strat_list[]; double r_list[]; int tf_list[];
                    // Map (symbol|strategy) -> timeframe (last seen)
                    string tf_keys[]; int tf_vals[];
                    // Collect all observed (symbol|strategy) pairs to support synthesis when timeframe is absent
                    string all_keys[];

                    // Robust CSV scan: 5 fields per row, skip header if present
                    bool header_checked=false;
                    int parsed_rows=0;
                    int chunk_size = 30000000; // large chunks to reduce progress log frequency
                    while(!FileIsEnding(hf))
                      {
                       // Periodic watchdog/cancel check and progress logging
                       if((parsed_rows % chunk_size)==0 && parsed_rows > 0)
                        {
                         if(m_verbose) PrintFormat("InsightsBuilder: progress: parsed_rows=%d", parsed_rows);
                         Sleep(50); // yield to terminal, avoid watchdog
                         if(DeadlineExceeded(t0))
                           {
                            FileClose(hf);
                            PrintFormat("InsightsBuilder: aborting features.csv scan at row=%d due to timeout/cancel (timeout_ms=%d)", parsed_rows, m_timeout_ms);
                            return false;
                           }
                        }
                       string ts = FileReadString(hf);
                       if(ts=="" && FileIsEnding(hf)) break;
                       string sym = FileReadString(hf);
                       string strat = FileReadString(hf);
                       string feat = FileReadString(hf);
                       string val  = FileReadString(hf);

                       // Trim all fields to avoid CR/whitespace issues
                       ts    = StripQuotes(ts);
                       sym   = StripQuotes(sym);
                       strat = StripQuotes(strat);
                       feat  = StripQuotes(feat);
                       val   = StripQuotes(val);

                       if(feat=="") { continue; }
                       if(!header_checked)
                         {
                          header_checked=true;
                          string ts_norm = NormalizeToken(ts);
                          string feat_norm_dbg = NormalizeToken(feat);
                          int c0 = (StringLen(ts)>0? StringGetCharacter(ts,0) : -1);
                          #ifdef INSIGHTS_DEBUG
                          PrintFormat("InsightsBuilder: header-check ts_raw='%s' ts_norm='%s' c0=%d feat_raw='%s' feat_norm='%s'", ts, ts_norm, c0, feat, feat_norm_dbg);
                          #endif
                          if(ts_norm=="timestamp")
                              continue; // skip header row
                         }

                       // Count this parsed data row (post header)
                       parsed_rows++;
                       if(parsed_rows<=5)
                         {
                          string feat_norm = NormalizeToken(feat);
                          #ifdef INSIGHTS_DEBUG
                          PrintFormat("InsightsBuilder: sample row %d feat='%s' feat_norm='%s' val='%s'", parsed_rows-1, feat, feat_norm, val);
                          #endif
                         }

                       // Track (symbol|strategy) pairs from any feature rows
                       if(sym!="" && strat!="")
                         {
                          string key_any = sym + "|" + strat;
                          PushUnique(all_keys, key_any);
                         }

                       string feat_norm_main = NormalizeToken(feat);
                       if(feat_norm_main=="timeframe")
                         {
                          // record latest timeframe for (symbol,strategy)
                          int tfv = (int)StringToInteger(val);
                          string key = sym+"|"+strat;
                          bool updated=false;
                          for(int i=0;i<ArraySize(tf_keys);++i){ if(tf_keys[i]==key){ tf_vals[i]=tfv; updated=true; break; } }
                          if(!updated){ int k=ArraySize(tf_keys); ArrayResize(tf_keys,k+1); ArrayResize(tf_vals,k+1); tf_keys[k]=key; tf_vals[k]=tfv; }
                         }
                       else if(feat_norm_main=="r_multiple")
                         {
                          int n = ArraySize(r_list);
                          ArrayResize(ts_list, n+1);
                          ArrayResize(sym_list, n+1);
                          ArrayResize(strat_list, n+1);
                          ArrayResize(r_list, n+1);
                          ArrayResize(tf_list, n+1);
                          ts_list[n]    = StringToTime(ts);
                          sym_list[n]   = sym;
                          strat_list[n] = strat;
                          r_list[n]     = StringToDouble(val);
                          // lookup timeframe if available
                          int tfv=-1; string key2 = sym_list[n]+"|"+strat_list[n];
                          for(int j=0;j<ArraySize(tf_keys);++j){ if(tf_keys[j]==key2){ tfv=tf_vals[j]; break; } }
                          tf_list[n]    = tfv;
                          #ifdef INSIGHTS_DEBUG
                          if(n<5)
                            PrintFormat("InsightsBuilder: captured r_multiple row ts=%s sym=%s strat=%s r=%f", ts, sym, strat, r_list[n]);
                          #endif
                         }
                      }
                    FileClose(hf);

                    int total = ArraySize(r_list);
                    PrintFormat("InsightsBuilder: scanned rows=%d, r_multiple rows=%d from features '%s'", parsed_rows, total, m_features_path);
                    // --- Fallback: if no r_multiple labels were found, derive metrics from knowledge_base.csv using profit as surrogate
                    if(total==0)
                      {
                       // Secondary fallback: parse features.csv as raw text lines and split by comma
                       int ht = FileOpen(m_features_path, FILE_READ|FILE_COMMON|FILE_ANSI);
                       if(ht!=INVALID_HANDLE)
                         {
                          int added=0; bool header_seen=false; int scanned=0;
                          while(!FileIsEnding(ht))
                            {
                             string line = FileReadString(ht);
                              line = Trim(line);
                              if(line=="") continue;
                              // Periodic watchdog/cancel check
                              if((scanned % 5000)==0 && DeadlineExceeded(t0))
                                {
                                 FileClose(ht);
                                 PrintFormat("InsightsBuilder: aborting features.csv text-fallback at scanned=%d due to timeout/cancel (timeout_ms=%d)", scanned, m_timeout_ms);
                                 return false;
                                }
                              string parts[];
                              ushort sep = (ushort)StringGetCharacter(",",0);
                              int cnt = StringSplit(line, sep, parts);
                              if(cnt<5) continue;
                              // parts: 0 ts,1 sym,2 strat,3 feat,4 val
                              string ts_s = StripQuotes(parts[0]);
                             if(!header_seen)
                               {
                                header_seen=true;
                                if(NormalizeToken(ts_s)=="timestamp") continue; // skip header
                               }
                             // Track (symbol|strategy) pairs from any feature rows (text fallback)
                             string sy0 = StripQuotes(parts[1]);
                             string st0 = StripQuotes(parts[2]);
                             if(sy0!="" && st0!="") PushUnique(all_keys, sy0+"|"+st0);
                             string f_s  = StripQuotes(parts[3]);
                             string f_norm = NormalizeToken(f_s);
                             if(f_norm=="timeframe")
                               {
                                int tfv = (int)StringToInteger(StripQuotes(parts[4]));
                                string sy = StripQuotes(parts[1]); string st = StripQuotes(parts[2]);
                                string key = sy+"|"+st; bool updated=false;
                                for(int i2=0;i2<ArraySize(tf_keys);++i2){ if(tf_keys[i2]==key){ tf_vals[i2]=tfv; updated=true; break; } }
                                if(!updated){ int k2=ArraySize(tf_keys); ArrayResize(tf_keys,k2+1); ArrayResize(tf_vals,k2+1); tf_keys[k2]=key; tf_vals[k2]=tfv; }
                               }
                              else if(f_norm=="r_multiple")
                                {
                                 int n = ArraySize(r_list);
                                 ArrayResize(ts_list, n+1);
                                 ArrayResize(sym_list, n+1);
                                 ArrayResize(strat_list, n+1);
                                 ArrayResize(r_list, n+1);
                                 ArrayResize(tf_list, n+1);
                                 ts_list[n]    = StringToTime(ts_s);
                                 sym_list[n]   = StripQuotes(parts[1]);
                                 strat_list[n] = StripQuotes(parts[2]);
                                 r_list[n]     = StringToDouble(StripQuotes(parts[4]));
                                 // lookup timeframe if available
                                 int tfv=-1; string key2 = sym_list[n]+"|"+strat_list[n];
                                 for(int j2=0;j2<ArraySize(tf_keys);++j2){ if(tf_keys[j2]==key2){ tfv=tf_vals[j2]; break; } }
                                 tf_list[n]    = tfv;
                                 added++;
                                }
                            scanned++;
                            }
                          FileClose(ht);
                          if(added>0)
                            {
                             total = ArraySize(r_list);
                             PrintFormat("InsightsBuilder: text-fallback collected %d r_multiple rows from features.csv", added);
                            }
                         }
                       
                       Print("InsightsBuilder: no r_multiple rows found in features.csv; falling back to knowledge_base.csv using profit as surrogate");
                       // Check if knowledge_base.csv exists first
                       if(!FileIsExist(m_kb_path, FILE_COMMON))
                       {
                          Print("InsightsBuilder: knowledge_base.csv does not exist, skipping fallback");
                       }
                       else
                       {
                          // Open knowledge_base.csv with explicit comma delimiter
                          int hk = FileOpen(m_kb_path, FILE_READ|FILE_CSV|FILE_COMMON, (ushort)',' );
                       if(hk!=INVALID_HANDLE)
                         {
                          // No header expected in knowledge_base.csv
                          // Columns: ts,symbol,type,entry,sl,tp,close,profit,strategy
                          string sym_list2[]; string strat_list2[]; double r_list2[]; // use profit as R substitute
                          int kreads=0;
                          while(!FileIsEnding(hk))
                            {
                             string ts = FileReadString(hk); if(ts=="" && FileIsEnding(hk)) break;
                             string sym = FileReadString(hk);
                             string type= FileReadString(hk);
                             string entry=FileReadString(hk);
                             string sl  = FileReadString(hk);
                             string tp  = FileReadString(hk);
                             string close=FileReadString(hk);
                             string profit=FileReadString(hk);
                             string strat = FileReadString(hk);

                             // Trim fields
                             ts=Trim(ts); sym=Trim(sym); type=Trim(type); entry=Trim(entry); sl=Trim(sl); tp=Trim(tp); close=Trim(close); profit=Trim(profit); strat=Trim(strat);
                             if(profit=="") continue;
                             // consider only rows that look like closures (close price present OR profit non-zero)
                             double pr = StringToDouble(profit);
                             double cp = StringToDouble(close);
                             if(cp>0.0 || pr!=0.0)
                               {
                                int n = ArraySize(r_list2);
                                ArrayResize(r_list2, n+1);
                                ArrayResize(sym_list2, n+1);
                                ArrayResize(strat_list2, n+1);
                                r_list2[n] = pr;
                                sym_list2[n] = sym;
                                strat_list2[n] = strat;
                               }
                             // Periodic watchdog/cancel check
                             if((kreads % 5000)==0 && DeadlineExceeded(t0))
                               {
                                FileClose(hk);
                                PrintFormat("InsightsBuilder: aborting knowledge_base.csv scan at rows=%d due to timeout/cancel (timeout_ms=%d)", kreads, m_timeout_ms);
                                return false;
                               }
                             kreads++;
                            }
                          FileClose(hk);
                          // replace primary arrays for aggregation below
                          ArraySwap(r_list, r_list2);
                          ArraySwap(strat_list, strat_list2);
                          ArraySwap(sym_list, sym_list2);
                          total = ArraySize(r_list);
                          PrintFormat("InsightsBuilder: fallback collected %d rows from knowledge_base.csv", total);
                          // Initialize tf_list for KB-fallback rows using timeframe map from features
                          ArrayResize(tf_list, total);
                          for(int i3=0;i3<total;++i3)
                            {
                             tf_list[i3] = -1;
                             string key3 = sym_list[i3] + "|" + strat_list[i3];
                             for(int j3=0;j3<ArraySize(tf_keys);++j3)
                               {
                                if(tf_keys[j3]==key3){ tf_list[i3]=tf_vals[j3]; break; }
                               }
                             }
                          // If still no rows after KB fallback, synthesize dummy rows from timeframe features
                          if(total==0)
                            {
                             int combos = ArraySize(tf_keys);
                             for(int c=0;c<combos;++c)
                               {
                                int sep = StringFind(tf_keys[c], "|", 0);
                                if(sep<0) continue;
                                int n2 = ArraySize(r_list);
                                ArrayResize(r_list, n2+1);
                                ArrayResize(strat_list, n2+1);
                                ArrayResize(sym_list, n2+1);
                                ArrayResize(tf_list, n2+1);
                                sym_list[n2]   = StringSubstr(tf_keys[c], 0, sep);
                                strat_list[n2] = StringSubstr(tf_keys[c], sep+1);
                                r_list[n2]     = 0.0; // scaffold count only
                                tf_list[n2]    = tf_vals[c];
                               }
                             total = ArraySize(r_list);
                             if(total>0)
                                PrintFormat("InsightsBuilder: synthesized %d dummy rows from timeframe features (no r_multiple/profit found)", total);
                            }
                          }
                         }
                    }
                    // Final synthesis: if still no rows and no timeframe entries, synthesize from symbol/strategy pairs using default chart timeframe
                     if(total==0 && ArraySize(all_keys)>0)
                      {
                       int tf_def = (int)Period();
                       for(int c=0; c<ArraySize(all_keys); ++c)
                         {
                          int sep = StringFind(all_keys[c], "|", 0);
                          if(sep<0) continue;
                          int n2 = ArraySize(r_list);
                          ArrayResize(r_list, n2+1);
                          ArrayResize(strat_list, n2+1);
                          ArrayResize(sym_list, n2+1);
                          ArrayResize(tf_list, n2+1);
                          sym_list[n2]   = StringSubstr(all_keys[c], 0, sep);
                          strat_list[n2] = StringSubstr(all_keys[c], sep+1);
                          r_list[n2]     = 0.0; // scaffold count only
                          tf_list[n2]    = tf_def;
                         }
                       total = ArraySize(r_list);
                       if(total>0)
                          PrintFormat("InsightsBuilder: synthesized %d dummy rows from symbol/strategy pairs (default timeframe=%d)", total, tf_def);
                      }
                    // Ensure general coverage on current chart symbol across multiple timeframes for all default strategies
                    {
                     string def_strats[]; GetDefaultStrategyNames(def_strats);
                     string csym = Symbol();
                     int ctf = (int)Period();
                     // Small TF set including current
                     int tf_opts[]; ArrayResize(tf_opts, 5);
                     tf_opts[0]=ctf; tf_opts[1]=PERIOD_M5; tf_opts[2]=PERIOD_M10; tf_opts[3]=PERIOD_M15; tf_opts[4]=PERIOD_M30;
                     int added_before = ArraySize(r_list);
                     for(int t=0; t<ArraySize(tf_opts); ++t)
                       {
                        int tfc = tf_opts[t];
                        for(int ds=0; ds<ArraySize(def_strats); ++ds)
                          {
                           bool exists=false;
                           for(int i=0;i<total;++i)
                             {
                              if(sym_list[i]==csym && strat_list[i]==def_strats[ds] && tf_list[i]==tfc)
                                { exists=true; break; }
                             }
                           if(!exists)
                             {
                              int n3 = ArraySize(r_list);
                              ArrayResize(r_list, n3+1);
                              ArrayResize(strat_list, n3+1);
                              ArrayResize(sym_list, n3+1);
                              ArrayResize(tf_list, n3+1);
                              sym_list[n3]   = csym;
                              strat_list[n3] = def_strats[ds];
                              r_list[n3]     = 0.0;
                              tf_list[n3]    = tfc;
                             }
                          }
                         // Periodic watchdog/cancel check during coverage synthesis
                         if(DeadlineExceeded(t0))
                           {
                            PrintFormat("InsightsBuilder: aborting during coverage synthesis due to timeout/cancel (timeout_ms=%d)", m_timeout_ms);
                            return false;
                           }
                       }
                     int added_after = ArraySize(r_list);
                     if(added_after>added_before)
                        PrintFormat("InsightsBuilder: ensured multi-TF chart coverage by adding %d rows for symbol=%s across %d TFs and %d strategies", added_after-added_before, csym, ArraySize(tf_opts), ArraySize(def_strats));
                    }
                    // Unique strategies and symbols
                    string uniq_strats[]; string uniq_syms[];
                    for(int i=0;i<total;++i){ PushUnique(uniq_strats, strat_list[i]); PushUnique(uniq_syms, sym_list[i]); }

                    // Overall aggregates
                    double wins=0.0, sumR=0.0, gp=0.0, gl=0.0;
                    double r_all[];
                    ArrayResize(r_all, total);
                    for(int i=0;i<total;++i)
                      {
                       double r = r_list[i];
                       r_all[i]=r;
                       sumR += r;
                       if(r>0){ wins+=1.0; gp+=r; }
                       else if(r<0){ gl += -r; }
                      }
                    double win_rate       = SafeDiv(wins, (double)total);
                    double avg_R          = SafeDiv(sumR, (double)total);
                    double pf             = (gl>0.0)? gp/gl : (gp>0.0? 9999.0 : 0.0);
                    double median_R       = Median(r_all);
                    double maxdd_R        = MaxDrawdownR(r_all);

                    // Abort before writing if deadline exceeded to avoid partial files
                    if(DeadlineExceeded(t0))
                      {
                       PrintFormat("InsightsBuilder: aborting before write due to timeout/cancel (timeout_ms=%d)", m_timeout_ms);
                       return false;
                      }
                    // Prepare output
                    FolderCreate("DualEA", FILE_COMMON);
                    int out = FileOpen(m_out_path, FILE_WRITE|FILE_COMMON|FILE_TXT|FILE_ANSI);
                    if(out==INVALID_HANDLE)
                      {
                       PrintFormat("InsightsBuilder: cannot open output '%s'. Err=%d", m_out_path, GetLastError());
                       return false;
                      }
                    string now = TimeToString(TimeCurrent(), TIME_DATE|TIME_MINUTES|TIME_SECONDS);
                    FileWriteString(out, "{\n");
                    FileWriteString(out, "  \"schema_version\": \"1.0\",\n");
                    FileWriteString(out, "  \"generated_at\": \"" + now + "\",\n");
                    FileWriteString(out, "  \"totals\": {\n");
                    FileWriteString(out, "    \"trade_count\": " + IntegerToString(total) + ",\n");
                    FileWriteString(out, "    \"win_rate\": " + DoubleToString(win_rate, 6) + ",\n");
                    FileWriteString(out, "    \"avg_R\": " + DoubleToString(avg_R, 6) + ",\n");
                    FileWriteString(out, "    \"median_R\": " + DoubleToString(median_R, 6) + ",\n");
                    FileWriteString(out, "    \"profit_factor\": " + DoubleToString(pf, 6) + ",\n");
                    FileWriteString(out, "    \"expectancy\": " + DoubleToString(avg_R, 6) + ",\n");
                    FileWriteString(out, "    \"max_drawdown_R\": " + DoubleToString(maxdd_R, 6) + "\n");
                    
                    FileWriteString(out, "  },\n");

                    // by_strategy breakdown
                    FileWriteString(out, "  \"by_strategy\": [\n");
                    for(int s=0; s<ArraySize(uniq_strats); ++s)
                      {
                       string sname = uniq_strats[s];
                       // collect r for this strategy in file order
                       double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                       for(int i=0;i<total;++i)
                         if(strat_list[i]==sname)
                           {
                            int k = ArraySize(rs); ArrayResize(rs, k+1); rs[k]=r_list[i];
                            double rr = r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;}
                            cnt++;
                           }
                       double s_wr = SafeDiv(s_wins, (double)cnt);
                       double s_avg= SafeDiv(s_sum, (double)cnt);
                       double s_pf = (s_gl>0.0)? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0);
                       double s_med= Median(rs);
                       double s_dd = MaxDrawdownR(rs);
                       string comma = (s<ArraySize(uniq_strats)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"strategy\": \"%s\", \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                      sname, cnt, s_wr, s_avg, s_med, s_pf, s_avg, s_dd, comma));
                      }
                    FileWriteString(out, "  ],\n");

                    // by_timeframe breakdown (if available)
                    // collect unique timeframes present in tf_list (>=0)
                    int uniq_tf[]; for(int i=0;i<total;++i){ if(tf_list[i]>=0){ bool seen=false; for(int j=0;j<ArraySize(uniq_tf);++j){ if(uniq_tf[j]==tf_list[i]){seen=true;break;} } if(!seen){ int k=ArraySize(uniq_tf); ArrayResize(uniq_tf,k+1); uniq_tf[k]=tf_list[i]; } } }
                    FileWriteString(out, "  \"by_timeframe\": [\n");
                    for(int t=0;t<ArraySize(uniq_tf);++t)
                      {
                       int tfv = uniq_tf[t];
                       double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                       for(int i=0;i<total;++i)
                         if(tf_list[i]==tfv)
                           { int k=ArraySize(rs); ArrayResize(rs,k+1); rs[k]=r_list[i]; double rr=r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;} cnt++; }
                       double wr = (cnt>0? s_wins/cnt : 0.0);
                       double avg = (cnt>0? s_sum/cnt : 0.0);
                       double pf = (s_gl>0.0? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0));
                       double med= Median(rs);
                       double dd = MaxDrawdownR(rs);
                       string comma = (t<ArraySize(uniq_tf)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"timeframe\": %d, \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n", tfv, cnt, wr, avg, med, pf, avg, dd, comma));
                      }
                    FileWriteString(out, "  ],\n");

                    // by_symbol_strategy_timeframe breakdown
                    FileWriteString(out, "  \"by_symbol_strategy_timeframe\": [\n");
                    for(int s=0; s<ArraySize(uniq_strats); ++s)
                      {
                       string sname = uniq_strats[s];
                       for(int y=0; y<ArraySize(uniq_syms); ++y)
                         {
                          string yname = uniq_syms[y];
                          // collect unique tf for this (s,y)
                          int tf_set_sy[];
                          for(int i=0;i<total;++i) if(strat_list[i]==sname && sym_list[i]==yname && tf_list[i]>=0)
                            { bool seen=false; for(int j=0;j<ArraySize(tf_set_sy);++j){ if(tf_set_sy[j]==tf_list[i]){seen=true;break;} } if(!seen){ int k=ArraySize(tf_set_sy); ArrayResize(tf_set_sy,k+1); tf_set_sy[k]=tf_list[i]; } }
                          for(int t=0;t<ArraySize(tf_set_sy);++t)
                            {
                             int tfv = tf_set_sy[t];
                             double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                             for(int i=0;i<total;++i)
                               if(strat_list[i]==sname && sym_list[i]==yname && tf_list[i]==tfv)
                                 { int k=ArraySize(rs); ArrayResize(rs,k+1); rs[k]=r_list[i]; double rr=r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;} cnt++; }
                             double wr = (cnt>0? s_wins/cnt : 0.0);
                             double avg = (cnt>0? s_sum/cnt : 0.0);
                             double pf = (s_gl>0.0? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0));
                             double med= Median(rs);
                             double dd = MaxDrawdownR(rs);
                             // determine trailing comma: last item overall
                             bool lastBlock = (s==ArraySize(uniq_strats)-1) && (y==ArraySize(uniq_syms)-1) && (t==ArraySize(tf_set_sy)-1);
                             string comma = (lastBlock?"":",");
                             FileWriteString(out, StringFormat("    {\"strategy\": \"%s\", \"symbol\": \"%s\", \"timeframe\": %d, \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                        sname, yname, tfv, cnt, wr, avg, med, pf, avg, dd, comma));
                            }
                         }
                      }
                    FileWriteString(out, "  ],\n");

                    // by_symbol breakdown
                    FileWriteString(out, "  \"by_symbol\": [\n");
                    for(int y=0; y<ArraySize(uniq_syms); ++y)
                      {
                       string yname = uniq_syms[y];
                       double rs2[]; int cnt2=0; double s_sum2=0.0, s_wins2=0.0, s_gp2=0.0, s_gl2=0.0;
                       for(int i=0;i<total;++i)
                         if(sym_list[i]==yname)
                           {
                            int k = ArraySize(rs2); ArrayResize(rs2, k+1); rs2[k]=r_list[i];
                            double rr = r_list[i]; s_sum2+=rr; if(rr>0){s_wins2+=1.0; s_gp2+=rr;} else if(rr<0){s_gl2+=-rr;}
                            cnt2++;
                           }
                       double y_wr = SafeDiv(s_wins2, (double)cnt2);
                       double y_avg= SafeDiv(s_sum2, (double)cnt2);
                       double y_pf = (s_gl2>0.0)? s_gp2/s_gl2 : (s_gp2>0.0? 9999.0 : 0.0);
                       double y_med= Median(rs2);
                       double y_dd = MaxDrawdownR(rs2);
                       string comma2 = (y<ArraySize(uniq_syms)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"symbol\": \"%s\", \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                      yname, cnt2, y_wr, y_avg, y_med, y_pf, y_avg, y_dd, comma2));
                      }
                    FileWriteString(out, "  ]\n");

                    FileWriteString(out, "}\n");
                    FileClose(out);
                    string full = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_out_path;
                    PrintFormat("InsightsBuilder: wrote %s", full);
                    return true;
                    }
  };

//+------------------------------------------------------------------+
//| Constructor                                                      |
//+------------------------------------------------------------------+
CKnowledgeBase::CKnowledgeBase(string file_name="DualEA\\knowledge_base.csv", string delimiter=",")
  {
    m_csv_delimiter = delimiter;
    m_file_path = file_name; // Use subfolder under Common Files: DualEA\
    m_lock_handle = INVALID_HANDLE;
    // Derive a lock file path next to the KB file (e.g., DualEA\\knowledge_base.lock)
    string lp = m_file_path;
    int dot_lp = StringFind(lp, ".", 0);
    if(dot_lp > 0)
       m_lock_path = StringSubstr(lp, 0, dot_lp) + ".lock";
    else
       m_lock_path = lp + ".lock";

   // Ensure target subfolder exists in Common files
   // Common files base: TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files"
   // Create "DualEA" if missing
   FolderCreate("DualEA", FILE_COMMON);

   // Debug: show resolved common file path
   string full_main_path = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_file_path;
   PrintFormat("KnowledgeBase: writing main CSV to Common Files: %s", full_main_path);

   // --- Ensure the header exists (use direct FileOpen to avoid noisy error on first run)
   int h = FileOpen(m_file_path, FILE_READ|FILE_CSV|FILE_COMMON);
   if(h == INVALID_HANDLE)
     {
      // Create new file with header
      h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
      if(h != INVALID_HANDLE)
        {
         FileWriteString(h, "timestamp,symbol,type,entry_price,stop_loss,take_profit,close_price,profit,strategy_id\n");
         FileClose(h);
        }
     }
   else
     {
      // File exists; if empty, write header
      if(FileSize(h) == 0)
        {
         FileClose(h);
         h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
         if(h != INVALID_HANDLE)
           {
            FileWriteString(h, "timestamp,symbol,type,entry_price,stop_loss,take_profit,close_price,profit,strategy_id\n");
           }
        }
      if(h != INVALID_HANDLE)
         FileClose(h);
     }
  }
//+------------------------------------------------------------------+
//| Destructor                                                      |
//+------------------------------------------------------------------+
CKnowledgeBase::~CKnowledgeBase()
  {
  }

//+------------------------------------------------------------------+
//| Writes a trade record to the CSV file                            |
//+------------------------------------------------------------------+
bool CKnowledgeBase::WriteRecord(const TradeRecord &record)
  {
   PrintFormat("[KnowledgeBase] WriteRecord called: %s %s | entry=%.5f close=%.5f profit=%.2f | strategy=%s",
               record.symbol, TimeToString(record.timestamp),
               record.entry_price, record.close_price, record.profit,
               record.strategy_id);

   if(!AcquireLock(3000))
     {
      Print("[KnowledgeBase] WriteRecord failed: lock acquire timeout");
      return(false);
     }

   // IMPORTANT: use FILE_READ|FILE_WRITE to avoid truncation (FILE_WRITE alone clears the file)
   if(!OpenFile(FILE_READ|FILE_WRITE|FILE_CSV))
     {
      PrintFormat("[KnowledgeBase] WriteRecord failed: OpenFile failed, err=%d", GetLastError());
      ReleaseLock();
      return(false);
     }

   FileSeek(m_file_handle, 0, SEEK_END);

   string record_string = TimeToString(record.timestamp) + m_csv_delimiter +
                          record.symbol + m_csv_delimiter +
                          IntegerToString(record.type) + m_csv_delimiter +
                          DoubleToString(record.entry_price, _Digits) + m_csv_delimiter +
                          DoubleToString(record.stop_loss, _Digits) + m_csv_delimiter +
                          DoubleToString(record.take_profit, _Digits) + m_csv_delimiter +
                          DoubleToString(record.close_price, _Digits) + m_csv_delimiter +
                          DoubleToString(record.profit, 2) + m_csv_delimiter +
                          record.strategy_id;

   uint written = FileWriteString(m_file_handle, record_string + "\n");
   bool ok = (written > 0);

   CloseFile();
   ReleaseLock();

   if(ok)
      PrintFormat("[KnowledgeBase] WriteRecord SUCCESS: wrote %d bytes to %s", written, m_file_path);
   else
      PrintFormat("[KnowledgeBase] WriteRecord FAILED: FileWriteString returned %d, err=%d", written, GetLastError());

   return(ok);
  }

//+------------------------------------------------------------------+
//| Logs a trade event with basic MT5 result info                     |
//+------------------------------------------------------------------+
bool CKnowledgeBase::LogTrade(const string strategy_name, const int retcode, const ulong deal, const ulong order_id)
  {
   // Derive an events file from the main path, e.g., knowledge_base_events.csv
   string events_path = m_file_path;
   int dot = StringFind(events_path, ".", 0);
   if(dot > 0)
      events_path = StringSubstr(events_path, 0, dot) + "_events.csv";
   else
      events_path = events_path + "_events.csv";

   // Ensure folder exists in Common files
   FolderCreate("DualEA", FILE_COMMON);

   // Debug: show resolved events file path
   string full_events_path = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + events_path;
   PrintFormat("KnowledgeBase: writing events CSV to Common Files: %s", full_events_path);

   int handle = FileOpen(events_path, FILE_READ|FILE_WRITE|FILE_CSV|FILE_SHARE_WRITE|FILE_COMMON);
   if(handle == INVALID_HANDLE)
     {
      PrintFormat("Error opening knowledge base events file '%s'. Error code: %d", events_path, GetLastError());
      return(false);
     }

   // Write header if file is empty
   if(FileSize(handle) == 0)
     {
      FileWriteString(handle, "timestamp,strategy,retcode,deal,order\n");
     }

   FileSeek(handle, 0, SEEK_END);
   string line = TimeToString(TimeCurrent()) + m_csv_delimiter +
                 strategy_name + m_csv_delimiter +
                 IntegerToString(retcode) + m_csv_delimiter +
                 IntegerToString((int)deal) + m_csv_delimiter +
                 IntegerToString((int)order_id);
   FileWriteString(handle, line + "\n");
   FileClose(handle);
   return(true);
  }

//+------------------------------------------------------------------+
//| Opens the file                                                   |
//+------------------------------------------------------------------+
bool CKnowledgeBase::OpenFile(int flags=FILE_WRITE|FILE_READ|FILE_CSV)
  {
   // Always target the Common files area so results are shared across Tester, Paper, and Live
   // The path m_file_path should include the subfolder, e.g. "DualEA\\knowledge_base.csv"
   m_file_handle = INVALID_HANDLE;
   // Retry open with read/write sharing to mitigate transient locks (e.g., validators or other EAs)
   for(int attempt=0; attempt<10 && m_file_handle==INVALID_HANDLE; ++attempt)
     {
      m_file_handle = FileOpen(m_file_path, (flags | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON));
      if(m_file_handle==INVALID_HANDLE)
         Sleep(25);
     }
   if(m_file_handle == INVALID_HANDLE)
     {
      PrintFormat("Error opening knowledge base file '%s'. Error code: %d", m_file_path, GetLastError());
      return(false);
     }
   return(true);
  }

//+------------------------------------------------------------------+
//| Closes the file                                                  |
//+------------------------------------------------------------------+
void CKnowledgeBase::CloseFile()
  {
   if(m_file_handle != INVALID_HANDLE)
      FileClose(m_file_handle);
  }

//+------------------------------------------------------------------+
//| Acquire exclusive lock via a temporary lock file                  |
//+------------------------------------------------------------------+
bool CKnowledgeBase::AcquireLock(const int timeout_ms)
  {
   m_lock_handle = INVALID_HANDLE;
   int waited = 0;
   // Do not specify share flags to request exclusive access on the lock file
   while(waited < timeout_ms && m_lock_handle==INVALID_HANDLE)
     {
      m_lock_handle = FileOpen(m_lock_path, FILE_WRITE|FILE_COMMON);
      if(m_lock_handle==INVALID_HANDLE)
        {
         Sleep(25);
         waited += 25;
        }
     }
   if(m_lock_handle==INVALID_HANDLE)
     {
      PrintFormat("KnowledgeBase: lock acquire timeout for %s (err=%d)", m_lock_path, GetLastError());
      return false;
     }
   // Optional: write small token so file exists for observers
   FileWriteString(m_lock_handle, "lock\n");
   return true;
  }

//+------------------------------------------------------------------+
//| Release the lock                                                  |
//+------------------------------------------------------------------+
void CKnowledgeBase::ReleaseLock()
  {
   if(m_lock_handle!=INVALID_HANDLE)
     {
      FileClose(m_lock_handle);
      m_lock_handle = INVALID_HANDLE;
      // Best-effort delete to avoid stale locks
      FileDelete(m_lock_path, FILE_COMMON);
     }
  }