// KnowledgeBase.mqh
// Shared functions for reading from and writing to the knowledge base.

#property copyright "2025, Windsurf Engineering"
#property link      "https://www.windsurf.ai"

#include <Files\File.mqh>
// Centralized default strategy names
#include <Strategies\Registry.mqh>

// --- Defines the structure for a single trade record
struct TradeRecord
  {
   datetime          timestamp;        // Trade execution timestamp
   string            symbol;           // Trading symbol
   ENUM_ORDER_TYPE   type;             // Order type (e.g., ORDER_TYPE_BUY, ORDER_TYPE_SELL)
   double            entry_price;      // Price at which the trade was opened
   double            stop_loss;        // Stop loss level
   double            take_profit;      // Take profit level
   double            close_price;      // Price at which the trade was closed
   double            profit;           // Profit or loss from the trade
   string            strategy_id;      // ID of the strategy that generated the trade
  };
// Uncomment to enable verbose parsing diagnostics
//#define INSIGHTS_DEBUG

// --- Class to manage knowledge base operations
class CKnowledgeBase
  {
private:
   string            m_file_path;      // Path to the knowledge base file
   int               m_file_handle;    // File handle
   string            m_csv_delimiter;  // CSV delimiter
   string            m_lock_path;      // Path to lock file in Common Files
   int               m_lock_handle;    // Lock file handle

public:
                     CKnowledgeBase(string file_name="DualEA\\knowledge_base.csv", string delimiter=",");
                     ~CKnowledgeBase();

   // --- Methods for data handling
   bool              WriteRecord(const TradeRecord &record);
   bool              LogTrade(const string strategy_name, const int retcode, const ulong deal, const ulong order_id);

private:
   bool              OpenFile(int flags=FILE_WRITE|FILE_READ|FILE_CSV);
   void              CloseFile();
   bool              AcquireLock(const int timeout_ms=3000);
   void              ReleaseLock();
  };

// --- Class to manage features.csv logging (long format: one feature per row)
class CFeaturesKB
  {
 private:
    string m_file_path;
    int    m_file_handle;
    string m_csv_delim;
    void   EnsureHeader()
      {
       // Ensure folder exists and header is present
       FolderCreate("DualEA", FILE_COMMON);
       int h = FileOpen(m_file_path, FILE_READ|FILE_CSV|FILE_COMMON);
       if(h==INVALID_HANDLE)
         {
          h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
          if(h!=INVALID_HANDLE)
            {
             FileWriteString(h, "timestamp,symbol,strategy,feature,value\n");
             FileClose(h);
            }
         }
       else
         {
          if(FileSize(h)==0)
            {
             FileClose(h);
             h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
             if(h!=INVALID_HANDLE)
                FileWriteString(h, "timestamp,symbol,strategy,feature,value\n");
            }
          if(h!=INVALID_HANDLE) FileClose(h);
         }
       string full = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_file_path;
       PrintFormat("KnowledgeBase: writing features CSV to Common Files: %s", full);
      }
 public:
                  CFeaturesKB(string file_name="DualEA\\features.csv", string delimiter=",")
                    {
                     m_file_path = file_name;
                     m_csv_delim = delimiter;
                     m_file_handle = INVALID_HANDLE;
                     EnsureHeader();
                    }
                  ~CFeaturesKB() {}
    bool          WriteKV(const datetime ts, const string symbol, const string strategy, const string feature, const double value)
                    {
                     int h = FileOpen(m_file_path, FILE_READ|FILE_WRITE|FILE_CSV|FILE_SHARE_WRITE|FILE_COMMON);
                     if(h==INVALID_HANDLE)
                       {
                        PrintFormat("Error opening features file '%s'. Error: %d", m_file_path, GetLastError());
                        return false;
                       }
                     if(FileSize(h)==0)
                        FileWriteString(h, "timestamp,symbol,strategy,feature,value\n");
                     FileSeek(h, 0, SEEK_END);
                     string line = TimeToString(ts) + m_csv_delim + symbol + m_csv_delim + strategy + m_csv_delim + feature + m_csv_delim + DoubleToString(value, 8);
                     FileWriteString(h, line + "\n");
                     FileClose(h);
                     return true;
                    }
  };

// --- Insights builder: reads KB and emits insights.json (basic counts scaffold)
class CInsightsBuilder
  {
 private:
    string m_kb_path;
    string m_features_path;
    string m_out_path;
    string m_delim;
    // --- helpers
    string Trim(string s)
      {
       StringTrimLeft(s);
        StringTrimRight(s);
        return s;
      }
    string StripQuotes(string s)
      {
       s = Trim(s);
       // strip BOM if present (U+FEFF or UTF-8 BOM 0xEF 0xBB 0xBF)
       if(StringLen(s)>0)
         {
          int c0 = StringGetCharacter(s,0);
          // U+FEFF
          if(c0==65279)
             s = StringSubstr(s,1);
          else if(StringLen(s)>=3)
            {
             int b0 = c0;
             int b1 = StringGetCharacter(s,1);
             int b2 = StringGetCharacter(s,2);
             if(b0==239 && b1==187 && b2==191)
                s = StringSubstr(s,3);
            }
         }
       // remove surrounding quotes
       if(StringLen(s)>=2 && StringSubstr(s,0,1)=="\"" && StringSubstr(s,StringLen(s)-1,1)=="\"")
          s = StringSubstr(s,1,StringLen(s)-2);
       // remove stray quotes
       StringReplace(s, "\"", "");
       // remove CR/LF and NBSP
       StringReplace(s, "\r", "");
        StringReplace(s, "\n", "");
        // remove non-breaking space (char code 160)
        for(int i=0;i<StringLen(s);++i){ if(StringGetCharacter(s,i)==160){ s = StringSubstr(s,0,i)+StringSubstr(s,i+1); i--; } }
       return s;
      }
    string NormalizeToken(string s)
      {
       s = StripQuotes(s);
       // StringToLower modifies in place; do not assign its return value
       StringToLower(s);
       string out="";
       for(int i=0;i<StringLen(s);++i)
         {
          int ch = StringGetCharacter(s,i);
          bool ok = (ch>='a' && ch<='z') || (ch>='0' && ch<='9') || (ch=='_');
          if(ok)
             out = out + StringSubstr(s,i,1);
         }
       return out;
      }
    int    FindIndexString(string &arr[], const string key)
      {
       for(int i=0;i<ArraySize(arr);++i)
         if(arr[i]==key) return i;
       return -1;
      }
    void   PushUnique(string &arr[], const string key)
      {
       if(FindIndexString(arr,key)<0)
         {
          int n = ArraySize(arr);
          ArrayResize(arr, n+1);
          arr[n] = key;
         }
      }
    double SafeDiv(const double a, const double b)
      {
       if(b==0.0) return 0.0;
       return a/b;
      }
    double Median(double &values[])
      {
       int n = ArraySize(values);
       if(n==0) return 0.0;
       // Work on a copy to avoid side effects
       double tmp[];
       ArrayResize(tmp, n);
       for(int i=0;i<n;++i) tmp[i]=values[i];
       ArraySort(tmp);
       if((n % 2)==1)
          return tmp[n/2];
       return 0.5*(tmp[n/2-1] + tmp[n/2]);
      }
    double MaxDrawdownR(double &r_series[])
      {
       // Computes max peak-to-trough drawdown on cumulative R
       int n = ArraySize(r_series);
       if(n==0) return 0.0;
       double cum=0.0, peak=0.0, maxdd=0.0;
       for(int i=0;i<n;++i)
         {
          cum += r_series[i];
          if(cum>peak) peak=cum;
          double dd = peak - cum; // in R
          if(dd>maxdd) maxdd=dd;
         }
       return maxdd;
      }
  public:
                  CInsightsBuilder(string kb_path="DualEA\\knowledge_base.csv", string features_path="DualEA\\features.csv", string out_path="DualEA\\insights.json", string delim=",")
                    {
                     m_kb_path = kb_path;
                     m_features_path = features_path;
                     m_out_path = out_path;
                     m_delim = delim;
                    }
    bool          Build()
                    {
                    // Build insights from features.csv using r_multiple rows
                    // Open features.csv with explicit comma delimiter
                    int hf = FileOpen(m_features_path, FILE_READ|FILE_CSV|FILE_SHARE_READ|FILE_COMMON, (ushort)',' );
                    if(hf==INVALID_HANDLE)
                      {
                       PrintFormat("InsightsBuilder: cannot open features '%s'. Err=%d", m_features_path, GetLastError());
                       return false;
                      }

                    // Storage of r-multiples with metadata (plus optional timeframe)
                    datetime ts_list[]; string sym_list[]; string strat_list[]; double r_list[]; int tf_list[];
                    // Map (symbol|strategy) -> timeframe (last seen)
                    string tf_keys[]; int tf_vals[];
                    // Collect all observed (symbol|strategy) pairs to support synthesis when timeframe is absent
                    string all_keys[];

                    // Robust CSV scan: 5 fields per row, skip header if present
                    bool header_checked=false;
                    int parsed_rows=0;
                    while(!FileIsEnding(hf))
                      {
                       string ts = FileReadString(hf);
                       if(ts=="" && FileIsEnding(hf)) break;
                       string sym = FileReadString(hf);
                       string strat = FileReadString(hf);
                       string feat = FileReadString(hf);
                       string val  = FileReadString(hf);

                       // Trim all fields to avoid CR/whitespace issues
                       ts    = StripQuotes(ts);
                       sym   = StripQuotes(sym);
                       strat = StripQuotes(strat);
                       feat  = StripQuotes(feat);
                       val   = StripQuotes(val);

                       if(feat=="") { continue; }
                       if(!header_checked)
                         {
                          header_checked=true;
                          string ts_norm = NormalizeToken(ts);
                          string feat_norm_dbg = NormalizeToken(feat);
                          int c0 = (StringLen(ts)>0? StringGetCharacter(ts,0) : -1);
                          #ifdef INSIGHTS_DEBUG
                          PrintFormat("InsightsBuilder: header-check ts_raw='%s' ts_norm='%s' c0=%d feat_raw='%s' feat_norm='%s'", ts, ts_norm, c0, feat, feat_norm_dbg);
                          #endif
                          if(ts_norm=="timestamp")
                              continue; // skip header row
                         }

                       // Count this parsed data row (post header)
                       parsed_rows++;
                       if(parsed_rows<=5)
                         {
                          string feat_norm = NormalizeToken(feat);
                          #ifdef INSIGHTS_DEBUG
                          PrintFormat("InsightsBuilder: sample row %d feat='%s' feat_norm='%s' val='%s'", parsed_rows-1, feat, feat_norm, val);
                          #endif
                         }

                       // Track (symbol|strategy) pairs from any feature rows
                       if(sym!="" && strat!="")
                         {
                          string key_any = sym + "|" + strat;
                          PushUnique(all_keys, key_any);
                         }

                       string feat_norm_main = NormalizeToken(feat);
                       if(feat_norm_main=="timeframe")
                         {
                          // record latest timeframe for (symbol,strategy)
                          int tfv = (int)StringToInteger(val);
                          string key = sym+"|"+strat;
                          bool updated=false;
                          for(int i=0;i<ArraySize(tf_keys);++i){ if(tf_keys[i]==key){ tf_vals[i]=tfv; updated=true; break; } }
                          if(!updated){ int k=ArraySize(tf_keys); ArrayResize(tf_keys,k+1); ArrayResize(tf_vals,k+1); tf_keys[k]=key; tf_vals[k]=tfv; }
                         }
                       else if(feat_norm_main=="r_multiple")
                         {
                          int n = ArraySize(r_list);
                          ArrayResize(ts_list, n+1);
                          ArrayResize(sym_list, n+1);
                          ArrayResize(strat_list, n+1);
                          ArrayResize(r_list, n+1);
                          ArrayResize(tf_list, n+1);
                          ts_list[n]    = StringToTime(ts);
                          sym_list[n]   = sym;
                          strat_list[n] = strat;
                          r_list[n]     = StringToDouble(val);
                          // lookup timeframe if available
                          int tfv=-1; string key2 = sym_list[n]+"|"+strat_list[n];
                          for(int j=0;j<ArraySize(tf_keys);++j){ if(tf_keys[j]==key2){ tfv=tf_vals[j]; break; } }
                          tf_list[n]    = tfv;
                          #ifdef INSIGHTS_DEBUG
                          if(n<5)
                            PrintFormat("InsightsBuilder: captured r_multiple row ts=%s sym=%s strat=%s r=%f", ts, sym, strat, r_list[n]);
                          #endif
                         }
                      }
                    FileClose(hf);

                    int total = ArraySize(r_list);
                    PrintFormat("InsightsBuilder: scanned rows=%d, r_multiple rows=%d from features '%s'", parsed_rows, total, m_features_path);
                    // --- Fallback: if no r_multiple labels were found, derive metrics from knowledge_base.csv using profit as surrogate
                    if(total==0)
                      {
                       // Secondary fallback: parse features.csv as raw text lines and split by comma
                       int ht = FileOpen(m_features_path, FILE_READ|FILE_COMMON|FILE_ANSI);
                       if(ht!=INVALID_HANDLE)
                         {
                          int added=0; bool header_seen=false;
                          while(!FileIsEnding(ht))
                            {
                             string line = FileReadString(ht);
                              line = Trim(line);
                              if(line=="") continue;
                              string parts[];
                              ushort sep = (ushort)StringGetCharacter(",",0);
                              int cnt = StringSplit(line, sep, parts);
                              if(cnt<5) continue;
                              // parts: 0 ts,1 sym,2 strat,3 feat,4 val
                              string ts_s = StripQuotes(parts[0]);
                             if(!header_seen)
                               {
                                header_seen=true;
                                if(NormalizeToken(ts_s)=="timestamp") continue; // skip header
                               }
                             // Track (symbol|strategy) pairs from any feature rows (text fallback)
                             string sy0 = StripQuotes(parts[1]);
                             string st0 = StripQuotes(parts[2]);
                             if(sy0!="" && st0!="") PushUnique(all_keys, sy0+"|"+st0);
                             string f_s  = StripQuotes(parts[3]);
                             string f_norm = NormalizeToken(f_s);
                             if(f_norm=="timeframe")
                               {
                                int tfv = (int)StringToInteger(StripQuotes(parts[4]));
                                string sy = StripQuotes(parts[1]); string st = StripQuotes(parts[2]);
                                string key = sy+"|"+st; bool updated=false;
                                for(int i2=0;i2<ArraySize(tf_keys);++i2){ if(tf_keys[i2]==key){ tf_vals[i2]=tfv; updated=true; break; } }
                                if(!updated){ int k2=ArraySize(tf_keys); ArrayResize(tf_keys,k2+1); ArrayResize(tf_vals,k2+1); tf_keys[k2]=key; tf_vals[k2]=tfv; }
                               }
                              else if(f_norm=="r_multiple")
                                {
                                 int n = ArraySize(r_list);
                                 ArrayResize(ts_list, n+1);
                                 ArrayResize(sym_list, n+1);
                                 ArrayResize(strat_list, n+1);
                                 ArrayResize(r_list, n+1);
                                 ArrayResize(tf_list, n+1);
                                 ts_list[n]    = StringToTime(ts_s);
                                 sym_list[n]   = StripQuotes(parts[1]);
                                 strat_list[n] = StripQuotes(parts[2]);
                                 r_list[n]     = StringToDouble(StripQuotes(parts[4]));
                                 // lookup timeframe if available
                                 int tfv=-1; string key2 = sym_list[n]+"|"+strat_list[n];
                                 for(int j2=0;j2<ArraySize(tf_keys);++j2){ if(tf_keys[j2]==key2){ tfv=tf_vals[j2]; break; } }
                                 tf_list[n]    = tfv;
                                 added++;
                                }
                            }
                          FileClose(ht);
                          if(added>0)
                            {
                             total = ArraySize(r_list);
                             PrintFormat("InsightsBuilder: text-fallback collected %d r_multiple rows from features.csv", added);
                            }
                         }
                       
                       Print("InsightsBuilder: no r_multiple rows found in features.csv; falling back to knowledge_base.csv using profit as surrogate");
                       // Open knowledge_base.csv with explicit comma delimiter
                       int hk = FileOpen(m_kb_path, FILE_READ|FILE_CSV|FILE_SHARE_READ|FILE_COMMON, (ushort)',' );
                       if(hk!=INVALID_HANDLE)
                         {
                          // No header expected in knowledge_base.csv
                          // Columns: ts,symbol,type,entry,sl,tp,close,profit,strategy
                          string sym_list2[]; string strat_list2[]; double r_list2[]; // use profit as R substitute
                          while(!FileIsEnding(hk))
                            {
                             string ts = FileReadString(hk); if(ts=="" && FileIsEnding(hk)) break;
                             string sym = FileReadString(hk);
                             string type= FileReadString(hk);
                             string entry=FileReadString(hk);
                             string sl  = FileReadString(hk);
                             string tp  = FileReadString(hk);
                             string close=FileReadString(hk);
                             string profit=FileReadString(hk);
                             string strat = FileReadString(hk);

                             // Trim fields
                             ts=Trim(ts); sym=Trim(sym); type=Trim(type); entry=Trim(entry); sl=Trim(sl); tp=Trim(tp); close=Trim(close); profit=Trim(profit); strat=Trim(strat);
                             if(profit=="") continue;
                             // consider only rows that look like closures (close price present OR profit non-zero)
                             double pr = StringToDouble(profit);
                             double cp = StringToDouble(close);
                             if(cp>0.0 || pr!=0.0)
                               {
                                int n = ArraySize(r_list2);
                                ArrayResize(r_list2, n+1);
                                ArrayResize(sym_list2, n+1);
                                ArrayResize(strat_list2, n+1);
                                r_list2[n] = pr;
                                sym_list2[n] = sym;
                                strat_list2[n] = strat;
                               }
                            }
                          FileClose(hk);
                          // replace primary arrays for aggregation below
                          ArraySwap(r_list, r_list2);
                          ArraySwap(strat_list, strat_list2);
                          ArraySwap(sym_list, sym_list2);
                          total = ArraySize(r_list);
                          PrintFormat("InsightsBuilder: fallback collected %d rows from knowledge_base.csv", total);
                          // Initialize tf_list for KB-fallback rows using timeframe map from features
                          ArrayResize(tf_list, total);
                          for(int i3=0;i3<total;++i3)
                            {
                             tf_list[i3] = -1;
                             string key3 = sym_list[i3] + "|" + strat_list[i3];
                             for(int j3=0;j3<ArraySize(tf_keys);++j3)
                               {
                                if(tf_keys[j3]==key3){ tf_list[i3]=tf_vals[j3]; break; }
                               }
                             }
                          // If still no rows after KB fallback, synthesize dummy rows from timeframe features
                          if(total==0)
                            {
                             int combos = ArraySize(tf_keys);
                             for(int c=0;c<combos;++c)
                               {
                                int sep = StringFind(tf_keys[c], "|", 0);
                                if(sep<0) continue;
                                int n2 = ArraySize(r_list);
                                ArrayResize(r_list, n2+1);
                                ArrayResize(strat_list, n2+1);
                                ArrayResize(sym_list, n2+1);
                                ArrayResize(tf_list, n2+1);
                                sym_list[n2]   = StringSubstr(tf_keys[c], 0, sep);
                                strat_list[n2] = StringSubstr(tf_keys[c], sep+1);
                                r_list[n2]     = 0.0; // scaffold count only
                                tf_list[n2]    = tf_vals[c];
                               }
                             total = ArraySize(r_list);
                             if(total>0)
                                PrintFormat("InsightsBuilder: synthesized %d dummy rows from timeframe features (no r_multiple/profit found)", total);
                            }
                          }
                         }
                    // Final synthesis: if still no rows and no timeframe entries, synthesize from symbol/strategy pairs using default chart timeframe
                     if(total==0 && ArraySize(all_keys)>0)
                      {
                       int tf_def = (int)Period();
                       for(int c=0; c<ArraySize(all_keys); ++c)
                         {
                          int sep = StringFind(all_keys[c], "|", 0);
                          if(sep<0) continue;
                          int n2 = ArraySize(r_list);
                          ArrayResize(r_list, n2+1);
                          ArrayResize(strat_list, n2+1);
                          ArrayResize(sym_list, n2+1);
                          ArrayResize(tf_list, n2+1);
                          sym_list[n2]   = StringSubstr(all_keys[c], 0, sep);
                          strat_list[n2] = StringSubstr(all_keys[c], sep+1);
                          r_list[n2]     = 0.0; // scaffold count only
                          tf_list[n2]    = tf_def;
                         }
                       total = ArraySize(r_list);
                       if(total>0)
                          PrintFormat("InsightsBuilder: synthesized %d dummy rows from symbol/strategy pairs (default timeframe=%d)", total, tf_def);
                      }
                    // Ensure general coverage on current chart symbol across multiple timeframes for all default strategies
                    {
                     string def_strats[]; GetDefaultStrategyNames(def_strats);
                     string csym = Symbol();
                     int ctf = (int)Period();
                     // Small TF set including current
                     int tf_opts[]; ArrayResize(tf_opts, 5);
                     tf_opts[0]=ctf; tf_opts[1]=PERIOD_M5; tf_opts[2]=PERIOD_M10; tf_opts[3]=PERIOD_M15; tf_opts[4]=PERIOD_M30;
                     int added_before = ArraySize(r_list);
                     for(int t=0; t<ArraySize(tf_opts); ++t)
                       {
                        int tfc = tf_opts[t];
                        for(int ds=0; ds<ArraySize(def_strats); ++ds)
                          {
                           bool exists=false;
                           for(int i=0;i<total;++i)
                             {
                              if(sym_list[i]==csym && strat_list[i]==def_strats[ds] && tf_list[i]==tfc)
                                { exists=true; break; }
                             }
                           if(!exists)
                             {
                              int n3 = ArraySize(r_list);
                              ArrayResize(r_list, n3+1);
                              ArrayResize(strat_list, n3+1);
                              ArrayResize(sym_list, n3+1);
                              ArrayResize(tf_list, n3+1);
                              sym_list[n3]   = csym;
                              strat_list[n3] = def_strats[ds];
                              r_list[n3]     = 0.0;
                              tf_list[n3]    = tfc;
                             }
                          }
                       }
                     int added_after = ArraySize(r_list);
                     if(added_after>added_before)
                        PrintFormat("InsightsBuilder: ensured multi-TF chart coverage by adding %d rows for symbol=%s across %d TFs and %d strategies", added_after-added_before, csym, ArraySize(tf_opts), ArraySize(def_strats));
                    }
                    // Unique strategies and symbols
                    string uniq_strats[]; string uniq_syms[];
                    for(int i=0;i<total;++i){ PushUnique(uniq_strats, strat_list[i]); PushUnique(uniq_syms, sym_list[i]); }

                    // Overall aggregates
                    double wins=0.0, sumR=0.0, gp=0.0, gl=0.0;
                    double r_all[];
                    ArrayResize(r_all, total);
                    for(int i=0;i<total;++i)
                      {
                       double r = r_list[i];
                       r_all[i]=r;
                       sumR += r;
                       if(r>0){ wins+=1.0; gp+=r; }
                       else if(r<0){ gl += -r; }
                      }
                    double win_rate       = SafeDiv(wins, (double)total);
                    double avg_R          = SafeDiv(sumR, (double)total);
                    double pf             = (gl>0.0)? gp/gl : (gp>0.0? 9999.0 : 0.0);
                    double median_R       = Median(r_all);
                    double maxdd_R        = MaxDrawdownR(r_all);

                    // Prepare output
                    FolderCreate("DualEA", FILE_COMMON);
                    int out = FileOpen(m_out_path, FILE_WRITE|FILE_COMMON|FILE_TXT|FILE_ANSI);
                    if(out==INVALID_HANDLE)
                      {
                       PrintFormat("InsightsBuilder: cannot open output '%s'. Err=%d", m_out_path, GetLastError());
                       return false;
                      }
                    string now = TimeToString(TimeCurrent(), TIME_DATE|TIME_MINUTES|TIME_SECONDS);
                    FileWriteString(out, "{\n");
                    FileWriteString(out, "  \"schema_version\": \"1.0\",\n");
                    FileWriteString(out, "  \"generated_at\": \"" + now + "\",\n");
                    FileWriteString(out, "  \"totals\": {\n");
                    FileWriteString(out, "    \"trade_count\": " + IntegerToString(total) + ",\n");
                    FileWriteString(out, "    \"win_rate\": " + DoubleToString(win_rate, 6) + ",\n");
                    FileWriteString(out, "    \"avg_R\": " + DoubleToString(avg_R, 6) + ",\n");
                    FileWriteString(out, "    \"median_R\": " + DoubleToString(median_R, 6) + ",\n");
                    FileWriteString(out, "    \"profit_factor\": " + DoubleToString(pf, 6) + ",\n");
                    FileWriteString(out, "    \"expectancy\": " + DoubleToString(avg_R, 6) + ",\n");
                    FileWriteString(out, "    \"max_drawdown_R\": " + DoubleToString(maxdd_R, 6) + "\n");
                    
                    FileWriteString(out, "  },\n");

                    // by_strategy breakdown
                    FileWriteString(out, "  \"by_strategy\": [\n");
                    for(int s=0; s<ArraySize(uniq_strats); ++s)
                      {
                       string sname = uniq_strats[s];
                       // collect r for this strategy in file order
                       double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                       for(int i=0;i<total;++i)
                         if(strat_list[i]==sname)
                           {
                            int k = ArraySize(rs); ArrayResize(rs, k+1); rs[k]=r_list[i];
                            double rr = r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;}
                            cnt++;
                           }
                       double s_wr = SafeDiv(s_wins, (double)cnt);
                       double s_avg= SafeDiv(s_sum, (double)cnt);
                       double s_pf = (s_gl>0.0)? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0);
                       double s_med= Median(rs);
                       double s_dd = MaxDrawdownR(rs);
                       string comma = (s<ArraySize(uniq_strats)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"strategy\": \"%s\", \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                      sname, cnt, s_wr, s_avg, s_med, s_pf, s_avg, s_dd, comma));
                      }
                    FileWriteString(out, "  ],\n");

                    // by_timeframe breakdown (if available)
                    // collect unique timeframes present in tf_list (>=0)
                    int uniq_tf[]; for(int i=0;i<total;++i){ if(tf_list[i]>=0){ bool seen=false; for(int j=0;j<ArraySize(uniq_tf);++j){ if(uniq_tf[j]==tf_list[i]){seen=true;break;} } if(!seen){ int k=ArraySize(uniq_tf); ArrayResize(uniq_tf,k+1); uniq_tf[k]=tf_list[i]; } } }
                    FileWriteString(out, "  \"by_timeframe\": [\n");
                    for(int t=0;t<ArraySize(uniq_tf);++t)
                      {
                       int tfv = uniq_tf[t];
                       double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                       for(int i=0;i<total;++i)
                         if(tf_list[i]==tfv)
                           { int k=ArraySize(rs); ArrayResize(rs,k+1); rs[k]=r_list[i]; double rr=r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;} cnt++; }
                       double wr = (cnt>0? s_wins/cnt : 0.0);
                       double avg = (cnt>0? s_sum/cnt : 0.0);
                       double pf = (s_gl>0.0? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0));
                       double med= Median(rs);
                       double dd = MaxDrawdownR(rs);
                       string comma = (t<ArraySize(uniq_tf)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"timeframe\": %d, \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n", tfv, cnt, wr, avg, med, pf, avg, dd, comma));
                      }
                    FileWriteString(out, "  ],\n");

                    // by_symbol_strategy_timeframe breakdown
                    FileWriteString(out, "  \"by_symbol_strategy_timeframe\": [\n");
                    for(int s=0; s<ArraySize(uniq_strats); ++s)
                      {
                       string sname = uniq_strats[s];
                       for(int y=0; y<ArraySize(uniq_syms); ++y)
                         {
                          string yname = uniq_syms[y];
                          // collect unique tf for this (s,y)
                          int tf_set_sy[];
                          for(int i=0;i<total;++i) if(strat_list[i]==sname && sym_list[i]==yname && tf_list[i]>=0)
                            { bool seen=false; for(int j=0;j<ArraySize(tf_set_sy);++j){ if(tf_set_sy[j]==tf_list[i]){seen=true;break;} } if(!seen){ int k=ArraySize(tf_set_sy); ArrayResize(tf_set_sy,k+1); tf_set_sy[k]=tf_list[i]; } }
                          for(int t=0;t<ArraySize(tf_set_sy);++t)
                            {
                             int tfv = tf_set_sy[t];
                             double rs[]; int cnt=0; double s_sum=0.0, s_wins=0.0, s_gp=0.0, s_gl=0.0;
                             for(int i=0;i<total;++i)
                               if(strat_list[i]==sname && sym_list[i]==yname && tf_list[i]==tfv)
                                 { int k=ArraySize(rs); ArrayResize(rs,k+1); rs[k]=r_list[i]; double rr=r_list[i]; s_sum+=rr; if(rr>0){s_wins+=1.0; s_gp+=rr;} else if(rr<0){s_gl+=-rr;} cnt++; }
                             double wr = (cnt>0? s_wins/cnt : 0.0);
                             double avg = (cnt>0? s_sum/cnt : 0.0);
                             double pf = (s_gl>0.0? s_gp/s_gl : (s_gp>0.0? 9999.0 : 0.0));
                             double med= Median(rs);
                             double dd = MaxDrawdownR(rs);
                             // determine trailing comma: last item overall
                             bool lastBlock = (s==ArraySize(uniq_strats)-1) && (y==ArraySize(uniq_syms)-1) && (t==ArraySize(tf_set_sy)-1);
                             string comma = (lastBlock?"":",");
                             FileWriteString(out, StringFormat("    {\"strategy\": \"%s\", \"symbol\": \"%s\", \"timeframe\": %d, \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                        sname, yname, tfv, cnt, wr, avg, med, pf, avg, dd, comma));
                            }
                         }
                      }
                    FileWriteString(out, "  ],\n");

                    // by_symbol breakdown
                    FileWriteString(out, "  \"by_symbol\": [\n");
                    for(int y=0; y<ArraySize(uniq_syms); ++y)
                      {
                       string yname = uniq_syms[y];
                       double rs2[]; int cnt2=0; double s_sum2=0.0, s_wins2=0.0, s_gp2=0.0, s_gl2=0.0;
                       for(int i=0;i<total;++i)
                         if(sym_list[i]==yname)
                           {
                            int k = ArraySize(rs2); ArrayResize(rs2, k+1); rs2[k]=r_list[i];
                            double rr = r_list[i]; s_sum2+=rr; if(rr>0){s_wins2+=1.0; s_gp2+=rr;} else if(rr<0){s_gl2+=-rr;}
                            cnt2++;
                           }
                       double y_wr = SafeDiv(s_wins2, (double)cnt2);
                       double y_avg= SafeDiv(s_sum2, (double)cnt2);
                       double y_pf = (s_gl2>0.0)? s_gp2/s_gl2 : (s_gp2>0.0? 9999.0 : 0.0);
                       double y_med= Median(rs2);
                       double y_dd = MaxDrawdownR(rs2);
                       string comma2 = (y<ArraySize(uniq_syms)-1)?",":"";
                       FileWriteString(out, StringFormat("    {\"symbol\": \"%s\", \"trade_count\": %d, \"win_rate\": %.6f, \"avg_R\": %.6f, \"median_R\": %.6f, \"profit_factor\": %.6f, \"expectancy\": %.6f, \"max_drawdown_R\": %.6f}%s\n",
                                                      yname, cnt2, y_wr, y_avg, y_med, y_pf, y_avg, y_dd, comma2));
                      }
                    FileWriteString(out, "  ]\n");

                    FileWriteString(out, "}\n");
                    FileClose(out);
                    string full = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_out_path;
                    PrintFormat("InsightsBuilder: wrote %s", full);
                    return true;
                    }
  };

//+------------------------------------------------------------------+
//| Constructor                                                      |
//+------------------------------------------------------------------+
CKnowledgeBase::CKnowledgeBase(string file_name="DualEA\\knowledge_base.csv", string delimiter=",")
  {
   m_csv_delimiter = delimiter;
   m_file_path = file_name; // Use subfolder under Common Files: DualEA\
   m_lock_handle = INVALID_HANDLE;
   // Derive a lock file path next to the KB file (e.g., DualEA\\knowledge_base.lock)
   string lp = m_file_path;
   int dot_lp = StringFind(lp, ".", 0);
   if(dot_lp > 0)
      m_lock_path = StringSubstr(lp, 0, dot_lp) + ".lock";
   else
      m_lock_path = lp + ".lock";

   // Ensure target subfolder exists in Common files
   // Common files base: TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files"
   // Create "DualEA" if missing
   FolderCreate("DualEA", FILE_COMMON);

   // Debug: show resolved common file path
   string full_main_path = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + m_file_path;
   PrintFormat("KnowledgeBase: writing main CSV to Common Files: %s", full_main_path);

   // --- Ensure the header exists (use direct FileOpen to avoid noisy error on first run)
   int h = FileOpen(m_file_path, FILE_READ|FILE_CSV|FILE_COMMON);
   if(h == INVALID_HANDLE)
     {
      // Create new file with header
      h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
      if(h != INVALID_HANDLE)
        {
         FileWriteString(h, "timestamp,symbol,type,entry_price,stop_loss,take_profit,close_price,profit,strategy_id\n");
         FileClose(h);
        }
     }
   else
     {
      // File exists; if empty, write header
      if(FileSize(h) == 0)
        {
         FileClose(h);
         h = FileOpen(m_file_path, FILE_WRITE|FILE_CSV|FILE_COMMON);
         if(h != INVALID_HANDLE)
           {
            FileWriteString(h, "timestamp,symbol,type,entry_price,stop_loss,take_profit,close_price,profit,strategy_id\n");
           }
        }
      if(h != INVALID_HANDLE)
         FileClose(h);
     }
  }
//+------------------------------------------------------------------+
//| Destructor                                                      |
//+------------------------------------------------------------------+
CKnowledgeBase::~CKnowledgeBase()
  {
  }

//+------------------------------------------------------------------+
//| Writes a trade record to the CSV file                            |
//+------------------------------------------------------------------+
bool CKnowledgeBase::WriteRecord(const TradeRecord &record)
  {
   // Acquire cross-terminal lock to serialize writers
   if(!AcquireLock(3000))
      return(false);

   // Use READ|WRITE to avoid truncation; FILE_WRITE alone can clear the file
   if(!OpenFile(FILE_READ|FILE_WRITE|FILE_CSV))
     {
      ReleaseLock();
      return(false);
     }

   FileSeek(m_file_handle, 0, SEEK_END);

   string record_string = TimeToString(record.timestamp) + m_csv_delimiter +
                          record.symbol + m_csv_delimiter +
                          IntegerToString(record.type) + m_csv_delimiter +
                          DoubleToString(record.entry_price, _Digits) + m_csv_delimiter +
                          DoubleToString(record.stop_loss, _Digits) + m_csv_delimiter +
                          DoubleToString(record.take_profit, _Digits) + m_csv_delimiter +
                          DoubleToString(record.close_price, _Digits) + m_csv_delimiter +
                          DoubleToString(record.profit, 2) + m_csv_delimiter +
                          record.strategy_id;

   FileWriteString(m_file_handle, record_string + "\n");

   CloseFile();
   ReleaseLock();
   return(true);
  }

//+------------------------------------------------------------------+
//| Logs a trade event with basic MT5 result info                     |
//+------------------------------------------------------------------+
bool CKnowledgeBase::LogTrade(const string strategy_name, const int retcode, const ulong deal, const ulong order_id)
  {
   // Derive an events file from the main path, e.g., knowledge_base_events.csv
   string events_path = m_file_path;
   int dot = StringFind(events_path, ".", 0);
   if(dot > 0)
      events_path = StringSubstr(events_path, 0, dot) + "_events.csv";
   else
      events_path = events_path + "_events.csv";

   // Ensure folder exists in Common files
   FolderCreate("DualEA", FILE_COMMON);

   // Debug: show resolved events file path
   string full_events_path = TerminalInfoString(TERMINAL_COMMONDATA_PATH) + "\\Files\\" + events_path;
   PrintFormat("KnowledgeBase: writing events CSV to Common Files: %s", full_events_path);

   int handle = FileOpen(events_path, FILE_READ|FILE_WRITE|FILE_CSV|FILE_SHARE_WRITE|FILE_COMMON);
   if(handle == INVALID_HANDLE)
     {
      PrintFormat("Error opening knowledge base events file '%s'. Error code: %d", events_path, GetLastError());
      return(false);
     }

   // Write header if file is empty
   if(FileSize(handle) == 0)
     {
      FileWriteString(handle, "timestamp,strategy,retcode,deal,order\n");
     }

   FileSeek(handle, 0, SEEK_END);
   string line = TimeToString(TimeCurrent()) + m_csv_delimiter +
                 strategy_name + m_csv_delimiter +
                 IntegerToString(retcode) + m_csv_delimiter +
                 IntegerToString((int)deal) + m_csv_delimiter +
                 IntegerToString((int)order_id);
   FileWriteString(handle, line + "\n");
   FileClose(handle);
   return(true);
  }

//+------------------------------------------------------------------+
//| Opens the file                                                   |
//+------------------------------------------------------------------+
bool CKnowledgeBase::OpenFile(int flags=FILE_WRITE|FILE_READ|FILE_CSV)
  {
   // Always target the Common files area so results are shared across Tester, Paper, and Live
   // The path m_file_path should include the subfolder, e.g. "DualEA\\knowledge_base.csv"
   m_file_handle = INVALID_HANDLE;
   // Retry with read/write sharing to mitigate transient locks
   for(int attempt=0; attempt<10 && m_file_handle==INVALID_HANDLE; ++attempt)
     {
      m_file_handle = FileOpen(m_file_path, (flags | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON));
      if(m_file_handle==INVALID_HANDLE)
         Sleep(25);
     }
   if(m_file_handle == INVALID_HANDLE)
     {
      PrintFormat("Error opening knowledge base file '%s'. Error code: %d", m_file_path, GetLastError());
      return(false);
     }
   return(true);
  }

//+------------------------------------------------------------------+
//| Closes the file                                                  |
//+------------------------------------------------------------------+
void CKnowledgeBase::CloseFile()
  {
   if(m_file_handle != INVALID_HANDLE)
      FileClose(m_file_handle);
  }

//+------------------------------------------------------------------+
//| Acquire exclusive lock via a temporary lock file                  |
//+------------------------------------------------------------------+
bool CKnowledgeBase::AcquireLock(const int timeout_ms)
  {
   m_lock_handle = INVALID_HANDLE;
   int waited = 0;
   // Do not specify share flags to request exclusive access on the lock file
   while(waited < timeout_ms && m_lock_handle==INVALID_HANDLE)
     {
      m_lock_handle = FileOpen(m_lock_path, FILE_WRITE|FILE_COMMON);
      if(m_lock_handle==INVALID_HANDLE)
        {
         Sleep(25);
         waited += 25;
        }
     }
   if(m_lock_handle==INVALID_HANDLE)
     {
      PrintFormat("KnowledgeBase: lock acquire timeout for %s (err=%d)", m_lock_path, GetLastError());
      return false;
     }
   // Optional: write small token so file exists for observers
   FileWriteString(m_lock_handle, "lock\n");
   return true;
  }

//+------------------------------------------------------------------+
//| Release the lock                                                  |
//+------------------------------------------------------------------+
void CKnowledgeBase::ReleaseLock()
  {
   if(m_lock_handle!=INVALID_HANDLE)
     {
      FileClose(m_lock_handle);
      m_lock_handle = INVALID_HANDLE;
      // Best-effort delete to avoid stale locks
      FileDelete(m_lock_path, FILE_COMMON);
     }
  }