devKit/corrStudy.mqh

//+------------------------------------------------------------------+
//|                                                   corrStudy.mqh  |
//|                           Correlation Study Framework (devKit)   |
//|                                                                  |
//|  PURPOSE                                                         |
//|  -------                                                         |
//|  Empirically study HOW and WHETHER a set of parameters           |
//|  (of mixed types) correlates with one fixed bool outcome.        |
//|                                                                  |
//|  Use-cases:                                                       |
//|    • What makes a candle's range (H-L) large?                   |
//|    • What triggers aggressive consecutive bullish/bearish runs?  |
//|    • Does LastCSID presence / roomToLeft / hour-of-day matter?  |
//|                                                                  |
//|  STATISTICS PRODUCED                                             |
//|  -------------------                                             |
//|  Bool  params  → Phi coefficient (φ, -1..+1)                   |
//|                   Lift = P(out|param=T) / P(out)                 |
//|                   Conditional rate table                         |
//|  Numeric params → Point-biserial r (-1..+1)                     |
//|                   Mean(param | out=T) vs Mean(param | out=F)     |
//|                   Ratio of means                                 |
//|                                                                  |
//|  QUICK-START                                                     |
//|  -----------                                                      |
//|  // 1. Define your outcome function (zero args, returns bool)    |
//|  bool IsBigRange() { return (crt / atr) >= 1.5; }               |
//|                                                                  |
//|  // 2. Instantiate                                               |
//|  CCorrelationStudy study("BigRange", IsBigRange);               |
//|  study.StartExport("BigRange_data.csv");                         |
//|                                                                  |
//|  // 3. In OnCalculate / OnTick, set params then observe          |
//|  study.SetBool  ("LastCSID",   hasCSID);                        |
//|  study.SetDouble("PrevRange",  prev_h - prev_l);                |
//|  study.SetDouble("RoomToLeft", roomToLeft);                      |
//|  study.SetHour  ("Hour",       TimeCurrent());                   |
//|  study.Observe();   // records one observation                   |
//|                                                                  |
//|  // 4. Print report at any time (or on Deinit)                   |
//|  study.Report();                                                 |
//+------------------------------------------------------------------+
#pragma once
#include <Math\Stat\Math.mqh>   // MathSqrt, MathAbs — guaranteed available

//===================================================================
//  Function pointer type — the outcome to study
//  Zero arguments: the developer computes context outside.
//  Returns true  when the outcome of interest occurred.
//===================================================================
typedef bool (*OutcomeFunc)();

//===================================================================
//  Internal statistics structures
//===================================================================

// ---- Bool parameter -----------------------------------------------
struct BoolParamStat
  {
   string            name;
   bool              current_val;   // value set this observation

   // 2×2 contingency table
   // n[param_val][outcome_val]
   int               n[2][2];       // n[0/1][0/1]  — total 4 cells
   int               total;         // total observations touching this param
  };

// ---- Numeric parameter (double, int, hour) ------------------------
// Stores running Welford-style accumulators for online mean/variance.
struct NumParamStat
  {
   string            name;
   double            current_val;   // value set this observation

   // Global accumulators (all observations)
   double            sum_all;
   double            sum_sq_all;
   int               count_all;

   // Split by outcome
   double            sum_true;      // sum of param values when outcome=true
   double            sum_true_sq;
   int               count_true;

   double            sum_false;
   double            sum_false_sq;
   int               count_false;
  };

//===================================================================
//  CCorrelationStudy
//===================================================================
class CCorrelationStudy
  {
private:
   string            m_name;          // study label
   OutcomeFunc       m_outcome;       // the outcome function
   int               m_total;         // total observations recorded
   int               m_outcome_true;  // times outcome was true

   // Parameter slots
   BoolParamStat     m_bools[];
   int               m_bool_count;

   NumParamStat      m_nums[];
   int               m_num_count;

   // CSV export state
   int               m_file;          // file handle (-1 = not open)
   bool              m_exporting;

   //--- Find a bool slot by name, return -1 if absent
   int               FindBool(const string name) const
     {
      for(int i = 0; i < m_bool_count; i++)
         if(m_bools[i].name == name) return i;
      return -1;
     }

   //--- Find a numeric slot by name, return -1 if absent
   int               FindNum(const string name) const
     {
      for(int i = 0; i < m_num_count; i++)
         if(m_nums[i].name == name) return i;
      return -1;
     }

   //--- Get or create a bool slot
   int               EnsureBool(const string name)
     {
      int idx = FindBool(name);
      if(idx != -1) return idx;
      ArrayResize(m_bools, m_bool_count + 1);
      m_bools[m_bool_count].name        = name;
      m_bools[m_bool_count].current_val = false;
      m_bools[m_bool_count].total       = 0;
      for(int r = 0; r < 2; r++)
         for(int c = 0; c < 2; c++)
            m_bools[m_bool_count].n[r][c] = 0;
      return m_bool_count++;
     }

   //--- Get or create a numeric slot
   int               EnsureNum(const string name)
     {
      int idx = FindNum(name);
      if(idx != -1) return idx;
      ArrayResize(m_nums, m_num_count + 1);
      m_nums[m_num_count].name         = name;
      m_nums[m_num_count].current_val  = 0.0;
      m_nums[m_num_count].sum_all      = 0.0;
      m_nums[m_num_count].sum_sq_all   = 0.0;
      m_nums[m_num_count].count_all    = 0;
      m_nums[m_num_count].sum_true     = 0.0;
      m_nums[m_num_count].sum_true_sq  = 0.0;
      m_nums[m_num_count].count_true   = 0;
      m_nums[m_num_count].sum_false    = 0.0;
      m_nums[m_num_count].sum_false_sq = 0.0;
      m_nums[m_num_count].count_false  = 0;
      return m_num_count++;
     }

   //--- Compute Phi coefficient from a 2x2 contingency table
   //    phi = (n11*n00 - n10*n01) / sqrt(n1. * n0. * n.1 * n.0)
   //    Returns 0.0 when undefined (zero marginal)
   double            CalcPhi(const BoolParamStat &s) const
     {
      double n11 = s.n[1][1];  // param=T, outcome=T
      double n10 = s.n[1][0];  // param=T, outcome=F
      double n01 = s.n[0][1];  // param=F, outcome=T
      double n00 = s.n[0][0];  // param=F, outcome=F

      double row1 = n11 + n10;  // total param=T
      double row0 = n01 + n00;  // total param=F
      double col1 = n11 + n01;  // total outcome=T
      double col0 = n10 + n00;  // total outcome=F

      double denom = row1 * row0 * col1 * col0;
      if(denom <= 0.0) return 0.0;
      return (n11 * n00 - n10 * n01) / MathSqrt(denom);
     }

   //--- Compute Point-Biserial r
   //    r_pb = (M1-M0)/S * sqrt(n1*n0/n^2)
   //    Returns 0.0 when undefined
   double            CalcPointBiserial(const NumParamStat &s) const
     {
      if(s.count_all < 2 || s.count_true == 0 || s.count_false == 0)
         return 0.0;

      double M1 = s.sum_true  / s.count_true;
      double M0 = s.sum_false / s.count_false;

      // Population std dev of the full param series
      double mean_all = s.sum_all / s.count_all;
      double var_all  = (s.sum_sq_all / s.count_all) - mean_all * mean_all;
      if(var_all <= 0.0) return 0.0;

      double S = MathSqrt(var_all);
      double n = (double)s.count_all;

      return (M1 - M0) / S * MathSqrt((double)(s.count_true * s.count_false) / (n * n));
     }

   //--- Write one CSV row (called inside Observe())
   void              WriteCSVRow(bool outcome)
     {
      if(!m_exporting || m_file < 0) return;

      string row = TimeToString(TimeCurrent(), TIME_DATE | TIME_SECONDS) +
                   "," + (outcome ? "1" : "0");

      for(int i = 0; i < m_bool_count; i++)
         row += "," + (m_bools[i].current_val ? "1" : "0");

      for(int i = 0; i < m_num_count; i++)
         row += "," + DoubleToString(m_nums[i].current_val, 6);

      FileWrite(m_file, row);
     }

   //--- Write CSV header row
   void              WriteCSVHeader()
     {
      if(!m_exporting || m_file < 0) return;
      string hdr = "datetime,outcome";
      for(int i = 0; i < m_bool_count; i++)  hdr += "," + m_bools[i].name;
      for(int i = 0; i < m_num_count;  i++)  hdr += "," + m_nums[i].name;
      FileWrite(m_file, hdr);
     }

   //--- Bar separator string
   string            Bar(int len = 64) const
     {
      string s = "";
      for(int i = 0; i < len; i++) s += "-";
      return s;
     }

public:
   //=================================================================
   //  Construction / Destruction
   //=================================================================

                     CCorrelationStudy(const string study_name, OutcomeFunc outcome_func)
     {
      m_name         = study_name;
      m_outcome      = outcome_func;
      m_total        = 0;
      m_outcome_true = 0;
      m_bool_count   = 0;
      m_num_count    = 0;
      m_file         = INVALID_HANDLE;
      m_exporting    = false;
      ArrayResize(m_bools, 0);
      ArrayResize(m_nums,  0);
     }

                    ~CCorrelationStudy()
     {
      StopExport();
     }

   //=================================================================
   //  PARAMETER VALUE SETTERS
   //  Call these each candle/bar before Observe()
   //=================================================================

   //--- Boolean parameter (e.g. "LastCSID", "PrevBullish")
   void              SetBool(const string name, bool val)
     {
      int idx = EnsureBool(name);
      m_bools[idx].current_val = val;
     }

   //--- Continuous real-valued parameter (e.g. "PrevRange", "ATR", "RoomToLeft")
   void              SetDouble(const string name, double val)
     {
      int idx = EnsureNum(name);
      m_nums[idx].current_val = val;
     }

   //--- Integer parameter (e.g. "ConsecCandles", "SwingCount")
   void              SetInt(const string name, int val)
     {
      SetDouble(name, (double)val);
     }

   //--- Datetime → extracts hour-of-day [0..23] as a numeric param.
   //    Use this for session/time-of-day studies.
   void              SetHour(const string name, datetime dt)
     {
      MqlDateTime s;
      TimeToStruct(dt, s);
      SetDouble(name, (double)s.hour);
     }

   //--- Datetime → extracts day-of-week [0=Sun..6=Sat] as numeric.
   void              SetDayOfWeek(const string name, datetime dt)
     {
      MqlDateTime s;
      TimeToStruct(dt, s);
      SetDouble(name, (double)s.day_of_week);
     }

   //--- Raw datetime stored as seconds-since-epoch (for full resolution)
   void              SetDateTime(const string name, datetime dt)
     {
      SetDouble(name, (double)(long)dt);
     }

   //=================================================================
   //  OBSERVE  — record one data point
   //  Call once per candle / bar after setting all params.
   //  Returns the outcome value so the caller can branch if needed.
   //=================================================================
   bool              Observe()
     {
      if(m_outcome == NULL)
        {
         Print("CCorrelationStudy::Observe — outcome function is null.");
         return false;
        }

      bool out = m_outcome();
      int  ov  = out ? 1 : 0;

      m_total++;
      if(out) m_outcome_true++;

      // --- Update bool param stats
      for(int i = 0; i < m_bool_count; i++)
        {
         int pv = m_bools[i].current_val ? 1 : 0;
         m_bools[i].n[pv][ov]++;
         m_bools[i].total++;
        }

      // --- Update numeric param stats
      for(int i = 0; i < m_num_count; i++)
        {
         double v = m_nums[i].current_val;
         m_nums[i].sum_all    += v;
         m_nums[i].sum_sq_all += v * v;
         m_nums[i].count_all++;

         if(out)
           {
            m_nums[i].sum_true    += v;
            m_nums[i].sum_true_sq += v * v;
            m_nums[i].count_true++;
           }
         else
           {
            m_nums[i].sum_false    += v;
            m_nums[i].sum_false_sq += v * v;
            m_nums[i].count_false++;
           }
        }

      // --- CSV
      WriteCSVRow(out);

      return out;
     }

   //=================================================================
   //  REPORT  — print full analysis to Experts log
   //=================================================================
   void              Report() const
     {
      if(m_total == 0)
        {
         Print("CCorrelationStudy[", m_name, "] — no observations yet.");
         return;
        }

      double base_rate = (double)m_outcome_true / m_total;

      Print(Bar(66));
      PrintFormat("  CORRELATION STUDY: %s", m_name);
      PrintFormat("  Observations : %d", m_total);
      PrintFormat("  Outcome=TRUE : %d  (base rate = %.1f%%)", m_outcome_true, base_rate * 100.0);
      Print(Bar(66));

      // ---- BOOL PARAMETERS ------------------------------------------
      if(m_bool_count > 0)
        {
         Print("  [ BOOL PARAMETERS ]");
         PrintFormat("  %-20s  %6s  %6s  %6s  %6s  %6s  %6s",
                     "Name", "P(o|p=T)", "P(o|p=F)", "Lift", "Phi", "n(T)", "n(F)");
         Print("  ", Bar(74));

         for(int i = 0; i < m_bool_count; i++)
           {
            const BoolParamStat *s = GetPointer(m_bools[i]);

            int nT = s.n[1][1] + s.n[1][0];   // param=true total
            int nF = s.n[0][1] + s.n[0][0];   // param=false total

            double rate_T = (nT > 0) ? (double)s.n[1][1] / nT : 0.0;
            double rate_F = (nF > 0) ? (double)s.n[0][1] / nF : 0.0;
            double lift   = (base_rate > 0.0) ? rate_T / base_rate : 0.0;
            double phi    = CalcPhi(m_bools[i]);

            PrintFormat("  %-20s  %5.1f%%  %5.1f%%  %6.2f  %+6.3f  %6d  %6d",
                        s.name, rate_T * 100.0, rate_F * 100.0,
                        lift, phi, nT, nF);
           }
        }

      // ---- NUMERIC PARAMETERS ---------------------------------------
      if(m_num_count > 0)
        {
         Print("");
         Print("  [ NUMERIC PARAMETERS ]");
         PrintFormat("  %-20s  %8s  %8s  %6s  %6s  %6s",
                     "Name", "Mean|T", "Mean|F", "Ratio", "r_pb", "n");
         Print("  ", Bar(74));

         for(int i = 0; i < m_num_count; i++)
           {
            const NumParamStat *s = GetPointer(m_nums[i]);
            if(s.count_all == 0) continue;

            double M1 = (s.count_true  > 0) ? s.sum_true  / s.count_true  : 0.0;
            double M0 = (s.count_false > 0) ? s.sum_false / s.count_false : 0.0;
            double ratio = (M0 != 0.0) ? M1 / M0 : 0.0;
            double rpb   = CalcPointBiserial(m_nums[i]);

            PrintFormat("  %-20s  %8.4f  %8.4f  %6.2f  %+6.3f  %6d",
                        s.name, M1, M0, ratio, rpb, s.count_all);
           }
        }

      Print(Bar(66));
      Print("  INTERPRETATION GUIDE");
      Print("  Phi:  -1=negative, 0=none, +1=positive  (bool params)");
      Print("  Lift: >1 param=T raises outcome rate; <1 lowers it");
      Print("  r_pb: -1=negative, 0=none, +1=positive  (numeric params)");
      Print("  Ratio: Mean|T / Mean|F  (>1 → larger values → more outcome)");
      Print(Bar(66));
     }

   //=================================================================
   //  CSV EXPORT
   //=================================================================

   //--- Open a CSV file and start streaming raw observations.
   //    File is created in the MQL5/Files directory.
   //    Returns true on success.
   bool              StartExport(const string filename)
     {
      if(m_exporting)
        {
         Print("CCorrelationStudy::StartExport — already exporting. Call StopExport() first.");
         return false;
        }

      m_file = FileOpen(filename, FILE_WRITE | FILE_CSV | FILE_ANSI, ',');
      if(m_file == INVALID_HANDLE)
        {
         PrintFormat("CCorrelationStudy::StartExport — cannot open '%s'. Error: %d",
                     filename, GetLastError());
         return false;
        }

      m_exporting = true;
      WriteCSVHeader();
      PrintFormat("CCorrelationStudy[%s] — CSV export started: %s", m_name, filename);
      return true;
     }

   //--- Flush and close the CSV file.
   void              StopExport()
     {
      if(!m_exporting || m_file == INVALID_HANDLE) return;
      FileClose(m_file);
      m_file      = INVALID_HANDLE;
      m_exporting = false;
      PrintFormat("CCorrelationStudy[%s] — CSV export stopped.", m_name);
     }

   bool              IsExporting() const { return m_exporting; }

   //=================================================================
   //  QUICK STATS  (single param queries without full report)
   //=================================================================

   //--- Base rate of the outcome across all observations.
   double            BaseRate()    const { return (m_total > 0) ? (double)m_outcome_true / m_total : 0.0; }
   int               TotalObs()   const { return m_total; }
   int               OutcomeTrue() const { return m_outcome_true; }

   //--- Point-biserial r for one numeric param by name.
   double            GetRpb(const string name) const
     {
      int idx = FindNum(name);
      return (idx == -1) ? 0.0 : CalcPointBiserial(m_nums[idx]);
     }

   //--- Phi coefficient for one bool param by name.
   double            GetPhi(const string name) const
     {
      int idx = FindBool(name);
      return (idx == -1) ? 0.0 : CalcPhi(m_bools[idx]);
     }

   //--- Mean of a numeric param when outcome was true/false.
   double            MeanWhenTrue(const string name) const
     {
      int idx = FindNum(name);
      if(idx == -1 || m_nums[idx].count_true == 0) return 0.0;
      return m_nums[idx].sum_true / m_nums[idx].count_true;
     }

   double            MeanWhenFalse(const string name) const
     {
      int idx = FindNum(name);
      if(idx == -1 || m_nums[idx].count_false == 0) return 0.0;
      return m_nums[idx].sum_false / m_nums[idx].count_false;
     }

   //=================================================================
   //  RESET
   //=================================================================

   //--- Wipe all accumulated statistics but keep param slot names.
   void              ResetStats()
     {
      m_total        = 0;
      m_outcome_true = 0;

      for(int i = 0; i < m_bool_count; i++)
        {
         m_bools[i].total = 0;
         for(int r = 0; r < 2; r++)
            for(int c = 0; c < 2; c++)
               m_bools[i].n[r][c] = 0;
        }

      for(int i = 0; i < m_num_count; i++)
        {
         m_nums[i].sum_all      = 0; m_nums[i].sum_sq_all   = 0; m_nums[i].count_all   = 0;
         m_nums[i].sum_true     = 0; m_nums[i].sum_true_sq  = 0; m_nums[i].count_true  = 0;
         m_nums[i].sum_false    = 0; m_nums[i].sum_false_sq = 0; m_nums[i].count_false = 0;
        }
     }

   //--- Wipe everything including param slot definitions.
   void              FullReset()
     {
      ResetStats();
      m_bool_count = 0;
      m_num_count  = 0;
      ArrayResize(m_bools, 0);
      ArrayResize(m_nums,  0);
     }
  };

//+------------------------------------------------------------------+
//  END OF FRAMEWORK
//+------------------------------------------------------------------+