Crystal-DataExtractor-MT5/Crystal_DataExtractor.mq5

//+------------------------------------------------------------------+
//|                                      Crystal_DataExtractor.mq5  |
//|                                          Crystal AI Systems      |
//|                    https://www.mql5.com/en/users/jawadtrader22   |
//+------------------------------------------------------------------+
//
//  ╔══════════════════════════════════════════════════════════════════╗
//  ║           CRYSTAL DATA EXTRACTOR — USER GUIDE                  ║
//  ╠══════════════════════════════════════════════════════════════════╣
//  ║                                                                  ║
//  ║  WHAT THIS EA DOES:                                              ║
//  ║  Exports market data (Tick or OHLC) to a proper CSV file.        ║
//  ║  Columns are comma-separated — opens correctly in Excel.         ║
//  ║  Works on live charts AND in Strategy Tester.                    ║
//  ║                                                                  ║
//  ║  HOW TO USE — STEP BY STEP:                                      ║
//  ║                                                                  ║
//  ║  1. Attach EA to any chart (e.g. XAUUSD M1)                      ║
//  ║                                                                  ║
//  ║  2. In EA Inputs, set:                                           ║
//  ║     • Data Mode    → TICK = raw tick data (Bid/Ask/Last)         ║
//  ║                   → OHLC = closed candle bar data                ║
//  ║     • Timeframe    → Only used in OHLC mode                      ║
//  ║                      e.g. M15 = 15-minute candles                ║
//  ║     • File Prefix  → Optional label added to filename            ║
//  ║                      Leave blank for auto-naming                 ║
//  ║     • Lookback Bars→ Past OHLC bars dumped on startup            ║
//  ║                      (0 = skip historical dump)                  ║
//  ║                                                                  ║
//  ║  3. For STRATEGY TESTER:                                         ║
//  ║     • Set Tester chart TF LOWER than your OHLC input TF          ║
//  ║       Example: Want H1 OHLC? Run tester on M1 or M5             ║
//  ║     • Use "Every tick" model for best accuracy                   ║
//  ║     • Tick mode works on any tester model                        ║
//  ║                                                                  ║
//  ╠══════════════════════════════════════════════════════════════════╣
//  ║  📁  WHERE IS MY FILE? — READ CAREFULLY                          ║
//  ║                                                                  ║
//  ║  LIVE CHART (normal use):                                        ║
//  ║  MT5 menu → File → Open Data Folder                              ║
//  ║  Then navigate to:  MQL5\Files\                                  ║
//  ║  Example full path:                                              ║
//  ║  C:\Users\[You]\AppData\Roaming\MetaQuotes\                      ║
//  ║  Terminal\[TerminalID]\MQL5\Files\XAUUSD_M1_OHLC.csv            ║
//  ║                                                                  ║
//  ║  STRATEGY TESTER (different location!):                          ║
//  ║  MT5 menu → File → Open Data Folder                              ║
//  ║  Then navigate to:                                               ║
//  ║  Tester\[AgentID]\Agent-127.0.0.1-3000\MQL5\Files\              ║
//  ║  Example full path:                                              ║
//  ║  C:\Users\[You]\AppData\Roaming\MetaQuotes\                      ║
//  ║  Terminal\[TerminalID]\Tester\[AgentID]\                         ║
//  ║  Agent-127.0.0.1-3000\MQL5\Files\XAUUSD_M1_OHLC.csv            ║
//  ║                                                                  ║
//  ║  ⚠️  Tester files are NOT in the main MQL5\Files folder!          ║
//  ║      Always look inside the Tester\...\MQL5\Files\ subfolder.    ║
//  ║                                                                  ║
//  ╠══════════════════════════════════════════════════════════════════╣
//  ║  FILE NAMING FORMAT:                                             ║
//  ║  OHLC mode  →  [Symbol]_[TF]_OHLC.csv                           ║
//  ║               e.g.  XAUUSD_M15_OHLC.csv                         ║
//  ║  Tick mode  →  [Symbol]_Ticks.csv                                ║
//  ║               e.g.  XAUUSD_Ticks.csv                             ║
//  ║  With prefix→  [Prefix]_[Symbol]_[TF]_OHLC.csv                  ║
//  ║               e.g.  Test1_XAUUSD_M15_OHLC.csv                   ║
//  ║                                                                  ║
//  ║  CSV COLUMNS:                                                    ║
//  ║  Tick:  Time, Bid, Ask, Last, TickCount                          ║
//  ║  OHLC:  Time, Open, High, Low, Close, Volume, Spread             ║
//  ║                                                                  ║
//  ║  TIPS:                                                           ║
//  ║  • File uses comma separator — opens directly in Excel           ║
//  ║  • Each EA attach/tester run creates a FRESH file                ║
//  ║  • Journal tab prints the exact file path on start & stop        ║
//  ║                                                                  ║
//  ╚══════════════════════════════════════════════════════════════════╝
//
#property copyright "Crystal AI Systems"
#property link      "https://www.mql5.com/en/users/jawadtrader22/seller"
#property version   "3.00"
#property strict
#property description "Crystal Data Extractor — Tick or OHLC to proper comma-separated CSV."
#property description "Columns open correctly in Excel — no merging."
#property description " "
#property description "LIVE CHART file location:"
#property description "File > Open Data Folder > MQL5 > Files > [file].csv"
#property description " "
#property description "STRATEGY TESTER location (DIFFERENT from live!):"
#property description "File > Open Data Folder > Tester > [ID] > Agent-127.0.0.1-3000 > MQL5 > Files"
#property description " "
#property description "TICK file  ->  [Symbol]_Ticks.csv"
#property description "OHLC file  ->  [Symbol]_[TF]_OHLC.csv"
#property description " "
#property description "TESTER TIP: Chart TF must be LOWER than OHLC input TF"

//+------------------------------------------------------------------+
//|  ENUMS                                                           |
//+------------------------------------------------------------------+
enum EDATA_MODE
{
   MODE_TICK = 0,  // 📈 Tick Data  (Bid / Ask / Last)
   MODE_OHLC = 1   // 📊 OHLC Bars  (Open / High / Low / Close)
};

//+------------------------------------------------------------------+
//|  INPUT PARAMETERS                                                |
//+------------------------------------------------------------------+
input group "=== ⚙️  Data Settings ==="
input EDATA_MODE      InpDataMode     = MODE_TICK;       // Data Mode
input ENUM_TIMEFRAMES InpTimeframe    = PERIOD_M1;       // Timeframe (OHLC mode only)
input int             InpLookback     = 500;             // Lookback Bars on Start (OHLC, 0=off)

input group "=== 📁  File Settings ==="
input string          InpFilePrefix   = "";              // File Prefix (blank = auto)

input group "=== INFO: LIVE CHART file location ==="
input string          InpGuide1       = "File > Open Data Folder";               // Step 1
input string          InpGuide2       = "Then: MQL5 > Files > [file].csv";       // Step 2
input group "=== INFO: TESTER file location (different!) ==="
input string          InpGuide3       = "File > Open Data Folder";               // Step 1
input string          InpGuide4       = "Then: Tester > [ID] > Agent-127...";    // Step 2
input string          InpGuide5       = "> MQL5 > Files > [file].csv";           // Step 3
input group "=== INFO: File names ==="
input string          InpGuide6       = "TICK  ->  [Symbol]_Ticks.csv";          // Tick name
input string          InpGuide7       = "OHLC  ->  [Symbol]_[TF]_OHLC.csv";     // OHLC name
input string          InpGuide8       = "For Histry Data Run Tester and For Live attached With Chart";    // Tester tip

//+------------------------------------------------------------------+
//|  GLOBAL VARIABLES                                                |
//+------------------------------------------------------------------+
int      g_fh          = INVALID_HANDLE;
datetime g_lastBarTime = 0;
datetime g_lastTickTime = 0;
int      g_tickCount   = 0;
string   g_filename    = "";

//+------------------------------------------------------------------+
//|  HELPERS                                                         |
//+------------------------------------------------------------------+
string TFToString(ENUM_TIMEFRAMES tf)
{
   switch(tf)
   {
      case PERIOD_M1:  return "M1";
      case PERIOD_M2:  return "M2";
      case PERIOD_M3:  return "M3";
      case PERIOD_M4:  return "M4";
      case PERIOD_M5:  return "M5";
      case PERIOD_M6:  return "M6";
      case PERIOD_M10: return "M10";
      case PERIOD_M12: return "M12";
      case PERIOD_M15: return "M15";
      case PERIOD_M20: return "M20";
      case PERIOD_M30: return "M30";
      case PERIOD_H1:  return "H1";
      case PERIOD_H2:  return "H2";
      case PERIOD_H3:  return "H3";
      case PERIOD_H4:  return "H4";
      case PERIOD_H6:  return "H6";
      case PERIOD_H8:  return "H8";
      case PERIOD_H12: return "H12";
      case PERIOD_D1:  return "D1";
      case PERIOD_W1:  return "W1";
      case PERIOD_MN1: return "MN1";
      default:         return "TF" + IntegerToString((int)tf);
   }
}

//--- Build dynamic filename
string BuildFilename()
{
   string prefix = (InpFilePrefix != "") ? InpFilePrefix + "_" : "";
   if(InpDataMode == MODE_OHLC)
      return prefix + _Symbol + "_" + TFToString(InpTimeframe) + "_OHLC.csv";
   else
      return prefix + _Symbol + "_Ticks.csv";
}

//--- Write OHLC header
void WriteOHLCHeader()
{
   FileWriteString(g_fh, "Time,Open,High,Low,Close,Volume,Spread\n");
}

//--- Write Tick header
void WriteTickHeader()
{
   FileWriteString(g_fh, "Time,Bid,Ask,Last,TickCount\n");
}

//--- Dump historical OHLC bars on init
void DumpHistoricalOHLC()
{
   if(InpLookback <= 0) return;

   MqlRates rates[];
   int copied = CopyRates(_Symbol, InpTimeframe, 1, InpLookback, rates);
   if(copied <= 0)
   {
      Print("⚠️  CopyRates failed, error: ", GetLastError());
      return;
   }

   for(int i = copied - 1; i >= 0; i--)
   {
      string row = TimeToString(rates[i].time, TIME_DATE | TIME_SECONDS) + "," +
                   DoubleToString(rates[i].open,  _Digits)               + "," +
                   DoubleToString(rates[i].high,  _Digits)               + "," +
                   DoubleToString(rates[i].low,   _Digits)               + "," +
                   DoubleToString(rates[i].close, _Digits)               + "," +
                   IntegerToString(rates[i].tick_volume)                 + "," +
                   IntegerToString(rates[i].spread)                      + "\n";
      FileWriteString(g_fh, row);
   }

   PrintFormat("✅  Dumped %d historical OHLC bars [%s %s]",
               copied, _Symbol, TFToString(InpTimeframe));

   //--- Set last bar time to most recent dumped bar so we don't duplicate
   g_lastBarTime = rates[0].time;
}

//+------------------------------------------------------------------+
//|  OnInit                                                          |
//+------------------------------------------------------------------+
int OnInit()
{
   //--- Tester TF sanity warning (OHLC mode)
   if(InpDataMode == MODE_OHLC && Period() > InpTimeframe)
   {
      Print("⚠️  WARNING: Chart timeframe (", TFToString(Period()), ") is HIGHER than OHLC",
            " input TF (", TFToString(InpTimeframe), ").");
      Print("   Bar detection may miss candles. Set chart TF LOWER than OHLC TF.");
   }

   g_filename = BuildFilename();

   // FILE_TXT + FileWriteString = explicit comma separator → proper Excel columns
   // FILE_CSV uses tab by default which merges all columns into column A in Excel
   g_fh = FileOpen(g_filename, FILE_WRITE | FILE_TXT | FILE_ANSI);
   if(g_fh == INVALID_HANDLE)
   {
      PrintFormat("❌  Cannot open file [%s], error: %d", g_filename, GetLastError());
      return INIT_FAILED;
   }

   //--- Write header
   if(InpDataMode == MODE_OHLC)
      WriteOHLCHeader();
   else
      WriteTickHeader();

   //--- Historical dump (OHLC only)
   if(InpDataMode == MODE_OHLC)
      DumpHistoricalOHLC();

   PrintFormat("✅  Crystal Data Extractor started | Mode: %s | File: %s",
               (InpDataMode == MODE_OHLC ? "OHLC [" + TFToString(InpTimeframe) + "]" : "TICK"),
               g_filename);

   Print("📁  LIVE path:   File > Open Data Folder > MQL5 > Files > ", g_filename);
   Print("📁  TESTER path: File > Open Data Folder > Tester > [ID] > Agent-127.0.0.1-3000 > MQL5 > Files > ", g_filename);

   return INIT_SUCCEEDED;
}

//+------------------------------------------------------------------+
//|  OnTick — Tick Mode                                              |
//+------------------------------------------------------------------+
void ProcessTick()
{
   MqlTick t;
   if(!SymbolInfoTick(_Symbol, t)) return;

   //--- Tick counter resets each new second
   if(t.time != g_lastTickTime)
   {
      g_tickCount    = 1;
      g_lastTickTime = t.time;
   }
   else
   {
      g_tickCount++;
   }

   string row = TimeToString(t.time, TIME_DATE | TIME_SECONDS) + "," +
                DoubleToString(t.bid,  _Digits)               + "," +
                DoubleToString(t.ask,  _Digits)               + "," +
                DoubleToString(t.last, _Digits)               + "," +
                IntegerToString(g_tickCount)                  + "\n";
   FileWriteString(g_fh, row);
}

//+------------------------------------------------------------------+
//|  OnTick — OHLC Mode                                              |
//+------------------------------------------------------------------+
void ProcessOHLC()
{
   //--- Check if new bar has opened on the selected TF
   datetime currentBar = iTime(_Symbol, InpTimeframe, 0);
   if(currentBar == 0 || currentBar == g_lastBarTime) return;

   g_lastBarTime = currentBar;

   //--- Read bar index 1 = the just-closed completed bar
   MqlRates rates[1];
   if(CopyRates(_Symbol, InpTimeframe, 1, 1, rates) <= 0)
   {
      Print("⚠️  CopyRates bar[1] failed: ", GetLastError());
      return;
   }

   string row = TimeToString(rates[0].time, TIME_DATE | TIME_SECONDS) + "," +
                DoubleToString(rates[0].open,  _Digits)                + "," +
                DoubleToString(rates[0].high,  _Digits)                + "," +
                DoubleToString(rates[0].low,   _Digits)                + "," +
                DoubleToString(rates[0].close, _Digits)                + "," +
                IntegerToString(rates[0].tick_volume)                  + "," +
                IntegerToString(rates[0].spread)                       + "\n";
   FileWriteString(g_fh, row);
}

//+------------------------------------------------------------------+
//|  OnTick                                                          |
//+------------------------------------------------------------------+
void OnTick()
{
   if(g_fh == INVALID_HANDLE) return;

   if(InpDataMode == MODE_TICK)
      ProcessTick();
   else
      ProcessOHLC();
}

//+------------------------------------------------------------------+
//|  OnDeinit                                                        |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
{
   if(g_fh != INVALID_HANDLE)
   {
      FileFlush(g_fh);
      FileClose(g_fh);
      g_fh = INVALID_HANDLE;
      PrintFormat("✅  File saved and closed: %s", g_filename);
      Print("📁  LIVE path:   File > Open Data Folder > MQL5 > Files > ", g_filename);
      Print("📁  TESTER path: File > Open Data Folder > Tester > [ID] > Agent-127.0.0.1-3000 > MQL5 > Files > ", g_filename);
   }
}
//+------------------------------------------------------------------+