# -*- coding: utf-8 -*-
"""
Algo Forge - Tahap 4 (ML Hibrida): download_bars.py
====================================================
Unduh data OHLCV HISTORIS PANJANG (±20 tahun) dari terminal MetaTrader 5
(HFM Demo, simbol XAUUSD) secara chunked per tahun, lalu VERIFIKASI
kontinuitas (gap detection + sanity OHLC) dan simpan ke:

    MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.csv   (arsip teks, audit)
    MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.npz   (arrays numpy, cepat)

TIDAK menyentuh Files\\SniperGold_ML (cache baseline Fase 3 = deploy freeze).

Cara pakai:
    python download_bars.py [--symbol XAUUSD] [--tf M15,H1,H4,D1] [--force] [--probe]

  --probe  : hanya lapor ketersediaan (bar count, rentang tanggal) tanpa unduh.
  --force  : unduh ulang walau file sudah ada.
  --start  : ISO date awal (default: 2000-01-01 -> terminal kembalikan yang ada).

Verifikasi kontinuitas:
  - urutan waktu strict naik, tanpa duplikat
  - step antar bar dibandingkan dengan step TF; gap <= 3x step dianggap normal
  - gap 3x..4 hari = weekend/holiday (dicatat, bukan error)
  - gap > 4 hari = ANOMALI (dilaporkan; total & terpanjang)
  - sanity OHLC: high >= max(o,c), low <= min(o,c), volume >= 0
"""
import os
import sys
import json
import argparse
import datetime as dt

import numpy as np
import MetaTrader5 as mt5

# ----------------------------------------------------------------------
BASE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                    "..", "..", "..", "Files", "AlgoForge", "Data")
BASE = os.path.normpath(BASE)

TF_MAP = {
    "M15": mt5.TIMEFRAME_M15,
    "H1":  mt5.TIMEFRAME_H1,
    "H4":  mt5.TIMEFRAME_H4,
    "D1":  mt5.TIMEFRAME_D1,
}
STEP = {"M15": 15 * 60, "H1": 3600, "H4": 4 * 3600, "D1": 86400}
GAP_ANOMALY = 4 * 86400          # > 4 hari = anomali
GAP_WEEKEND = 3 * 3600           # batas "normal" (3x step utk M15)

DEFAULT_TFS = ["M15", "H1", "H4", "D1"]


def ensure_dir(path):
    os.makedirs(path, exist_ok=True)


def fetch_range_chunked(symbol, tf_enum, start, end, step_sec):
    """Unduh [start, end] per-chunk ~ 2 tahun, gabung, urutkan, dedup."""
    chunks = []
    cur = start
    while cur < end:
        nxt = min(cur + dt.timedelta(days=730), end)
        rates = mt5.copy_rates_range(symbol, tf_enum, cur, nxt)
        if rates is None or len(rates) == 0:
            err = mt5.last_error()
            print(f"    [chunk {cur.date()}..{nxt.date()}] kosong ({err})")
        else:
            chunks.append(rates)
            print(f"    [chunk {cur.date()}..{nxt.date()}] {len(rates)} bar")
        cur = nxt
    if not chunks:
        return None
    allr = np.concatenate(chunks)
    # urutkan & dedup (jaga-jaga overlap chunk)
    allr = allr[np.argsort(allr["time"], kind="stable")]
    _, idx = np.unique(allr["time"], return_index=True)
    allr = allr[idx]
    return allr


def verify_continuity(rates, tf_name, out_report):
    """Cek urutan, duplikat, gap, sanity OHLC. Kembali dict laporan."""
    t = rates["time"].astype(np.int64)
    step = STEP[tf_name]
    rep = {"tf": tf_name, "n": int(len(rates))}
    if len(rates) == 0:
        rep.update(status="EMPTY")
        return rep

    dup = int(np.sum(np.diff(t) == 0))
    not_inc = int(np.sum(np.diff(t) < 0))
    d = np.diff(t)
    gaps = d[d > step * 3]                 # lebih dari 3x step
    normal_gaps = int(np.sum((d > step) & (d <= step * 3)))
    weekend_gaps = int(np.sum((d > step * 3) & (d <= GAP_ANOMALY)))
    anomaly_gaps = int(np.sum(d > GAP_ANOMALY))
    longest = int(d.max()) if len(d) else 0

    # sanity OHLC
    bad_hl = int(np.sum(rates["high"] < np.maximum(rates["open"], rates["close"])))
    bad_low = int(np.sum(rates["low"] > np.minimum(rates["open"], rates["close"])))
    bad_vol = int(np.sum(rates["tick_volume"] < 0))

    first = dt.datetime.utcfromtimestamp(int(t[0])).strftime("%Y-%m-%d %H:%M")
    last = dt.datetime.utcfromtimestamp(int(t[-1])).strftime("%Y-%m-%d %H:%M")
    span_years = (t[-1] - t[0]) / (365.25 * 86400)

    rep.update(
        status="OK",
        first=first, last=last,
        span_years=round(float(span_years), 2),
        dup=dup, not_increasing=not_inc,
        normal_gaps=normal_gaps, weekend_gaps=weekend_gaps,
        anomaly_gaps=anomaly_gaps, longest_gap_sec=int(longest),
        bad_hl=bad_hl, bad_low=bad_low, bad_vol=bad_vol,
    )
    if dup or not_inc or anomaly_gaps or bad_hl or bad_low or bad_vol:
        rep["status"] = "WARN"
    return rep


def save(symbol, tf_name, rates):
    path_csv = os.path.join(BASE, f"{symbol}_{tf_name}.csv")
    path_npz = os.path.join(BASE, f"{symbol}_{tf_name}.npz")
    with open(path_csv, "w", encoding="utf-8") as f:
        f.write("time,open,high,low,close,tick_volume,spread\n")
        for r in rates:
            tt = dt.datetime.utcfromtimestamp(int(r["time"])).strftime("%Y-%m-%d %H:%M:%S")
            f.write(f"{tt},{r['open']:.5f},{r['high']:.5f},{r['low']:.5f},"
                    f"{r['close']:.5f},{int(r['tick_volume'])},{int(r['spread'])}\n")
    np.savez_compressed(path_npz,
                        time=rates["time"].astype(np.int64),
                        open=rates["open"].astype(np.float64),
                        high=rates["high"].astype(np.float64),
                        low=rates["low"].astype(np.float64),
                        close=rates["close"].astype(np.float64),
                        tick_volume=rates["tick_volume"].astype(np.int64),
                        spread=rates["spread"].astype(np.int64))
    print(f"    -> {path_csv}  ({len(rates)} bar)")
    print(f"    -> {path_npz}")
    return path_csv, path_npz


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--symbol", default="XAUUSD")
    ap.add_argument("--tf", default=",".join(DEFAULT_TFS))
    ap.add_argument("--start", default="2000-01-01")
    ap.add_argument("--force", action="store_true")
    ap.add_argument("--probe", action="store_true")
    args = ap.parse_args()

    tfs = [x.strip().upper() for x in args.tf.split(",") if x.strip()]
    for x in tfs:
        if x not in TF_MAP:
            print(f"TF tidak dikenal: {x} (pilih {list(TF_MAP)})")
            sys.exit(2)

    if not mt5.initialize():
        print("Gagal initialize():", mt5.last_error())
        sys.exit(1)
    info = mt5.terminal_info()
    si = mt5.symbol_info(args.symbol)
    if info is None or si is None:
        print(f"Terminal/simbol {args.symbol} tidak ditemukan:", mt5.last_error())
        mt5.shutdown()
        sys.exit(1)

    print(f"Terminal: {info.name} | {info.path}")
    print(f"Simbol  : {args.symbol} | {si.description} | digits={si.digits}")
    ensure_dir(BASE)

    start = dt.datetime.strptime(args.start, "%Y-%m-%d")
    end = dt.datetime.now(dt.timezone.utc).replace(tzinfo=None) + dt.timedelta(days=2)

    all_reports = []
    for tf_name in tfs:
        tf_enum = TF_MAP[tf_name]
        print(f"\n=== {args.symbol} {tf_name} ===")
        # probe: bar pertama & jumlah total
        r0 = mt5.copy_rates_from_pos(args.symbol, tf_enum, 0, 1)
        bars_total = mt5.symbol_info(args.symbol).history_bars(tf_enum, start, end) \
            if hasattr(mt5.symbol_info(args.symbol), "history_bars") else -1
        try:
            r_first = mt5.copy_rates_range(args.symbol, tf_enum, start, end)
        except Exception as e:  # noqa
            r_first = None
            print(f"    [probe gagal] {e}")
        if r0 is not None and len(r0):
            print(f"    bar terbaru : {dt.datetime.utcfromtimestamp(int(r0[0]['time']))}")
        if r_first is not None and len(r_first):
            print(f"    probe range : {len(r_first)} bar | "
                  f"mulai {dt.datetime.utcfromtimestamp(int(r_first[0]['time']))} | "
                  f"akhir {dt.datetime.utcfromtimestamp(int(r_first[-1]['time']))}")
            full = r_first
            rep = verify_continuity(full, tf_name, None)
            all_reports.append(rep)
            print(f"    [probe] n={rep['n']} span={rep['span_years']}y "
                  f"anomali_gap={rep['anomaly_gaps']} status={rep['status']}")
            if args.probe:
                continue
            out_csv, out_npz = save(args.symbol, tf_name, full)
            rep.update(csv=os.path.basename(out_csv), npz=os.path.basename(out_npz))
        else:
            print(f"    [probe] tidak ada data di range (mulai {args.start})")

    mt5.shutdown()

    # laporan ringkas
    print("\n=== RINGKASAN KONTINUITAS ===")
    for rep in all_reports:
        print(f"  {rep['tf']:4s} n={rep['n']:7d} span={rep['span_years']:6.2f}y "
              f"{rep.get('first')} .. {rep.get('last')} | dup={rep.get('dup',0)} "
              f"anom={rep.get('anomaly_gaps',0)} longest={rep.get('longest_gap_sec',0)}s "
              f"status={rep.get('status')}")

    if all_reports and not args.probe:
        path_rep = os.path.join(BASE, "download_report.json")
        with open(path_rep, "w", encoding="utf-8") as f:
            json.dump({"generated": dt.datetime.utcnow().isoformat(),
                       "symbol": args.symbol, "reports": all_reports},
                      f, indent=2)
        print(f"\nLaporan: {path_rep}")
    print("Selesai.")


if __name__ == "__main__":
    main()