forked from chiki2bum2/SniperGold_ML
230 lines
9 KiB
Python
230 lines
9 KiB
Python
 | # -*- coding: utf-8 -*-
| ||
"""
| |||
Algo Forge - Tahap 4 (ML Hibrida): download_bars.py
| |||
====================================================
| |||
Unduh data OHLCV HISTORIS PANJANG (±20 tahun) dari terminal MetaTrader 5
| |||
(HFM Demo, simbol XAUUSD) secara chunked per tahun, lalu VERIFIKASI
| |||
kontinuitas (gap detection + sanity OHLC) dan simpan ke:
| |||
| |||
MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.csv (arsip teks, audit)
| |||
MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.npz (arrays numpy, cepat)
| |||
| |||
TIDAK menyentuh Files\\SniperGold_ML (cache baseline Fase 3 = deploy freeze).
| |||
| |||
Cara pakai:
| |||
python download_bars.py [--symbol XAUUSD] [--tf M15,H1,H4,D1] [--force] [--probe]
| |||
| |||
--probe : hanya lapor ketersediaan (bar count, rentang tanggal) tanpa unduh.
| |||
--force : unduh ulang walau file sudah ada.
| |||
--start : ISO date awal (default: 2000-01-01 -> terminal kembalikan yang ada).
| |||
| |||
Verifikasi kontinuitas:
| |||
- urutan waktu strict naik, tanpa duplikat
| |||
- step antar bar dibandingkan dengan step TF; gap <= 3x step dianggap normal
| |||
- gap 3x..4 hari = weekend/holiday (dicatat, bukan error)
| |||
- gap > 4 hari = ANOMALI (dilaporkan; total & terpanjang)
| |||
- sanity OHLC: high >= max(o,c), low <= min(o,c), volume >= 0
| |||
"""
| |||
import os
| |||
import sys
| |||
import json
| |||
import argparse
| |||
import datetime as dt
| |||
| |||
import numpy as np
| |||
import MetaTrader5 as mt5
| |||
| |||
# ----------------------------------------------------------------------
| |||
BASE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
| |||
"..", "..", "..", "Files", "AlgoForge", "Data")
| |||
BASE = os.path.normpath(BASE)
| |||
| |||
TF_MAP = {
| |||
"M15": mt5.TIMEFRAME_M15,
| |||
"H1": mt5.TIMEFRAME_H1,
| |||
"H4": mt5.TIMEFRAME_H4,
| |||
"D1": mt5.TIMEFRAME_D1,
| |||
}
| |||
STEP = {"M15": 15 * 60, "H1": 3600, "H4": 4 * 3600, "D1": 86400}
| |||
GAP_ANOMALY = 4 * 86400 # > 4 hari = anomali
| |||
GAP_WEEKEND = 3 * 3600 # batas "normal" (3x step utk M15)
| |||
| |||
DEFAULT_TFS = ["M15", "H1", "H4", "D1"]
| |||
| |||
| |||
def ensure_dir(path):
| |||
os.makedirs(path, exist_ok=True)
| |||
| |||
| |||
def fetch_range_chunked(symbol, tf_enum, start, end, step_sec):
| |||
"""Unduh [start, end] per-chunk ~ 2 tahun, gabung, urutkan, dedup."""
| |||
chunks = []
| |||
cur = start
| |||
while cur < end:
| |||
nxt = min(cur + dt.timedelta(days=730), end)
| |||
rates = mt5.copy_rates_range(symbol, tf_enum, cur, nxt)
| |||
if rates is None or len(rates) == 0:
| |||
err = mt5.last_error()
| |||
print(f" [chunk {cur.date()}..{nxt.date()}] kosong ({err})")
| |||
else:
| |||
chunks.append(rates)
| |||
print(f" [chunk {cur.date()}..{nxt.date()}] {len(rates)} bar")
| |||
cur = nxt
| |||
if not chunks:
| |||
return None
| |||
allr = np.concatenate(chunks)
| |||
# urutkan & dedup (jaga-jaga overlap chunk)
| |||
allr = allr[np.argsort(allr["time"], kind="stable")]
| |||
_, idx = np.unique(allr["time"], return_index=True)
| |||
allr = allr[idx]
| |||
return allr
| |||
| |||
| |||
def verify_continuity(rates, tf_name, out_report):
| |||
"""Cek urutan, duplikat, gap, sanity OHLC. Kembali dict laporan."""
| |||
t = rates["time"].astype(np.int64)
| |||
step = STEP[tf_name]
| |||
rep = {"tf": tf_name, "n": int(len(rates))}
| |||
if len(rates) == 0:
| |||
rep.update(status="EMPTY")
| |||
return rep
| |||
| |||
dup = int(np.sum(np.diff(t) == 0))
| |||
not_inc = int(np.sum(np.diff(t) < 0))
| |||
d = np.diff(t)
| |||
gaps = d[d > step * 3] # lebih dari 3x step
| |||
normal_gaps = int(np.sum((d > step) & (d <= step * 3)))
| |||
weekend_gaps = int(np.sum((d > step * 3) & (d <= GAP_ANOMALY)))
| |||
anomaly_gaps = int(np.sum(d > GAP_ANOMALY))
| |||
longest = int(d.max()) if len(d) else 0
| |||
| |||
# sanity OHLC
| |||
bad_hl = int(np.sum(rates["high"] < np.maximum(rates["open"], rates["close"])))
| |||
bad_low = int(np.sum(rates["low"] > np.minimum(rates["open"], rates["close"])))
| |||
bad_vol = int(np.sum(rates["tick_volume"] < 0))
| |||
| |||
first = dt.datetime.utcfromtimestamp(int(t[0])).strftime("%Y-%m-%d %H:%M")
| |||
last = dt.datetime.utcfromtimestamp(int(t[-1])).strftime("%Y-%m-%d %H:%M")
| |||
span_years = (t[-1] - t[0]) / (365.25 * 86400)
| |||
| |||
rep.update(
| |||
status="OK",
| |||
first=first, last=last,
| |||
span_years=round(float(span_years), 2),
| |||
dup=dup, not_increasing=not_inc,
| |||
normal_gaps=normal_gaps, weekend_gaps=weekend_gaps,
| |||
anomaly_gaps=anomaly_gaps, longest_gap_sec=int(longest),
| |||
bad_hl=bad_hl, bad_low=bad_low, bad_vol=bad_vol,
| |||
)
| |||
if dup or not_inc or anomaly_gaps or bad_hl or bad_low or bad_vol:
| |||
rep["status"] = "WARN"
| |||
return rep
| |||
| |||
| |||
def save(symbol, tf_name, rates):
| |||
path_csv = os.path.join(BASE, f"{symbol}_{tf_name}.csv")
| |||
path_npz = os.path.join(BASE, f"{symbol}_{tf_name}.npz")
| |||
with open(path_csv, "w", encoding="utf-8") as f:
| |||
f.write("time,open,high,low,close,tick_volume,spread\n")
| |||
for r in rates:
| |||
tt = dt.datetime.utcfromtimestamp(int(r["time"])).strftime("%Y-%m-%d %H:%M:%S")
| |||
f.write(f"{tt},{r['open']:.5f},{r['high']:.5f},{r['low']:.5f},"
| |||
f"{r['close']:.5f},{int(r['tick_volume'])},{int(r['spread'])}\n")
| |||
np.savez_compressed(path_npz,
| |||
time=rates["time"].astype(np.int64),
| |||
open=rates["open"].astype(np.float64),
| |||
high=rates["high"].astype(np.float64),
| |||
low=rates["low"].astype(np.float64),
| |||
close=rates["close"].astype(np.float64),
| |||
tick_volume=rates["tick_volume"].astype(np.int64),
| |||
spread=rates["spread"].astype(np.int64))
| |||
print(f" -> {path_csv} ({len(rates)} bar)")
| |||
print(f" -> {path_npz}")
| |||
return path_csv, path_npz
| |||
| |||
| |||
def main():
| |||
ap = argparse.ArgumentParser()
| |||
ap.add_argument("--symbol", default="XAUUSD")
| |||
ap.add_argument("--tf", default=",".join(DEFAULT_TFS))
| |||
ap.add_argument("--start", default="2000-01-01")
| |||
ap.add_argument("--force", action="store_true")
| |||
ap.add_argument("--probe", action="store_true")
| |||
args = ap.parse_args()
| |||
| |||
tfs = [x.strip().upper() for x in args.tf.split(",") if x.strip()]
| |||
for x in tfs:
| |||
if x not in TF_MAP:
| |||
print(f"TF tidak dikenal: {x} (pilih {list(TF_MAP)})")
| |||
sys.exit(2)
| |||
| |||
if not mt5.initialize():
| |||
print("Gagal initialize():", mt5.last_error())
| |||
sys.exit(1)
| |||
info = mt5.terminal_info()
| |||
si = mt5.symbol_info(args.symbol)
| |||
if info is None or si is None:
| |||
print(f"Terminal/simbol {args.symbol} tidak ditemukan:", mt5.last_error())
| |||
mt5.shutdown()
| |||
sys.exit(1)
| |||
| |||
print(f"Terminal: {info.name} | {info.path}")
| |||
print(f"Simbol : {args.symbol} | {si.description} | digits={si.digits}")
| |||
ensure_dir(BASE)
| |||
| |||
start = dt.datetime.strptime(args.start, "%Y-%m-%d")
| |||
end = dt.datetime.now(dt.timezone.utc).replace(tzinfo=None) + dt.timedelta(days=2)
| |||
| |||
all_reports = []
| |||
for tf_name in tfs:
| |||
tf_enum = TF_MAP[tf_name]
| |||
print(f"\n=== {args.symbol} {tf_name} ===")
| |||
# probe: bar pertama & jumlah total
| |||
r0 = mt5.copy_rates_from_pos(args.symbol, tf_enum, 0, 1)
| |||
bars_total = mt5.symbol_info(args.symbol).history_bars(tf_enum, start, end) \
| |||
if hasattr(mt5.symbol_info(args.symbol), "history_bars") else -1
| |||
try:
| |||
r_first = mt5.copy_rates_range(args.symbol, tf_enum, start, end)
| |||
except Exception as e: # noqa
| |||
r_first = None
| |||
print(f" [probe gagal] {e}")
| |||
if r0 is not None and len(r0):
| |||
print(f" bar terbaru : {dt.datetime.utcfromtimestamp(int(r0[0]['time']))}")
| |||
if r_first is not None and len(r_first):
| |||
print(f" probe range : {len(r_first)} bar | "
| |||
f"mulai {dt.datetime.utcfromtimestamp(int(r_first[0]['time']))} | "
| |||
f"akhir {dt.datetime.utcfromtimestamp(int(r_first[-1]['time']))}")
| |||
full = r_first
| |||
rep = verify_continuity(full, tf_name, None)
| |||
all_reports.append(rep)
| |||
print(f" [probe] n={rep['n']} span={rep['span_years']}y "
| |||
f"anomali_gap={rep['anomaly_gaps']} status={rep['status']}")
| |||
if args.probe:
| |||
continue
| |||
out_csv, out_npz = save(args.symbol, tf_name, full)
| |||
rep.update(csv=os.path.basename(out_csv), npz=os.path.basename(out_npz))
| |||
else:
| |||
print(f" [probe] tidak ada data di range (mulai {args.start})")
| |||
| |||
mt5.shutdown()
| |||
| |||
# laporan ringkas
| |||
print("\n=== RINGKASAN KONTINUITAS ===")
| |||
for rep in all_reports:
| |||
print(f" {rep['tf']:4s} n={rep['n']:7d} span={rep['span_years']:6.2f}y "
| |||
f"{rep.get('first')} .. {rep.get('last')} | dup={rep.get('dup',0)} "
| |||
f"anom={rep.get('anomaly_gaps',0)} longest={rep.get('longest_gap_sec',0)}s "
| |||
f"status={rep.get('status')}")
| |||
| |||
if all_reports and not args.probe:
| |||
path_rep = os.path.join(BASE, "download_report.json")
| |||
with open(path_rep, "w", encoding="utf-8") as f:
| |||
json.dump({"generated": dt.datetime.utcnow().isoformat(),
| |||
"symbol": args.symbol, "reports": all_reports},
| |||
f, indent=2)
| |||
print(f"\nLaporan: {path_rep}")
| |||
print("Selesai.")
| |||
| |||
| |||
if __name__ == "__main__":
| |||
main()
|