230 lines
9 KiB
Python
230 lines
9 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
Algo Forge - Tahap 4 (ML Hibrida): download_bars.py
|
||
|
|
====================================================
|
||
|
|
Unduh data OHLCV HISTORIS PANJANG (±20 tahun) dari terminal MetaTrader 5
|
||
|
|
(HFM Demo, simbol XAUUSD) secara chunked per tahun, lalu VERIFIKASI
|
||
|
|
kontinuitas (gap detection + sanity OHLC) dan simpan ke:
|
||
|
|
|
||
|
|
MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.csv (arsip teks, audit)
|
||
|
|
MQL5\\Files\\AlgoForge\\Data\\XAUUSD_<TF>.npz (arrays numpy, cepat)
|
||
|
|
|
||
|
|
TIDAK menyentuh Files\\SniperGold_ML (cache baseline Fase 3 = deploy freeze).
|
||
|
|
|
||
|
|
Cara pakai:
|
||
|
|
python download_bars.py [--symbol XAUUSD] [--tf M15,H1,H4,D1] [--force] [--probe]
|
||
|
|
|
||
|
|
--probe : hanya lapor ketersediaan (bar count, rentang tanggal) tanpa unduh.
|
||
|
|
--force : unduh ulang walau file sudah ada.
|
||
|
|
--start : ISO date awal (default: 2000-01-01 -> terminal kembalikan yang ada).
|
||
|
|
|
||
|
|
Verifikasi kontinuitas:
|
||
|
|
- urutan waktu strict naik, tanpa duplikat
|
||
|
|
- step antar bar dibandingkan dengan step TF; gap <= 3x step dianggap normal
|
||
|
|
- gap 3x..4 hari = weekend/holiday (dicatat, bukan error)
|
||
|
|
- gap > 4 hari = ANOMALI (dilaporkan; total & terpanjang)
|
||
|
|
- sanity OHLC: high >= max(o,c), low <= min(o,c), volume >= 0
|
||
|
|
"""
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import json
|
||
|
|
import argparse
|
||
|
|
import datetime as dt
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import MetaTrader5 as mt5
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------
|
||
|
|
BASE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
|
|
"..", "..", "..", "Files", "AlgoForge", "Data")
|
||
|
|
BASE = os.path.normpath(BASE)
|
||
|
|
|
||
|
|
TF_MAP = {
|
||
|
|
"M15": mt5.TIMEFRAME_M15,
|
||
|
|
"H1": mt5.TIMEFRAME_H1,
|
||
|
|
"H4": mt5.TIMEFRAME_H4,
|
||
|
|
"D1": mt5.TIMEFRAME_D1,
|
||
|
|
}
|
||
|
|
STEP = {"M15": 15 * 60, "H1": 3600, "H4": 4 * 3600, "D1": 86400}
|
||
|
|
GAP_ANOMALY = 4 * 86400 # > 4 hari = anomali
|
||
|
|
GAP_WEEKEND = 3 * 3600 # batas "normal" (3x step utk M15)
|
||
|
|
|
||
|
|
DEFAULT_TFS = ["M15", "H1", "H4", "D1"]
|
||
|
|
|
||
|
|
|
||
|
|
def ensure_dir(path):
|
||
|
|
os.makedirs(path, exist_ok=True)
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_range_chunked(symbol, tf_enum, start, end, step_sec):
|
||
|
|
"""Unduh [start, end] per-chunk ~ 2 tahun, gabung, urutkan, dedup."""
|
||
|
|
chunks = []
|
||
|
|
cur = start
|
||
|
|
while cur < end:
|
||
|
|
nxt = min(cur + dt.timedelta(days=730), end)
|
||
|
|
rates = mt5.copy_rates_range(symbol, tf_enum, cur, nxt)
|
||
|
|
if rates is None or len(rates) == 0:
|
||
|
|
err = mt5.last_error()
|
||
|
|
print(f" [chunk {cur.date()}..{nxt.date()}] kosong ({err})")
|
||
|
|
else:
|
||
|
|
chunks.append(rates)
|
||
|
|
print(f" [chunk {cur.date()}..{nxt.date()}] {len(rates)} bar")
|
||
|
|
cur = nxt
|
||
|
|
if not chunks:
|
||
|
|
return None
|
||
|
|
allr = np.concatenate(chunks)
|
||
|
|
# urutkan & dedup (jaga-jaga overlap chunk)
|
||
|
|
allr = allr[np.argsort(allr["time"], kind="stable")]
|
||
|
|
_, idx = np.unique(allr["time"], return_index=True)
|
||
|
|
allr = allr[idx]
|
||
|
|
return allr
|
||
|
|
|
||
|
|
|
||
|
|
def verify_continuity(rates, tf_name, out_report):
|
||
|
|
"""Cek urutan, duplikat, gap, sanity OHLC. Kembali dict laporan."""
|
||
|
|
t = rates["time"].astype(np.int64)
|
||
|
|
step = STEP[tf_name]
|
||
|
|
rep = {"tf": tf_name, "n": int(len(rates))}
|
||
|
|
if len(rates) == 0:
|
||
|
|
rep.update(status="EMPTY")
|
||
|
|
return rep
|
||
|
|
|
||
|
|
dup = int(np.sum(np.diff(t) == 0))
|
||
|
|
not_inc = int(np.sum(np.diff(t) < 0))
|
||
|
|
d = np.diff(t)
|
||
|
|
gaps = d[d > step * 3] # lebih dari 3x step
|
||
|
|
normal_gaps = int(np.sum((d > step) & (d <= step * 3)))
|
||
|
|
weekend_gaps = int(np.sum((d > step * 3) & (d <= GAP_ANOMALY)))
|
||
|
|
anomaly_gaps = int(np.sum(d > GAP_ANOMALY))
|
||
|
|
longest = int(d.max()) if len(d) else 0
|
||
|
|
|
||
|
|
# sanity OHLC
|
||
|
|
bad_hl = int(np.sum(rates["high"] < np.maximum(rates["open"], rates["close"])))
|
||
|
|
bad_low = int(np.sum(rates["low"] > np.minimum(rates["open"], rates["close"])))
|
||
|
|
bad_vol = int(np.sum(rates["tick_volume"] < 0))
|
||
|
|
|
||
|
|
first = dt.datetime.utcfromtimestamp(int(t[0])).strftime("%Y-%m-%d %H:%M")
|
||
|
|
last = dt.datetime.utcfromtimestamp(int(t[-1])).strftime("%Y-%m-%d %H:%M")
|
||
|
|
span_years = (t[-1] - t[0]) / (365.25 * 86400)
|
||
|
|
|
||
|
|
rep.update(
|
||
|
|
status="OK",
|
||
|
|
first=first, last=last,
|
||
|
|
span_years=round(float(span_years), 2),
|
||
|
|
dup=dup, not_increasing=not_inc,
|
||
|
|
normal_gaps=normal_gaps, weekend_gaps=weekend_gaps,
|
||
|
|
anomaly_gaps=anomaly_gaps, longest_gap_sec=int(longest),
|
||
|
|
bad_hl=bad_hl, bad_low=bad_low, bad_vol=bad_vol,
|
||
|
|
)
|
||
|
|
if dup or not_inc or anomaly_gaps or bad_hl or bad_low or bad_vol:
|
||
|
|
rep["status"] = "WARN"
|
||
|
|
return rep
|
||
|
|
|
||
|
|
|
||
|
|
def save(symbol, tf_name, rates):
|
||
|
|
path_csv = os.path.join(BASE, f"{symbol}_{tf_name}.csv")
|
||
|
|
path_npz = os.path.join(BASE, f"{symbol}_{tf_name}.npz")
|
||
|
|
with open(path_csv, "w", encoding="utf-8") as f:
|
||
|
|
f.write("time,open,high,low,close,tick_volume,spread\n")
|
||
|
|
for r in rates:
|
||
|
|
tt = dt.datetime.utcfromtimestamp(int(r["time"])).strftime("%Y-%m-%d %H:%M:%S")
|
||
|
|
f.write(f"{tt},{r['open']:.5f},{r['high']:.5f},{r['low']:.5f},"
|
||
|
|
f"{r['close']:.5f},{int(r['tick_volume'])},{int(r['spread'])}\n")
|
||
|
|
np.savez_compressed(path_npz,
|
||
|
|
time=rates["time"].astype(np.int64),
|
||
|
|
open=rates["open"].astype(np.float64),
|
||
|
|
high=rates["high"].astype(np.float64),
|
||
|
|
low=rates["low"].astype(np.float64),
|
||
|
|
close=rates["close"].astype(np.float64),
|
||
|
|
tick_volume=rates["tick_volume"].astype(np.int64),
|
||
|
|
spread=rates["spread"].astype(np.int64))
|
||
|
|
print(f" -> {path_csv} ({len(rates)} bar)")
|
||
|
|
print(f" -> {path_npz}")
|
||
|
|
return path_csv, path_npz
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
ap.add_argument("--symbol", default="XAUUSD")
|
||
|
|
ap.add_argument("--tf", default=",".join(DEFAULT_TFS))
|
||
|
|
ap.add_argument("--start", default="2000-01-01")
|
||
|
|
ap.add_argument("--force", action="store_true")
|
||
|
|
ap.add_argument("--probe", action="store_true")
|
||
|
|
args = ap.parse_args()
|
||
|
|
|
||
|
|
tfs = [x.strip().upper() for x in args.tf.split(",") if x.strip()]
|
||
|
|
for x in tfs:
|
||
|
|
if x not in TF_MAP:
|
||
|
|
print(f"TF tidak dikenal: {x} (pilih {list(TF_MAP)})")
|
||
|
|
sys.exit(2)
|
||
|
|
|
||
|
|
if not mt5.initialize():
|
||
|
|
print("Gagal initialize():", mt5.last_error())
|
||
|
|
sys.exit(1)
|
||
|
|
info = mt5.terminal_info()
|
||
|
|
si = mt5.symbol_info(args.symbol)
|
||
|
|
if info is None or si is None:
|
||
|
|
print(f"Terminal/simbol {args.symbol} tidak ditemukan:", mt5.last_error())
|
||
|
|
mt5.shutdown()
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
print(f"Terminal: {info.name} | {info.path}")
|
||
|
|
print(f"Simbol : {args.symbol} | {si.description} | digits={si.digits}")
|
||
|
|
ensure_dir(BASE)
|
||
|
|
|
||
|
|
start = dt.datetime.strptime(args.start, "%Y-%m-%d")
|
||
|
|
end = dt.datetime.now(dt.timezone.utc).replace(tzinfo=None) + dt.timedelta(days=2)
|
||
|
|
|
||
|
|
all_reports = []
|
||
|
|
for tf_name in tfs:
|
||
|
|
tf_enum = TF_MAP[tf_name]
|
||
|
|
print(f"\n=== {args.symbol} {tf_name} ===")
|
||
|
|
# probe: bar pertama & jumlah total
|
||
|
|
r0 = mt5.copy_rates_from_pos(args.symbol, tf_enum, 0, 1)
|
||
|
|
bars_total = mt5.symbol_info(args.symbol).history_bars(tf_enum, start, end) \
|
||
|
|
if hasattr(mt5.symbol_info(args.symbol), "history_bars") else -1
|
||
|
|
try:
|
||
|
|
r_first = mt5.copy_rates_range(args.symbol, tf_enum, start, end)
|
||
|
|
except Exception as e: # noqa
|
||
|
|
r_first = None
|
||
|
|
print(f" [probe gagal] {e}")
|
||
|
|
if r0 is not None and len(r0):
|
||
|
|
print(f" bar terbaru : {dt.datetime.utcfromtimestamp(int(r0[0]['time']))}")
|
||
|
|
if r_first is not None and len(r_first):
|
||
|
|
print(f" probe range : {len(r_first)} bar | "
|
||
|
|
f"mulai {dt.datetime.utcfromtimestamp(int(r_first[0]['time']))} | "
|
||
|
|
f"akhir {dt.datetime.utcfromtimestamp(int(r_first[-1]['time']))}")
|
||
|
|
full = r_first
|
||
|
|
rep = verify_continuity(full, tf_name, None)
|
||
|
|
all_reports.append(rep)
|
||
|
|
print(f" [probe] n={rep['n']} span={rep['span_years']}y "
|
||
|
|
f"anomali_gap={rep['anomaly_gaps']} status={rep['status']}")
|
||
|
|
if args.probe:
|
||
|
|
continue
|
||
|
|
out_csv, out_npz = save(args.symbol, tf_name, full)
|
||
|
|
rep.update(csv=os.path.basename(out_csv), npz=os.path.basename(out_npz))
|
||
|
|
else:
|
||
|
|
print(f" [probe] tidak ada data di range (mulai {args.start})")
|
||
|
|
|
||
|
|
mt5.shutdown()
|
||
|
|
|
||
|
|
# laporan ringkas
|
||
|
|
print("\n=== RINGKASAN KONTINUITAS ===")
|
||
|
|
for rep in all_reports:
|
||
|
|
print(f" {rep['tf']:4s} n={rep['n']:7d} span={rep['span_years']:6.2f}y "
|
||
|
|
f"{rep.get('first')} .. {rep.get('last')} | dup={rep.get('dup',0)} "
|
||
|
|
f"anom={rep.get('anomaly_gaps',0)} longest={rep.get('longest_gap_sec',0)}s "
|
||
|
|
f"status={rep.get('status')}")
|
||
|
|
|
||
|
|
if all_reports and not args.probe:
|
||
|
|
path_rep = os.path.join(BASE, "download_report.json")
|
||
|
|
with open(path_rep, "w", encoding="utf-8") as f:
|
||
|
|
json.dump({"generated": dt.datetime.utcnow().isoformat(),
|
||
|
|
"symbol": args.symbol, "reports": all_reports},
|
||
|
|
f, indent=2)
|
||
|
|
print(f"\nLaporan: {path_rep}")
|
||
|
|
print("Selesai.")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|