mql5-tick-compression/tests/data/fetch_samples.sh

#!/usr/bin/env bash
# Download the public tick samples used for the benchmarks in README.md and
# convert them to the raw test format (int64 time_msc, double bid, double ask).
#   tests/data/fetch_samples.sh [output_dir]        (default: tests/build/ticks)
# Requires python3 and curl; the Dukascopy files use ZSTD pages and need libzstd.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
out="${1:-$here/../build/ticks}"
mkdir -p "$out/parquet"
hf=https://huggingface.co/datasets
fetch() { # url file
  [ -s "$out/parquet/$2" ] || curl -sSL --fail --max-time 600 -o "$out/parquet/$2" "$1"
}
fetch "$hf/Henrik404/dukascopy-ticks/resolve/main/data/EURUSD/EURUSD_2023-11.parquet" duka_eurusd_2023_11.parquet
fetch "$hf/Henrik404/dukascopy-ticks/resolve/main/data/XAUUSD/XAUUSD_2023-09.parquet" duka_xauusd_2023_09.parquet
fetch "$hf/CarlosSilva1/xauusd-ticks/resolve/main/year=2026/month=05/XAUUSD-2026-05-part0003.parquet" brk_xauusd_2026_05.parquet
fetch "$hf/CarlosSilva1/us500-ticks/resolve/main/year=2025/month=02/US500-2025-02-part0001.parquet" tks_us500_2025_02.parquet
py="python3 -I -B $here/to_ticks.py"
$py "$out/parquet/duka_eurusd_2023_11.parquet" "$out/duka_eurusd_2023_11.ticks" 100000
$py "$out/parquet/duka_xauusd_2023_09.parquet" "$out/duka_xauusd_2023_09.ticks" 1000
$py "$out/parquet/brk_xauusd_2026_05.parquet" "$out/brk_xauusd_2026_05.ticks"
$py "$out/parquet/tks_us500_2025_02.parquet" "$out/tks_us500_2025_02.ticks"