| #!/usr/bin/env python3
|
| """Convert a tick parquet file into the raw test format:
|
| little-endian records of (int64 time_msc, double bid, double ask), 24 bytes each.
|
|
|
| usage: to_ticks.py input.parquet output.ticks [point_scale]
|
| If point_scale is given, integer price columns are divided by it (Dukascopy layout).
|
| """
|
| import os
|
| import struct
|
| import sys
|
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| from parquet_mini import read_parquet # noqa: E402
|
|
|
|
|
| def main():
|
| src, dst = sys.argv[1], sys.argv[2]
|
| scale = int(sys.argv[3]) if len(sys.argv) > 3 else 0
|
| cols, leaves, nrows = read_parquet(src)
|
| names = [l[0] for l in leaves]
|
| ts = cols['timestamp']
|
| if 'bid' in names:
|
| bid, ask = cols['bid'], cols['ask']
|
| else:
|
| bid, ask = cols['bid_price'], cols['ask_price']
|
| out = bytearray()
|
| pack = struct.Struct('<qdd').pack
|
| for i in range(len(ts)):
|
| b = bid[i]
|
| a = ask[i]
|
| if scale:
|
| b = b / scale
|
| a = a / scale
|
| out += pack(ts[i], float(b), float(a))
|
| with open(dst, 'wb') as f:
|
| f.write(out)
|
| print(dst, len(ts), 'ticks')
|
|
|
|
|
| if __name__ == '__main__':
|
| main()
|
541e992bab