forked from animatedread/Warrior_EA
f8a15c7b50
- Implemented `catalog.py` to define instrument metadata and price scaling checks. - Developed `sqx.py` for decoding SQX tick .dat files, handling scaling and volume adjustments. - Created `sqxbars.py` to decode SQX bar files, ensuring price integrity against known ranges. - Introduced `volume_gate.py` for relative volume analysis on D1 bars, facilitating backtesting strategies.
182 lines
8.2 KiB
Python
182 lines
8.2 KiB
Python
"""What each instrument IS - the one place that answers it.
|
|
|
|
Two dicts in `breadth.py` used to answer this for five symbols, keyed by names like
|
|
`USA500IDXUSD_dukascopy__the5ers`. The SQX export has since been rewritten with ONE
|
|
underscore, so every one of those keys now raises FileNotFoundError, and a second copy of
|
|
the same knowledge had started to accumulate in the seasonal screens. This module is that
|
|
knowledge, once, for all thirteen instruments the install actually carries.
|
|
|
|
THE DECIMAL SCALE
|
|
-----------------
|
|
Prices are integers and the exponent is not in the file - the trap recorded twice in this
|
|
project's notes, because a wrong scale crashes nothing and silently rescales every
|
|
ATR-normalised feature downstream. There is no longer an MT5 reference series to fit
|
|
against, so the scale is pinned by three independent checks that all have to agree, and
|
|
they do, on every file, at 1e6:
|
|
|
|
price level the implied median lands in the known real range for the instrument, and
|
|
the neighbouring exponents miss it by 10x - never a marginal call
|
|
tick size the GCD of distinct closes is a round tick at 1e6 (0.01 on BTCUSD and the
|
|
index CFDs, 0.001 on FTSE/WTI/USDJPY, 1e-5 on USDCAD) and is NOT round at
|
|
any other exponent
|
|
prior run FTSE 7246, WTI 65.4, USDCAD 1.26 reproduce the medians recorded when the
|
|
decoder was last validated against the tick-derived SP500 at corr 1.000000
|
|
|
|
1e6 is simply what this SQX install writes. It is asserted per file at load, not assumed:
|
|
`load` re-checks the price range and refuses rather than guessing.
|
|
|
|
SPREADS ARE SYNTHESISED AND THAT LIMITS WHAT THEY MAY BE USED FOR
|
|
-----------------------------------------------------------------
|
|
These are MID bar files with no ask, so `spread_bp` is a FULL relative spread in basis
|
|
points of price, applied multiplicatively by `breadth.MidBook`. Relative rather than
|
|
absolute keeps it sane across a 23-year sample and makes every statistic invariant to the
|
|
decimal scale. Values are measured where a real book once existed (EURUSD 0.45, USDJPY
|
|
0.64, SP500 1.3, XAUUSD 2.1) and rounded UP elsewhere, so the cost is pessimistic.
|
|
|
|
This is only honest for measuring a SLOPE across buckets, or for rejecting an effect that
|
|
fails to clear it. A cost assumption shifts every bucket together; it cannot manufacture or
|
|
hide a slope. Never quote a BASE level from a synthesised book as if it were tradeable, and
|
|
never use Dukascopy's own quoted spread as a cost.
|
|
"""
|
|
import os
|
|
|
|
#--- Outside the repository on purpose: a 2.1 GB .dat was committed here once and cost a
|
|
#--- history rewrite. Nothing under ROOT is ever tracked.
|
|
ROOT = 'c:/Users/admin/Documents/Workspaces/Market Data/'
|
|
HIST = ROOT + 'History/'
|
|
CACHE = ROOT + 'sqxbars/' # decoded bar arrays
|
|
ROWS = ROOT + 'edge_rows/' # scored screen output, so re-analysis never needs a re-run
|
|
DECIMALS = 6
|
|
|
|
#--- key -> (nice, asset class, full spread in bp of price, plausible close range)
|
|
#--- The range is the scale guard, not decoration: it is what makes 1e6 checkable per file.
|
|
SYMS = {
|
|
'BTCUSD_the5ers': ('BTCUSD', 'crypto', 10.0, (100, 500_000)),
|
|
'GBRIDXGBP_dukascopy_the5ers': ('FTSE_d', 'index', 1.5, (3_000, 12_000)),
|
|
'UK100_the5ers': ('FTSE_5', 'index', 1.5, (3_000, 12_000)),
|
|
'LIGHTCMDUSD_dukascopy_the5ers': ('WTI_d', 'energy', 4.0, (-50, 200)),
|
|
'XTIUSD_the5ers': ('WTI_5', 'energy', 4.0, (-50, 200)),
|
|
'SP500_the5ers': ('SP500_5', 'index', 1.3, (500, 12_000)),
|
|
'USA500IDXUSD_dukascopy_the5ers': ('SP500_d', 'index', 1.3, (500, 12_000)),
|
|
'USDCAD_dukascopy_the5ers': ('USDCAD_d', 'fx', 1.0, (0.8, 2.0)),
|
|
'USDCAD_the5ers': ('USDCAD_5', 'fx', 1.0, (0.8, 2.0)),
|
|
'USDJPY_dukascopy_the5ers': ('USDJPY_d', 'fx', 0.64, (50, 300)),
|
|
'USDJPY_the5ers': ('USDJPY_5', 'fx', 0.64, (50, 300)),
|
|
'XAUUSD_dukascopy_the5ers': ('XAUUSD_d', 'metal', 2.1, (200, 10_000)),
|
|
'XAUUSD_the5ers': ('XAUUSD_5', 'metal', 2.1, (200, 10_000)),
|
|
'@EW': ('ES_fut', 'futures', 1.4, (300, 12_000)),
|
|
'SPY_benchmark.D': ('SPY_d1', 'equity', 1.0, (10, 2_000)),
|
|
}
|
|
|
|
#--- the two symbols SQX files under a nested vendor tree rather than one flat folder
|
|
_NESTED = {'@EW': 'sq_futures/@/@EW/', 'SPY_benchmark.D': 'sq_equity/S/SPY_benchmark.D/'}
|
|
|
|
#--- same market, independent feed. A real effect must show in BOTH halves of a pair; this
|
|
#--- is the only free replication axis the install offers and it costs nothing to check.
|
|
PAIRS = [('FTSE_d', 'FTSE_5'), ('WTI_d', 'WTI_5'), ('SP500_d', 'SP500_5'),
|
|
('USDCAD_d', 'USDCAD_5'), ('USDJPY_d', 'USDJPY_5'), ('XAUUSD_d', 'XAUUSD_5')]
|
|
|
|
#--- The 15 feeds are NOT 15 markets. Each PAIRS entry is one market quoted by two vendors
|
|
#--- (correlation >= 0.999986 on five of the six), and ES_fut, SPY_d1, SP500_d and SP500_5 are
|
|
#--- four claims on the same index. Any statistic that counts feeds as independent tests
|
|
#--- overstates its own power by roughly the square root of two - so pooling across the
|
|
#--- catalog has to collapse to THIS, not to the symbol list.
|
|
MARKET = {}
|
|
for _a, _b in PAIRS:
|
|
MARKET[_a] = MARKET[_b] = _a.rsplit('_', 1)[0]
|
|
MARKET.update({'ES_fut': 'SP500', 'SPY_d1': 'SP500', 'BTCUSD': 'BTCUSD',
|
|
'FTSE_d': 'FTSE', 'FTSE_5': 'FTSE'})
|
|
MARKETS = sorted(set(MARKET.values()))
|
|
|
|
|
|
def market(sym):
|
|
"""The distinct market a NICE symbol belongs to. Unknown symbols stand alone."""
|
|
return MARKET.get(sym, sym)
|
|
|
|
|
|
NICE = {k: v[0] for k, v in SYMS.items()}
|
|
KEY = {v[0]: k for k, v in SYMS.items()}
|
|
|
|
|
|
def nice(key):
|
|
return SYMS[key][0]
|
|
|
|
|
|
def asset_class(key):
|
|
return SYMS[key][1]
|
|
|
|
|
|
def spread_bp(key):
|
|
return SYMS[key][2]
|
|
|
|
|
|
def px_range(key):
|
|
return SYMS[key][3]
|
|
|
|
|
|
def path(key, tf='M1'):
|
|
return f"{HIST}{_NESTED.get(key, key + '/')}{key}_{tf}.dat"
|
|
|
|
|
|
def available(key, tf='M1'):
|
|
return os.path.exists(path(key, tf))
|
|
|
|
|
|
def keys(cls=None):
|
|
"""Catalog keys, optionally one asset class, and only those with SOME bar file present.
|
|
|
|
Filtered on `base_tf`, not on M1: the futures tree ships M60 and the SPY benchmark ships
|
|
D1 only, and keying this on M1 silently dropped both from every screen that iterated it -
|
|
a missing instrument looks exactly like an instrument with nothing to report.
|
|
"""
|
|
out = []
|
|
for k in SYMS:
|
|
if cls is not None and asset_class(k) != cls:
|
|
continue
|
|
try:
|
|
base_tf(k)
|
|
except FileNotFoundError:
|
|
continue
|
|
out.append(k)
|
|
return out
|
|
|
|
|
|
#--- coarsest-to-finest, so `base_tf` returns the finest series a symbol actually ships.
|
|
#--- SQX names hourly bars M60 in the futures tree and H1 in the CFD tree; both mean an hour.
|
|
_BASE_ORDER = ('M1', 'M60', 'H1', 'D1')
|
|
_BASE_SEC = {'M1': 60, 'M60': 3600, 'H1': 3600, 'D1': 86400}
|
|
|
|
|
|
def base_tf(key):
|
|
"""The finest timeframe this symbol has a file for. Raises if it has none."""
|
|
for tf in _BASE_ORDER:
|
|
if available(key, tf):
|
|
return tf
|
|
raise FileNotFoundError(f"{nice(key)}: no bar file at any timeframe")
|
|
|
|
|
|
def base_sec(key):
|
|
return _BASE_SEC[base_tf(key)]
|
|
|
|
|
|
#--- catalog key -> the name the alt-data collectors use (research/altdata/*, keyed on the
|
|
#--- instrument the FUTURES contract tracks, not on our feed name). None = no alt coverage.
|
|
#--- SP500_d/SP500_5/ES_fut/SPY_d1 all map to SP500: one COT series describes the E-mini, and
|
|
#--- every one of those four is a claim on the same index.
|
|
ALT_SYMBOL = {
|
|
'USDJPY_dukascopy_the5ers': 'USDJPY', 'USDJPY_the5ers': 'USDJPY',
|
|
'USDCAD_dukascopy_the5ers': 'USDCAD', 'USDCAD_the5ers': 'USDCAD',
|
|
'XAUUSD_dukascopy_the5ers': 'XAUUSD', 'XAUUSD_the5ers': 'XAUUSD',
|
|
'SP500_the5ers': 'SP500', 'USA500IDXUSD_dukascopy_the5ers': 'SP500',
|
|
'@EW': 'SP500', 'SPY_benchmark.D': 'SP500',
|
|
'LIGHTCMDUSD_dukascopy_the5ers': 'XTIUSD', 'XTIUSD_the5ers': 'XTIUSD',
|
|
'BTCUSD_the5ers': 'BTCUSD',
|
|
'GBRIDXGBP_dukascopy_the5ers': None, 'UK100_the5ers': None, # no COT for FTSE
|
|
}
|
|
|
|
#--- instrument-specific implied-vol FRED series, where one exists for the actual underlying
|
|
IVOL_SERIES = {'XAUUSD': 'GVZCLS', 'XTIUSD': 'OVXCLS', 'SP500': 'VXDCLS'}
|
|
|
|
|
|
def alt_symbol(key):
|
|
return ALT_SYMBOL.get(key)
|