forked from mnbvc188199/Warrior_EA
f8a15c7b50
- Implemented `catalog.py` to define instrument metadata and price scaling checks. - Developed `sqx.py` for decoding SQX tick .dat files, handling scaling and volume adjustments. - Created `sqxbars.py` to decode SQX bar files, ensuring price integrity against known ranges. - Introduced `volume_gate.py` for relative volume analysis on D1 bars, facilitating backtesting strategies.
182 lines
8.2 KiB
Python
182 lines
8.2 KiB
Python
"""What each instrument IS - the one place that answers it.
| |
| |
Two dicts in `breadth.py` used to answer this for five symbols, keyed by names like
| |
`USA500IDXUSD_dukascopy__the5ers`. The SQX export has since been rewritten with ONE
| |
underscore, so every one of those keys now raises FileNotFoundError, and a second copy of
| |
the same knowledge had started to accumulate in the seasonal screens. This module is that
| |
knowledge, once, for all thirteen instruments the install actually carries.
| |
| |
THE DECIMAL SCALE
| |
-----------------
| |
Prices are integers and the exponent is not in the file - the trap recorded twice in this
| |
project's notes, because a wrong scale crashes nothing and silently rescales every
| |
ATR-normalised feature downstream. There is no longer an MT5 reference series to fit
| |
against, so the scale is pinned by three independent checks that all have to agree, and
| |
they do, on every file, at 1e6:
| |
| |
price level the implied median lands in the known real range for the instrument, and
| |
the neighbouring exponents miss it by 10x - never a marginal call
| |
tick size the GCD of distinct closes is a round tick at 1e6 (0.01 on BTCUSD and the
| |
index CFDs, 0.001 on FTSE/WTI/USDJPY, 1e-5 on USDCAD) and is NOT round at
| |
any other exponent
| |
prior run FTSE 7246, WTI 65.4, USDCAD 1.26 reproduce the medians recorded when the
| |
decoder was last validated against the tick-derived SP500 at corr 1.000000
| |
| |
1e6 is simply what this SQX install writes. It is asserted per file at load, not assumed:
| |
`load` re-checks the price range and refuses rather than guessing.
| |
| |
SPREADS ARE SYNTHESISED AND THAT LIMITS WHAT THEY MAY BE USED FOR
| |
-----------------------------------------------------------------
| |
These are MID bar files with no ask, so `spread_bp` is a FULL relative spread in basis
| |
points of price, applied multiplicatively by `breadth.MidBook`. Relative rather than
| |
absolute keeps it sane across a 23-year sample and makes every statistic invariant to the
| |
decimal scale. Values are measured where a real book once existed (EURUSD 0.45, USDJPY
| |
0.64, SP500 1.3, XAUUSD 2.1) and rounded UP elsewhere, so the cost is pessimistic.
| |
| |
This is only honest for measuring a SLOPE across buckets, or for rejecting an effect that
| |
fails to clear it. A cost assumption shifts every bucket together; it cannot manufacture or
| |
hide a slope. Never quote a BASE level from a synthesised book as if it were tradeable, and
| |
never use Dukascopy's own quoted spread as a cost.
| |
"""
| |
import os
| |
| |
#--- Outside the repository on purpose: a 2.1 GB .dat was committed here once and cost a
| |
#--- history rewrite. Nothing under ROOT is ever tracked.
| |
ROOT = 'c:/Users/admin/Documents/Workspaces/Market Data/'
| |
HIST = ROOT + 'History/'
| |
CACHE = ROOT + 'sqxbars/' # decoded bar arrays
| |
ROWS = ROOT + 'edge_rows/' # scored screen output, so re-analysis never needs a re-run
| |
DECIMALS = 6
| |
| |
#--- key -> (nice, asset class, full spread in bp of price, plausible close range)
| |
#--- The range is the scale guard, not decoration: it is what makes 1e6 checkable per file.
| |
SYMS = {
| |
'BTCUSD_the5ers': ('BTCUSD', 'crypto', 10.0, (100, 500_000)),
| |
'GBRIDXGBP_dukascopy_the5ers': ('FTSE_d', 'index', 1.5, (3_000, 12_000)),
| |
'UK100_the5ers': ('FTSE_5', 'index', 1.5, (3_000, 12_000)),
| |
'LIGHTCMDUSD_dukascopy_the5ers': ('WTI_d', 'energy', 4.0, (-50, 200)),
| |
'XTIUSD_the5ers': ('WTI_5', 'energy', 4.0, (-50, 200)),
| |
'SP500_the5ers': ('SP500_5', 'index', 1.3, (500, 12_000)),
| |
'USA500IDXUSD_dukascopy_the5ers': ('SP500_d', 'index', 1.3, (500, 12_000)),
| |
'USDCAD_dukascopy_the5ers': ('USDCAD_d', 'fx', 1.0, (0.8, 2.0)),
| |
'USDCAD_the5ers': ('USDCAD_5', 'fx', 1.0, (0.8, 2.0)),
| |
'USDJPY_dukascopy_the5ers': ('USDJPY_d', 'fx', 0.64, (50, 300)),
| |
'USDJPY_the5ers': ('USDJPY_5', 'fx', 0.64, (50, 300)),
| |
'XAUUSD_dukascopy_the5ers': ('XAUUSD_d', 'metal', 2.1, (200, 10_000)),
| |
'XAUUSD_the5ers': ('XAUUSD_5', 'metal', 2.1, (200, 10_000)),
| |
'@EW': ('ES_fut', 'futures', 1.4, (300, 12_000)),
| |
'SPY_benchmark.D': ('SPY_d1', 'equity', 1.0, (10, 2_000)),
| |
}
| |
| |
#--- the two symbols SQX files under a nested vendor tree rather than one flat folder
| |
_NESTED = {'@EW': 'sq_futures/@/@EW/', 'SPY_benchmark.D': 'sq_equity/S/SPY_benchmark.D/'}
| |
| |
#--- same market, independent feed. A real effect must show in BOTH halves of a pair; this
| |
#--- is the only free replication axis the install offers and it costs nothing to check.
| |
PAIRS = [('FTSE_d', 'FTSE_5'), ('WTI_d', 'WTI_5'), ('SP500_d', 'SP500_5'),
| |
('USDCAD_d', 'USDCAD_5'), ('USDJPY_d', 'USDJPY_5'), ('XAUUSD_d', 'XAUUSD_5')]
| |
| |
#--- The 15 feeds are NOT 15 markets. Each PAIRS entry is one market quoted by two vendors
| |
#--- (correlation >= 0.999986 on five of the six), and ES_fut, SPY_d1, SP500_d and SP500_5 are
| |
#--- four claims on the same index. Any statistic that counts feeds as independent tests
| |
#--- overstates its own power by roughly the square root of two - so pooling across the
| |
#--- catalog has to collapse to THIS, not to the symbol list.
| |
MARKET = {}
| |
for _a, _b in PAIRS:
| |
MARKET[_a] = MARKET[_b] = _a.rsplit('_', 1)[0]
| |
MARKET.update({'ES_fut': 'SP500', 'SPY_d1': 'SP500', 'BTCUSD': 'BTCUSD',
| |
'FTSE_d': 'FTSE', 'FTSE_5': 'FTSE'})
| |
MARKETS = sorted(set(MARKET.values()))
| |
| |
| |
def market(sym):
| |
"""The distinct market a NICE symbol belongs to. Unknown symbols stand alone."""
| |
return MARKET.get(sym, sym)
| |
| |
| |
NICE = {k: v[0] for k, v in SYMS.items()}
| |
KEY = {v[0]: k for k, v in SYMS.items()}
| |
| |
| |
def nice(key):
| |
return SYMS[key][0]
| |
| |
| |
def asset_class(key):
| |
return SYMS[key][1]
| |
| |
| |
def spread_bp(key):
| |
return SYMS[key][2]
| |
| |
| |
def px_range(key):
| |
return SYMS[key][3]
| |
| |
| |
def path(key, tf='M1'):
| |
return f"{HIST}{_NESTED.get(key, key + '/')}{key}_{tf}.dat"
| |
| |
| |
def available(key, tf='M1'):
| |
return os.path.exists(path(key, tf))
| |
| |
| |
def keys(cls=None):
| |
"""Catalog keys, optionally one asset class, and only those with SOME bar file present.
| |
| |
Filtered on `base_tf`, not on M1: the futures tree ships M60 and the SPY benchmark ships
| |
D1 only, and keying this on M1 silently dropped both from every screen that iterated it -
| |
a missing instrument looks exactly like an instrument with nothing to report.
| |
"""
| |
out = []
| |
for k in SYMS:
| |
if cls is not None and asset_class(k) != cls:
| |
continue
| |
try:
| |
base_tf(k)
| |
except FileNotFoundError:
| |
continue
| |
out.append(k)
| |
return out
| |
| |
| |
#--- coarsest-to-finest, so `base_tf` returns the finest series a symbol actually ships.
| |
#--- SQX names hourly bars M60 in the futures tree and H1 in the CFD tree; both mean an hour.
| |
_BASE_ORDER = ('M1', 'M60', 'H1', 'D1')
| |
_BASE_SEC = {'M1': 60, 'M60': 3600, 'H1': 3600, 'D1': 86400}
| |
| |
| |
def base_tf(key):
| |
"""The finest timeframe this symbol has a file for. Raises if it has none."""
| |
for tf in _BASE_ORDER:
| |
if available(key, tf):
| |
return tf
| |
raise FileNotFoundError(f"{nice(key)}: no bar file at any timeframe")
| |
| |
| |
def base_sec(key):
| |
return _BASE_SEC[base_tf(key)]
| |
| |
| |
#--- catalog key -> the name the alt-data collectors use (research/altdata/*, keyed on the
| |
#--- instrument the FUTURES contract tracks, not on our feed name). None = no alt coverage.
| |
#--- SP500_d/SP500_5/ES_fut/SPY_d1 all map to SP500: one COT series describes the E-mini, and
| |
#--- every one of those four is a claim on the same index.
| |
ALT_SYMBOL = {
| |
'USDJPY_dukascopy_the5ers': 'USDJPY', 'USDJPY_the5ers': 'USDJPY',
| |
'USDCAD_dukascopy_the5ers': 'USDCAD', 'USDCAD_the5ers': 'USDCAD',
| |
'XAUUSD_dukascopy_the5ers': 'XAUUSD', 'XAUUSD_the5ers': 'XAUUSD',
| |
'SP500_the5ers': 'SP500', 'USA500IDXUSD_dukascopy_the5ers': 'SP500',
| |
'@EW': 'SP500', 'SPY_benchmark.D': 'SP500',
| |
'LIGHTCMDUSD_dukascopy_the5ers': 'XTIUSD', 'XTIUSD_the5ers': 'XTIUSD',
| |
'BTCUSD_the5ers': 'BTCUSD',
| |
'GBRIDXGBP_dukascopy_the5ers': None, 'UK100_the5ers': None, # no COT for FTSE
| |
}
| |
| |
#--- instrument-specific implied-vol FRED series, where one exists for the actual underlying
| |
IVOL_SERIES = {'XAUUSD': 'GVZCLS', 'XTIUSD': 'OVXCLS', 'SP500': 'VXDCLS'}
| |
| |
| |
def alt_symbol(key):
| |
return ALT_SYMBOL.get(key)
|