| """What each instrument IS - the one place that answers it.
|
|
|
| Two dicts in `breadth.py` used to answer this for five symbols, keyed by names like
|
| `USA500IDXUSD_dukascopy__the5ers`. The SQX export has since been rewritten with ONE
|
| underscore, so every one of those keys now raises FileNotFoundError, and a second copy of
|
| the same knowledge had started to accumulate in the seasonal screens. This module is that
|
| knowledge, once, for all thirteen instruments the install actually carries.
|
|
|
| THE DECIMAL SCALE
|
| -----------------
|
| Prices are integers and the exponent is not in the file - the trap recorded twice in this
|
| project's notes, because a wrong scale crashes nothing and silently rescales every
|
| ATR-normalised feature downstream. There is no longer an MT5 reference series to fit
|
| against, so the scale is pinned by three independent checks that all have to agree, and
|
| they do, on every file, at 1e6:
|
|
|
| price level the implied median lands in the known real range for the instrument, and
|
| the neighbouring exponents miss it by 10x - never a marginal call
|
| tick size the GCD of distinct closes is a round tick at 1e6 (0.01 on BTCUSD and the
|
| index CFDs, 0.001 on FTSE/WTI/USDJPY, 1e-5 on USDCAD) and is NOT round at
|
| any other exponent
|
| prior run FTSE 7246, WTI 65.4, USDCAD 1.26 reproduce the medians recorded when the
|
| decoder was last validated against the tick-derived SP500 at corr 1.000000
|
|
|
| 1e6 is simply what this SQX install writes. It is asserted per file at load, not assumed:
|
| `load` re-checks the price range and refuses rather than guessing.
|
|
|
| SPREADS ARE SYNTHESISED AND THAT LIMITS WHAT THEY MAY BE USED FOR
|
| -----------------------------------------------------------------
|
| These are MID bar files with no ask, so `spread_bp` is a FULL relative spread in basis
|
| points of price, applied multiplicatively by `breadth.MidBook`. Relative rather than
|
| absolute keeps it sane across a 23-year sample and makes every statistic invariant to the
|
| decimal scale. Values are measured where a real book once existed (EURUSD 0.45, USDJPY
|
| 0.64, SP500 1.3, XAUUSD 2.1) and rounded UP elsewhere, so the cost is pessimistic.
|
|
|
| This is only honest for measuring a SLOPE across buckets, or for rejecting an effect that
|
| fails to clear it. A cost assumption shifts every bucket together; it cannot manufacture or
|
| hide a slope. Never quote a BASE level from a synthesised book as if it were tradeable, and
|
| never use Dukascopy's own quoted spread as a cost.
|
| """
|
| import os
|
|
|
| #--- Outside the repository on purpose: a 2.1 GB .dat was committed here once and cost a
|
| #--- history rewrite. Nothing under ROOT is ever tracked.
|
| ROOT = 'c:/Users/admin/Documents/Workspaces/Market Data/'
|
| HIST = ROOT + 'History/'
|
| CACHE = ROOT + 'sqxbars/' # decoded bar arrays
|
| ROWS = ROOT + 'edge_rows/' # scored screen output, so re-analysis never needs a re-run
|
| DECIMALS = 6
|
|
|
| #--- key -> (nice, asset class, full spread in bp of price, plausible close range)
|
| #--- The range is the scale guard, not decoration: it is what makes 1e6 checkable per file.
|
| SYMS = {
|
| 'BTCUSD_the5ers': ('BTCUSD', 'crypto', 10.0, (100, 500_000)),
|
| 'GBRIDXGBP_dukascopy_the5ers': ('FTSE_d', 'index', 1.5, (3_000, 12_000)),
|
| 'UK100_the5ers': ('FTSE_5', 'index', 1.5, (3_000, 12_000)),
|
| 'LIGHTCMDUSD_dukascopy_the5ers': ('WTI_d', 'energy', 4.0, (-50, 200)),
|
| 'XTIUSD_the5ers': ('WTI_5', 'energy', 4.0, (-50, 200)),
|
| 'SP500_the5ers': ('SP500_5', 'index', 1.3, (500, 12_000)),
|
| 'USA500IDXUSD_dukascopy_the5ers': ('SP500_d', 'index', 1.3, (500, 12_000)),
|
| 'USDCAD_dukascopy_the5ers': ('USDCAD_d', 'fx', 1.0, (0.8, 2.0)),
|
| 'USDCAD_the5ers': ('USDCAD_5', 'fx', 1.0, (0.8, 2.0)),
|
| 'USDJPY_dukascopy_the5ers': ('USDJPY_d', 'fx', 0.64, (50, 300)),
|
| 'USDJPY_the5ers': ('USDJPY_5', 'fx', 0.64, (50, 300)),
|
| 'XAUUSD_dukascopy_the5ers': ('XAUUSD_d', 'metal', 2.1, (200, 10_000)),
|
| 'XAUUSD_the5ers': ('XAUUSD_5', 'metal', 2.1, (200, 10_000)),
|
| '@EW': ('ES_fut', 'futures', 1.4, (300, 12_000)),
|
| 'SPY_benchmark.D': ('SPY_d1', 'equity', 1.0, (10, 2_000)),
|
| }
|
|
|
| #--- the two symbols SQX files under a nested vendor tree rather than one flat folder
|
| _NESTED = {'@EW': 'sq_futures/@/@EW/', 'SPY_benchmark.D': 'sq_equity/S/SPY_benchmark.D/'}
|
|
|
| #--- same market, independent feed. A real effect must show in BOTH halves of a pair; this
|
| #--- is the only free replication axis the install offers and it costs nothing to check.
|
| PAIRS = [('FTSE_d', 'FTSE_5'), ('WTI_d', 'WTI_5'), ('SP500_d', 'SP500_5'),
|
| ('USDCAD_d', 'USDCAD_5'), ('USDJPY_d', 'USDJPY_5'), ('XAUUSD_d', 'XAUUSD_5')]
|
|
|
| #--- The 15 feeds are NOT 15 markets. Each PAIRS entry is one market quoted by two vendors
|
| #--- (correlation >= 0.999986 on five of the six), and ES_fut, SPY_d1, SP500_d and SP500_5 are
|
| #--- four claims on the same index. Any statistic that counts feeds as independent tests
|
| #--- overstates its own power by roughly the square root of two - so pooling across the
|
| #--- catalog has to collapse to THIS, not to the symbol list.
|
| MARKET = {}
|
| for _a, _b in PAIRS:
|
| MARKET[_a] = MARKET[_b] = _a.rsplit('_', 1)[0]
|
| MARKET.update({'ES_fut': 'SP500', 'SPY_d1': 'SP500', 'BTCUSD': 'BTCUSD',
|
| 'FTSE_d': 'FTSE', 'FTSE_5': 'FTSE'})
|
| MARKETS = sorted(set(MARKET.values()))
|
|
|
|
|
| def market(sym):
|
| """The distinct market a NICE symbol belongs to. Unknown symbols stand alone."""
|
| return MARKET.get(sym, sym)
|
|
|
|
|
| NICE = {k: v[0] for k, v in SYMS.items()}
|
| KEY = {v[0]: k for k, v in SYMS.items()}
|
|
|
|
|
| def nice(key):
|
| return SYMS[key][0]
|
|
|
|
|
| def asset_class(key):
|
| return SYMS[key][1]
|
|
|
|
|
| def spread_bp(key):
|
| return SYMS[key][2]
|
|
|
|
|
| def px_range(key):
|
| return SYMS[key][3]
|
|
|
|
|
| def path(key, tf='M1'):
|
| return f"{HIST}{_NESTED.get(key, key + '/')}{key}_{tf}.dat"
|
|
|
|
|
| def available(key, tf='M1'):
|
| return os.path.exists(path(key, tf))
|
|
|
|
|
| def keys(cls=None):
|
| """Catalog keys, optionally one asset class, and only those with SOME bar file present.
|
|
|
| Filtered on `base_tf`, not on M1: the futures tree ships M60 and the SPY benchmark ships
|
| D1 only, and keying this on M1 silently dropped both from every screen that iterated it -
|
| a missing instrument looks exactly like an instrument with nothing to report.
|
| """
|
| out = []
|
| for k in SYMS:
|
| if cls is not None and asset_class(k) != cls:
|
| continue
|
| try:
|
| base_tf(k)
|
| except FileNotFoundError:
|
| continue
|
| out.append(k)
|
| return out
|
|
|
|
|
| #--- coarsest-to-finest, so `base_tf` returns the finest series a symbol actually ships.
|
| #--- SQX names hourly bars M60 in the futures tree and H1 in the CFD tree; both mean an hour.
|
| _BASE_ORDER = ('M1', 'M60', 'H1', 'D1')
|
| _BASE_SEC = {'M1': 60, 'M60': 3600, 'H1': 3600, 'D1': 86400}
|
|
|
|
|
| def base_tf(key):
|
| """The finest timeframe this symbol has a file for. Raises if it has none."""
|
| for tf in _BASE_ORDER:
|
| if available(key, tf):
|
| return tf
|
| raise FileNotFoundError(f"{nice(key)}: no bar file at any timeframe")
|
|
|
|
|
| def base_sec(key):
|
| return _BASE_SEC[base_tf(key)]
|
|
|
|
|
| #--- catalog key -> the name the alt-data collectors use (research/altdata/*, keyed on the
|
| #--- instrument the FUTURES contract tracks, not on our feed name). None = no alt coverage.
|
| #--- SP500_d/SP500_5/ES_fut/SPY_d1 all map to SP500: one COT series describes the E-mini, and
|
| #--- every one of those four is a claim on the same index.
|
| ALT_SYMBOL = {
|
| 'USDJPY_dukascopy_the5ers': 'USDJPY', 'USDJPY_the5ers': 'USDJPY',
|
| 'USDCAD_dukascopy_the5ers': 'USDCAD', 'USDCAD_the5ers': 'USDCAD',
|
| 'XAUUSD_dukascopy_the5ers': 'XAUUSD', 'XAUUSD_the5ers': 'XAUUSD',
|
| 'SP500_the5ers': 'SP500', 'USA500IDXUSD_dukascopy_the5ers': 'SP500',
|
| '@EW': 'SP500', 'SPY_benchmark.D': 'SP500',
|
| 'LIGHTCMDUSD_dukascopy_the5ers': 'XTIUSD', 'XTIUSD_the5ers': 'XTIUSD',
|
| 'BTCUSD_the5ers': 'BTCUSD',
|
| 'GBRIDXGBP_dukascopy_the5ers': None, 'UK100_the5ers': None, # no COT for FTSE
|
| }
|
|
|
| #--- instrument-specific implied-vol FRED series, where one exists for the actual underlying
|
| IVOL_SERIES = {'XAUUSD': 'GVZCLS', 'XTIUSD': 'OVXCLS', 'SP500': 'VXDCLS'}
|
|
|
|
|
| def alt_symbol(key):
|
| return ALT_SYMBOL.get(key)
|
f8a15c7b50