okWarrior_EA/research/sqx_decode/catalog.py

"""What each instrument IS - the one place that answers it.

Two dicts in `breadth.py` used to answer this for five symbols, keyed by names like
`USA500IDXUSD_dukascopy__the5ers`. The SQX export has since been rewritten with ONE
underscore, so every one of those keys now raises FileNotFoundError, and a second copy of
the same knowledge had started to accumulate in the seasonal screens. This module is that
knowledge, once, for all thirteen instruments the install actually carries.

THE DECIMAL SCALE
-----------------
Prices are integers and the exponent is not in the file - the trap recorded twice in this
project's notes, because a wrong scale crashes nothing and silently rescales every
ATR-normalised feature downstream. There is no longer an MT5 reference series to fit
against, so the scale is pinned by three independent checks that all have to agree, and
they do, on every file, at 1e6:

  price level   the implied median lands in the known real range for the instrument, and
                the neighbouring exponents miss it by 10x - never a marginal call
  tick size     the GCD of distinct closes is a round tick at 1e6 (0.01 on BTCUSD and the
                index CFDs, 0.001 on FTSE/WTI/USDJPY, 1e-5 on USDCAD) and is NOT round at
                any other exponent
  prior run     FTSE 7246, WTI 65.4, USDCAD 1.26 reproduce the medians recorded when the
                decoder was last validated against the tick-derived SP500 at corr 1.000000

1e6 is simply what this SQX install writes. It is asserted per file at load, not assumed:
`load` re-checks the price range and refuses rather than guessing.

SPREADS ARE SYNTHESISED AND THAT LIMITS WHAT THEY MAY BE USED FOR
-----------------------------------------------------------------
These are MID bar files with no ask, so `spread_bp` is a FULL relative spread in basis
points of price, applied multiplicatively by `breadth.MidBook`. Relative rather than
absolute keeps it sane across a 23-year sample and makes every statistic invariant to the
decimal scale. Values are measured where a real book once existed (EURUSD 0.45, USDJPY
0.64, SP500 1.3, XAUUSD 2.1) and rounded UP elsewhere, so the cost is pessimistic.

This is only honest for measuring a SLOPE across buckets, or for rejecting an effect that
fails to clear it. A cost assumption shifts every bucket together; it cannot manufacture or
hide a slope. Never quote a BASE level from a synthesised book as if it were tradeable, and
never use Dukascopy's own quoted spread as a cost.
"""
import os

#--- Outside the repository on purpose: a 2.1 GB .dat was committed here once and cost a
#--- history rewrite. Nothing under ROOT is ever tracked.
ROOT = 'c:/Users/admin/Documents/Workspaces/Market Data/'
HIST = ROOT + 'History/'
CACHE = ROOT + 'sqxbars/'      # decoded bar arrays
ROWS = ROOT + 'edge_rows/'     # scored screen output, so re-analysis never needs a re-run
DECIMALS = 6

#--- key -> (nice, asset class, full spread in bp of price, plausible close range)
#--- The range is the scale guard, not decoration: it is what makes 1e6 checkable per file.
SYMS = {
    'BTCUSD_the5ers':                 ('BTCUSD',   'crypto', 10.0, (100, 500_000)),
    'GBRIDXGBP_dukascopy_the5ers':    ('FTSE_d',   'index',   1.5, (3_000, 12_000)),
    'UK100_the5ers':                  ('FTSE_5',   'index',   1.5, (3_000, 12_000)),
    'LIGHTCMDUSD_dukascopy_the5ers':  ('WTI_d',    'energy',  4.0, (-50, 200)),
    'XTIUSD_the5ers':                 ('WTI_5',    'energy',  4.0, (-50, 200)),
    'SP500_the5ers':                  ('SP500_5',  'index',   1.3, (500, 12_000)),
    'USA500IDXUSD_dukascopy_the5ers': ('SP500_d',  'index',   1.3, (500, 12_000)),
    'USDCAD_dukascopy_the5ers':       ('USDCAD_d', 'fx',      1.0, (0.8, 2.0)),
    'USDCAD_the5ers':                 ('USDCAD_5', 'fx',      1.0, (0.8, 2.0)),
    'USDJPY_dukascopy_the5ers':       ('USDJPY_d', 'fx',      0.64, (50, 300)),
    'USDJPY_the5ers':                 ('USDJPY_5', 'fx',      0.64, (50, 300)),
    'XAUUSD_dukascopy_the5ers':       ('XAUUSD_d', 'metal',   2.1, (200, 10_000)),
    'XAUUSD_the5ers':                 ('XAUUSD_5', 'metal',   2.1, (200, 10_000)),
    '@EW':                            ('ES_fut',   'futures', 1.4, (300, 12_000)),
    'SPY_benchmark.D':                ('SPY_d1',   'equity',  1.0, (10, 2_000)),
}

#--- the two symbols SQX files under a nested vendor tree rather than one flat folder
_NESTED = {'@EW': 'sq_futures/@/@EW/', 'SPY_benchmark.D': 'sq_equity/S/SPY_benchmark.D/'}

#--- same market, independent feed. A real effect must show in BOTH halves of a pair; this
#--- is the only free replication axis the install offers and it costs nothing to check.
PAIRS = [('FTSE_d', 'FTSE_5'), ('WTI_d', 'WTI_5'), ('SP500_d', 'SP500_5'),
         ('USDCAD_d', 'USDCAD_5'), ('USDJPY_d', 'USDJPY_5'), ('XAUUSD_d', 'XAUUSD_5')]

#--- The 15 feeds are NOT 15 markets. Each PAIRS entry is one market quoted by two vendors
#--- (correlation >= 0.999986 on five of the six), and ES_fut, SPY_d1, SP500_d and SP500_5 are
#--- four claims on the same index. Any statistic that counts feeds as independent tests
#--- overstates its own power by roughly the square root of two - so pooling across the
#--- catalog has to collapse to THIS, not to the symbol list.
MARKET = {}
for _a, _b in PAIRS:
    MARKET[_a] = MARKET[_b] = _a.rsplit('_', 1)[0]
MARKET.update({'ES_fut': 'SP500', 'SPY_d1': 'SP500', 'BTCUSD': 'BTCUSD',
               'FTSE_d': 'FTSE', 'FTSE_5': 'FTSE'})
MARKETS = sorted(set(MARKET.values()))


def market(sym):
    """The distinct market a NICE symbol belongs to. Unknown symbols stand alone."""
    return MARKET.get(sym, sym)


NICE = {k: v[0] for k, v in SYMS.items()}
KEY = {v[0]: k for k, v in SYMS.items()}


def nice(key):
    return SYMS[key][0]


def asset_class(key):
    return SYMS[key][1]


def spread_bp(key):
    return SYMS[key][2]


def px_range(key):
    return SYMS[key][3]


def path(key, tf='M1'):
    return f"{HIST}{_NESTED.get(key, key + '/')}{key}_{tf}.dat"


def available(key, tf='M1'):
    return os.path.exists(path(key, tf))


def keys(cls=None):
    """Catalog keys, optionally one asset class, and only those with SOME bar file present.

    Filtered on `base_tf`, not on M1: the futures tree ships M60 and the SPY benchmark ships
    D1 only, and keying this on M1 silently dropped both from every screen that iterated it -
    a missing instrument looks exactly like an instrument with nothing to report.
    """
    out = []
    for k in SYMS:
        if cls is not None and asset_class(k) != cls:
            continue
        try:
            base_tf(k)
        except FileNotFoundError:
            continue
        out.append(k)
    return out


#--- coarsest-to-finest, so `base_tf` returns the finest series a symbol actually ships.
#--- SQX names hourly bars M60 in the futures tree and H1 in the CFD tree; both mean an hour.
_BASE_ORDER = ('M1', 'M60', 'H1', 'D1')
_BASE_SEC = {'M1': 60, 'M60': 3600, 'H1': 3600, 'D1': 86400}


def base_tf(key):
    """The finest timeframe this symbol has a file for. Raises if it has none."""
    for tf in _BASE_ORDER:
        if available(key, tf):
            return tf
    raise FileNotFoundError(f"{nice(key)}: no bar file at any timeframe")


def base_sec(key):
    return _BASE_SEC[base_tf(key)]


#--- catalog key -> the name the alt-data collectors use (research/altdata/*, keyed on the
#--- instrument the FUTURES contract tracks, not on our feed name). None = no alt coverage.
#--- SP500_d/SP500_5/ES_fut/SPY_d1 all map to SP500: one COT series describes the E-mini, and
#--- every one of those four is a claim on the same index.
ALT_SYMBOL = {
    'USDJPY_dukascopy_the5ers': 'USDJPY', 'USDJPY_the5ers': 'USDJPY',
    'USDCAD_dukascopy_the5ers': 'USDCAD', 'USDCAD_the5ers': 'USDCAD',
    'XAUUSD_dukascopy_the5ers': 'XAUUSD', 'XAUUSD_the5ers': 'XAUUSD',
    'SP500_the5ers': 'SP500', 'USA500IDXUSD_dukascopy_the5ers': 'SP500',
    '@EW': 'SP500', 'SPY_benchmark.D': 'SP500',
    'LIGHTCMDUSD_dukascopy_the5ers': 'XTIUSD', 'XTIUSD_the5ers': 'XTIUSD',
    'BTCUSD_the5ers': 'BTCUSD',
    'GBRIDXGBP_dukascopy_the5ers': None, 'UK100_the5ers': None,   # no COT for FTSE
}

#--- instrument-specific implied-vol FRED series, where one exists for the actual underlying
IVOL_SERIES = {'XAUUSD': 'GVZCLS', 'XTIUSD': 'OVXCLS', 'SP500': 'VXDCLS'}


def alt_symbol(key):
    return ALT_SYMBOL.get(key)