Warrior_EA/research/fills.py

"""A fill model that cannot be got wrong by accident.

Three separate results today were invalidated by fill errors, all of the same family:
a price level was used as an entry while the outcome was measured from somewhere else.
The defence is not vigilance, it is an interface where the mistake is unavailable - so
this module owns the ENTIRE lifecycle of a trade and no test may open a position any
other way.

THE RULES, EXPLICIT
-------------------
  side +1 (long)  enters at the ASK, exits at the BID
  side -1 (short) enters at the BID, exits at the ASK

  MARKET  fills at the next bar's open, on the correct side. No ambiguity, no level.
  STOP    a buy stop triggers when the ASK reaches it; a sell stop when the BID does.
          Fill price is the trigger price, or the bar's open if the bar GAPPED past it -
          which is the honest treatment of a gap and the main source of real slippage.
  LIMIT   a buy limit fills when the ASK falls to it; a sell limit when the BID rises.
          Same gap rule, except a gap favours the limit and is capped at the level.

  THE OUTCOME CLOCK STARTS AT THE FILL BAR, NEVER BEFORE. This is the invariant the
  previous round violated: it entered at a stop-order level but measured from the bar's
  open, which sits on the far side of that level by construction, handing the trade a free
  run toward its target. Here the fill index IS the start index; they cannot differ.

  Protective stop and target are then resting orders, checked every bar from the fill bar
  onward, on the correct side of the book.

AMBIGUITY IS REPORTED, NOT ASSUMED AWAY
---------------------------------------
When one M1 bar contains both the stop and the target, the order they were touched is
unknowable at this resolution. The loss is booked - the convention used everywhere in this
project - and the FRACTION of trades decided that way is returned with every result. If
that fraction is large the result is resolution-limited and says so out loud, instead of
quietly depending on a coin flip.
"""
import numpy as np

BIDASK = 'c:/Users/admin/Documents/Workspaces/Market Data/bidask/'
MARKET, STOP, LIMIT = 0, 1, 2


def load(sym):
    z = np.load(f"{BIDASK}{sym}_M1_bidask.npz", allow_pickle=True)
    a = z['bars']
    I = {str(c): k for k, c in enumerate(z['columns'])}
    return a, I


class Book:
    """Bid/ask M1 series with the four price paths a fill needs."""

    def __init__(self, sym):
        a, I = load(sym)
        self.t = a[:, I['time']].astype(np.int64)
        self.bo, self.bh, self.bl, self.bc = (a[:, I[k]] for k in ('bo', 'bh', 'bl', 'bc'))
        self.ao, self.ah, self.al, self.ac = (a[:, I[k]] for k in ('ao', 'ah', 'al', 'ac'))
        self.n = len(self.t)

    def index_at(self, t_ms):
        """First M1 bar at or after a timestamp."""
        return np.searchsorted(self.t, np.asarray(t_ms, np.int64), 'left')


def _entry(bk, start, side, kind, level, window):
    """-> (fill_index, fill_price). -1 where the order never filled inside `window`."""
    n = bk.n
    m = len(start)
    fi = np.full(m, -1, np.int64)
    fp = np.full(m, np.nan)
    if kind == MARKET:
        j = np.minimum(start, n - 1)
        ok = start < n
        fi[ok] = j[ok]
        fp[ok] = np.where(side[ok] > 0, bk.ao[j[ok]], bk.bo[j[ok]])
        return fi, fp
    live = np.ones(m, bool)
    for k in range(window + 1):
        j = start + k
        ok = live & (j < n)
        if not ok.any():
            break
        jj = j[ok]
        lo = np.nonzero(ok)[0]
        # the side of the book the order watches
        if kind == STOP:
            hit = np.where(side[ok] > 0, bk.ah[jj] >= level[ok], bk.bl[jj] <= level[ok])
            opn = np.where(side[ok] > 0, bk.ao[jj], bk.bo[jj])
            #--- a gap past the level fills at the open: worse than the level, which is
            #--- exactly what a stop order does and where real slippage comes from
            px = np.where(side[ok] > 0, np.maximum(level[ok], opn),
                          np.minimum(level[ok], opn))
        else:
            hit = np.where(side[ok] > 0, bk.al[jj] <= level[ok], bk.bh[jj] >= level[ok])
            opn = np.where(side[ok] > 0, bk.ao[jj], bk.bo[jj])
            #--- a gap through a limit fills at the level at worst; cap it there
            px = np.where(side[ok] > 0, np.minimum(level[ok], opn),
                          np.maximum(level[ok], opn))
        w = lo[hit]
        fi[w] = j[w]; fp[w] = px[hit]
        live[w] = False
    return fi, fp


def simulate(bk, start, side, stop_px, targ_px, horizon,
             entry=MARKET, entry_px=None, entry_window=0,
             commission_bp=0.0, slippage_bp=0.0,
             swap_bp_long=0.0, swap_bp_short=0.0):
    """Open a position and run it to stop, target or horizon. Everything in one call.

    COSTS, AND WHAT EACH ONE IS FOR
    -------------------------------
    The spread is charged by construction - a long enters at the ask and exits at the bid,
    at the real quotes on that minute. The other three are NOT in the data and must be
    supplied, because leaving them at zero silently prices a trade nobody can actually do:

      commission_bp   per SIDE, in basis points of price. On a raw-spread account this is
                      comparable to the spread itself and can double the round-trip cost.
      slippage_bp     adverse price movement on each side beyond the quoted level. Gap
                      fills are already modelled exactly (a stop that gaps fills at the
                      open); this covers the ordinary case where a market order does not
                      get the top of book.
      swap_bp_long/   FINANCING, per calendar night held, in basis points of price. This is
      swap_bp_short   the one that decides hold-based strategies: it is negligible on a
                      4-hour trade and dominant on a 4-month one, so a drift or carry result
                      computed without it is not a result. Sign convention: POSITIVE means
                      it COSTS you. A genuinely positive carry is a negative number here.

    Everything is in basis points of price so the same figure is meaningful across a 1.10 FX
    rate and a 5,000 index, and so nothing depends on lot size or account currency.

    Nights are counted as calendar-day boundaries crossed between fill and exit. That is an
    approximation in two known directions - it ignores the triple-swap Wednesday convention
    on FX, and weekend financing is applied as two nights rather than the broker's own rule -
    so treat a swap-dominated result as accurate to roughly +/-20%, and say so.

    Returns dict with R per trade, plus the diagnostics a result should never be quoted
    without: fill rate, how many trades were decided by same-bar ambiguity, and the
    realised entry slippage against the intended level.
    """
    start = np.asarray(start, np.int64)
    side = np.asarray(side, np.int64)
    fi, fp = _entry(bk, start, side, entry,
                    np.asarray(entry_px) if entry_px is not None else None, entry_window)
    filled = fi >= 0
    if not filled.any():
        return None
    idx = fi[filled]; px = fp[filled]; sd = side[filled]
    sl = np.asarray(stop_px)[filled]; tg = np.asarray(targ_px)[filled]
    risk = np.abs(px - sl)
    good = risk > 0
    idx, px, sd, sl, tg, risk = (v[good] for v in (idx, px, sd, sl, tg, risk))
    rew = np.abs(tg - px)

    n = bk.n
    m = len(idx)
    hz = np.full(m, horizon, np.int64) if np.isscalar(horizon) else \
        np.asarray(horizon, np.int64)[filled][good]
    res = np.zeros(m, np.int8)
    both = np.zeros(m, bool)
    xit = idx + np.minimum(hz, n - 1 - idx)
    #--- walk only the trades still open. Without this the loop touches every trade on every
    #--- one of `horizon` steps; at M1 resolution a 200-bar H1 horizon is 12,000 steps and
    #--- the difference is minutes versus hours.
    act = np.arange(m)
    for k in range(int(hz.max()) + 1):
        if not act.size:
            break
        jj = idx[act] + k
        inb = (jj < n) & (k <= hz[act])
        if not inb.all():
            act = act[inb]; jj = jj[inb]
            if not act.size:
                break
        s_ = sd[act]
        #--- a long exits on the BID, a short on the ASK
        hit_s = np.where(s_ > 0, bk.bl[jj] <= sl[act], bk.ah[jj] >= sl[act])
        hit_t = np.where(s_ > 0, bk.bh[jj] >= tg[act], bk.al[jj] <= tg[act])
        r = np.where(hit_s, -1, np.where(hit_t, 1, 0)).astype(np.int8)   # stop wins ties
        got = r != 0
        if got.any():
            w = act[got]
            res[w] = r[got]
            both[w] = (hit_s & hit_t)[got]
            xit[w] = jj[got]
            act = act[~got]

    R = np.where(res > 0, rew / risk, np.where(res < 0, -1.0, 0.0))
    un = res == 0
    if un.any():
        j = xit[un]
        exit_px = np.where(sd[un] > 0, bk.bc[j], bk.ac[j])
        R[un] = (exit_px - px[un]) * sd[un] / risk[un]

    #--- COSTS BEYOND THE SPREAD. Charged in price units and then divided by the trade's own
    #--- risk, so they land in the same R units as the outcome. Commission and slippage are
    #--- per side and hit every trade equally; financing scales with TIME HELD, which is what
    #--- makes it invisible in a barrier test and decisive in a hold.
    cost = np.zeros(len(idx))
    if commission_bp or slippage_bp:
        cost += 2.0 * (commission_bp + slippage_bp) * 1e-4 * px
    nights = np.zeros(len(idx))
    if swap_bp_long or swap_bp_short:
        #--- calendar-day boundaries crossed, in broker time, between fill and exit
        day_in = bk.t[idx] // 86400000
        day_out = bk.t[xit] // 86400000
        nights = (day_out - day_in).astype(float)
        rate = np.where(sd > 0, swap_bp_long, swap_bp_short)
        cost += nights * rate * 1e-4 * px
    R = R - cost / risk

    return dict(R=R, idx=idx, exit_idx=xit, side=sd, fill_px=px, risk=risk, rr=rew / risk,
                bars_held=xit - idx, nights=nights,
                cost_R=cost / risk,
                filled=filled, kept=good,
                fill_rate=float(filled.mean()),
                ambiguous=float(both.mean()),
                unresolved=float(un.mean()),
                n=len(R))


def summary(out, label=''):
    if out is None or out['n'] < 30:
        return f"  {label:<34} - too few"
    R = out['R']
    se = R.std(ddof=1) / np.sqrt(len(R))
    return (f"  {label:<34} n={out['n']:>6}  expR {R.mean():+7.3f}  t {R.mean()/max(se,1e-12):+6.2f}"
            f"   fill {100*out['fill_rate']:5.1f}%  same-bar {100*out['ambiguous']:4.1f}%"
            f"  unres {100*out['unresolved']:4.1f}%")