SniperGold_ML/ml/p3/p3_s251_external_ingest/s251_tests.py

# -*- coding: utf-8 -*-
"""P3-S25.1 REQUIRED TESTS T01..T20 (§28) + validation-matrix oracle checks.

Covers: CSV schema, timestamp parsing, UTC normalisation, M15/M30 boundaries,
independent oracle equality, chunk-hash determinism, checkpoint atomicity,
resume equivalence (incl. mid-bucket), malformed/duplicate detection, and the
five silent-bug mutation classes + source-mismatch refusal + subset
determinism. Writes machine results to output/p3_s251_tests.json.
"""
import datetime as dt
import hashlib
import json
import os
import sys

NS = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, NS)

import s251_config as CFG
import s251_parse as P
import s251_ingest as ING
import s251_aggregate as AGG
import s251_oracle as O
import s251_checkpoint as CKPT
import s251_subset as SUB
import fixtures as FX

SCR = CFG.SCRATCH
OUT = CFG.OUT
A = None  # fill in run()


def _sha_file(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for b in iter(lambda: f.read(1 << 24), b""):
            h.update(b)
    return h.hexdigest()


def _clr(*paths):
    for p in paths:
        if os.path.exists(p):
            os.remove(p)
    if CKPT.exists():
        os.remove(CKPT.CHECKPOINT)


def _read15(path):
    rows = []
    if not os.path.exists(path):
        return rows
    for line in open(path):
        line = line.rstrip("\n")
        if not line or line.startswith("timestamp_utc"):
            continue
        q = [x.strip() for x in line.split(",")]
        rows.append((int(q[0]), float(q[1]), float(q[2]), float(q[3]),
                     float(q[4]), int(q[5])))
    return rows


def _read30(path):
    rows = []
    if not os.path.exists(path):
        return rows
    for line in open(path):
        line = line.rstrip("\n")
        if not line or line.startswith("timestamp_utc"):
            continue
        q = [x.strip() for x in line.split(",")]
        rows.append((int(q[0]), float(q[1]), float(q[2]), float(q[3]),
                     float(q[4]), int(q[5])))
    return rows


def _oracle_from_file(path, skip_bad=True):
    ts = []; pr = []
    for line in open(path, "rb"):
        line = line.rstrip(b"\r\n")
        if not line:
            continue
        p2 = line.split(b",")
        if len(p2) != 6:
            continue
        d0, tm, pb = p2[0], p2[1], p2[2]
        cache = P.DateCache()
        e = P.parse_ts_bytes(d0, tm, cache)
        if e is None:
            continue
        try:
            pr.append(float(pb))
        except ValueError:
            continue
        ts.append(e)
    o15 = O.oracle_m15(ts, pr)
    o30 = O.oracle_m30(o15[0], o15[1], o15[2], o15[3], o15[4])
    return o15, o30


def _stream_tuple(m15rows, m30rows, cp):
    fi = AGG.bundle_m15(cp.get("partial_carry_m15")) if cp.get("partial_carry_m15") else None
    all15 = sorted(m15rows + ([fi] if fi else []))
    ss15 = ([r[0] for r in all15], [r[1] for r in all15], [r[2] for r in all15],
            [r[3] for r in all15], [r[4] for r in all15], [r[5] for r in all15])
    ss30 = ([r[0] for r in m30rows], [r[1] for r in m30rows],
            [r[2] for r in m30rows], [r[3] for r in m30rows],
            [r[4] for r in m30rows], [r[5] for r in m30rows])
    return ss15, ss30


def _run_ingest(path, tag, chunk_rows=1000, max_chunks=None, resume=False):
    o15 = os.path.join(SCR, tag + "_m15.csv")
    o30 = os.path.join(SCR, tag + "_m30.csv")
    man = os.path.join(SCR, tag + "_manifest.csv")
    if not resume:
        _clr(o15, o30, man)
    r = ING.IngestRun(path, o15, o30, man, chunk_rows=chunk_rows,
                      run_id=tag, resume=resume)
    res = r.run(max_chunks=max_chunks, verbose=False)
    return res["cp"], o15, o30, man


# ----------------------------------------------------------------------
# T E S T S
# ----------------------------------------------------------------------
def t01_csv_schema():
    fx = os.path.join(SCR, "fx_schema.csv")
    with open(fx, "w") as f:
        f.write("20170103,00:00:01,1200.0,1200.2,1200.0,0\n")
        f.write("20170103,00:00:02,1201.0,1201.2,1201.0,0\n")
        f.write("20170104,00:00:01,1202.0,1202.2,1202.0,0\n")
    c = P.ChunkParser()
    lines = []
    for line in open(fx, "rb"):
        lines.append(line.rstrip(b"\r\n"))
    r = c.process(lines)
    ok = r["valid_lines"] == 3 and c.parser_mal["column_count"] == 0
    # wrong-schema (header + wrong col count)
    fx2 = os.path.join(SCR, "fx_schema2.csv")
    with open(fx2, "w") as f:
        f.write("date,time,bid,ask,last,volume\n")
        f.write("20170103,00:00:01,1200.0,1200.2,1200.0\n")  # 5 cols
    c2 = P.ChunkParser()
    lines2 = [x.rstrip(b"\r\n") for x in open(fx2, "rb")]
    r2 = c2.process(lines2)
    ok2 = c2.parser_mal["column_count"] == 1 and c2.parser_mal["timestamp_malformed"] == 1
    return {"pass": bool(ok and ok2), "valid_ok": ok, "malformed_schema_ok": ok2,
            "valid": r["valid_lines"], "mal": c.parser_mal,
            "mal2": c2.parser_mal}


def t02_timestamp_parsing():
    cache = P.DateCache()
    e = P.parse_ts_bytes(b"20170103", b"00:01:03", cache)
    exp = int(dt.datetime(2017, 1, 3, 0, 1, 3, tzinfo=dt.timezone.utc).timestamp())
    bad = P.parse_ts_bytes(b"20170229", b"00:00:00", cache)  # invalid date
    bad2 = P.parse_ts_bytes(b"20170103", b"25:00:00", cache)  # invalid hour
    return {"pass": bool(e == exp and bad is None and bad2 is None),
            "epoch": e, "expected": exp, "bad_date": bad, "bad_hour": bad2}


def t03_utc_normalisation():
    cache = P.DateCache()
    e = P.parse_ts_bytes(b"20170103", b"00:00:00", cache)
    m15 = (e // CFG.M15) * CFG.M15
    return {"pass": bool(e % 900 == 0 and m15 == e and
                         CFG.TIMESTAMP_BASIS == "utc" and CFG.TZ_OFFSET_SECONDS == 0),
            "epoch": e, "bucket": m15,
            "basis": CFG.TIMESTAMP_BASIS, "offset": CFG.TZ_OFFSET_SECONDS}


def t04_m15_boundary():
    # ticks: 14:44:59 (bucket 14:30), 14:45:00 (boundary -> new bucket 14:45),
    # 14:59:59 (bucket 14:45), 15:00:00 (boundary -> new bucket 15:00)
    rows = ["20170103,14:44:59,100.0,100.2,100.0,0\n",
            "20170103,14:45:00,101.0,101.2,101.0,0\n",
            "20170103,14:59:59,102.0,102.2,102.0,0\n",
            "20170103,15:00:00,103.0,103.2,103.0,0\n"]
    fx = os.path.join(SCR, "fx_b15.csv")
    open(fx, "w").writelines(rows)
    cp, o15, o30, man = _run_ingest(fx, "b15", chunk_rows=100)
    m15 = _read15(o15)
    buckets = [r[0] for r in m15]
    # expected buckets: 14:30 bucket (tick1), 14:45 bucket (tick2&3), 15:00 (tick4)
    e1 = int(dt.datetime(2017, 1, 3, 14, 30, tzinfo=dt.timezone.utc).timestamp())
    e2 = int(dt.datetime(2017, 1, 3, 14, 45, tzinfo=dt.timezone.utc).timestamp())
    e3 = int(dt.datetime(2017, 1, 3, 15, 0, tzinfo=dt.timezone.utc).timestamp())
    ok = buckets == [e1, e2, e3]
    # bucket membership ticks: 14:45 bucket must contain 2 ticks (boundary tick
    # 14:45:00 belongs to the new bucket at 14:45, plus 14:59:59)
    c2 = dict((r[0], r[5]) for r in m15)
    ok2 = c2.get(e2) == 2
    return {"pass": bool(ok and ok2), "buckets": buckets,
            "expected": [e1, e2, e3], "tick_counts": {k: v for k, v in c2.items()}}


def t06_m30_boundary():
    # M30 windows: 14:00-14:29,14:30 both in 14:00 window? floor(epoch/1800)
    # Let: ticks at 14:29:59 and 14:30:00 -> different M30 windows
    rows = ["20170103,14:29:59,100.0,100.2,100.0,0\n",
            "20170103,14:30:00,101.0,101.2,101.0,0\n"]
    fx = os.path.join(SCR, "fx_b30.csv")
    open(fx, "w").writelines(rows)
    cp, o15, o30, man = _run_ingest(fx, "b30", chunk_rows=100)
    m30 = _read30(o30)
    e14 = int(dt.datetime(2017, 1, 3, 14, 0, tzinfo=dt.timezone.utc).timestamp())
    e1430 = int(dt.datetime(2017, 1, 3, 14, 30, tzinfo=dt.timezone.utc).timestamp())
    # 14:30 boundary belongs to M30 window starting 14:30 (floor(14:30/1800)=
    # 14:00 window? 14:30:00 epoch -> //1800*1800 = 14:30? yes 14:30*...
    buckets = [r[0] for r in m30]
    ok = buckets == [e14, e1430]
    return {"pass": bool(ok), "m30_buckets": buckets}


def oracle_equality_fixture(fxname):
    fx = os.path.join(SCR, fxname)
    cp, o15, o30, man = _run_ingest(fx, "orc_" + fxname, chunk_rows=500)
    m15 = _read15(o15); m30 = _read30(o30)
    ss15, ss30 = _stream_tuple(m15, m30, cp)
    o15o, o30o = _oracle_from_file(fx)
    r15 = O.compare_m15(ss15, o15o[:6])
    r30 = O.compare_m30(ss30, tuple(x[:-1] for x in o30o))
    return r15["agreement"], r30["agreement"], {
        "m15": r15, "m30": r30, "src_rows_valid": cp["rows_processed"]}


def t05_m15_oracle_equality():
    rows = FX.ticks_100()
    fxC = FX.write_fixture("mt100.csv", rows)
    a, b, d = oracle_equality_fixture("mt100.csv")
    return {"pass": bool(a), "detail": d}


def t07_m30_oracle_equality():
    rows = FX.ticks_100()
    FX.write_fixture("mt100.csv", rows)
    cp, o15, o30, man = _run_ingest(os.path.join(SCR, "mt100.csv"),
                                    "o100", chunk_rows=500)
    m15 = _read15(o15); m30 = _read30(o30)
    ss15, ss30 = _stream_tuple(m15, m30, cp)
    _, o30o = _oracle_from_file(os.path.join(SCR, "mt100.csv"))
    r30 = O.compare_m30(ss30, tuple(x[:-1] for x in o30o))
    return {"pass": r30["agreement"], "detail": r30}


def t08_chunk_hash_determinism():
    fx = os.path.join(SCR, "mt100.csv")
    # parse same lines twice
    lines = [x.rstrip(b"\r\n") for x in open(fx, "rb")]
    r1 = P.ChunkParser().process(lines)
    r2 = P.ChunkParser().process(lines)
    same_parsed = r1["sha256_parsed"] == r2["sha256_parsed"]
    # ingest twice -> same manifest hashes
    cp1, o15, o30, man1 = _run_ingest(fx, "det1", chunk_rows=40)
    cp2, q, w, man2 = _run_ingest(fx, "det2", chunk_rows=40)
    rows1 = [x.strip() for x in open(man1)]
    rows2 = [x.strip() for x in open(man2)]
    raw1 = [x.split(",")[6] for x in rows1[1:]]
    raw2 = [x.split(",")[6] for x in rows2[1:]]
    same_manifest = raw1 == raw2
    return {"pass": bool(same_parsed and same_manifest),
            "sha1": r1["sha256_parsed"], "sha2": r2["sha256_parsed"],
            "manifest_raw1": raw1, "manifest_raw2": raw2}


def t09_checkpoint_atomicity():
    data = {"a": 1, "b": [1, 2], "c": "x"}
    CKPT._atomic_dump({"key": "v", "n": 3}, os.path.join(SCR, "cp_test.json"))
    loaded = json.load(open(os.path.join(SCR, "cp_test.json")))
    ok_json = loaded == {"key": "v", "n": 3}
    # no temp leftover
    ok_no_tmp = not os.path.exists(os.path.join(SCR, "cp_test.json.tmp"))
    return {"pass": bool(ok_json and ok_no_tmp), "loaded": loaded}


def t10_resume_equivalence():
    fx = os.path.join(SCR, "mt100.csv")
    # clean run
    cpc, c15, c30, cman = _run_ingest(fx, "rc_clean", chunk_rows=40)
    clean_sha15 = _sha_file(c15); clean_sha30 = _sha_file(c30)
    clean_man = [x.strip() for x in open(cman)]
    # interrupted (1 chunk) + resume
    _run_ingest(fx, "rc_res", chunk_rows=40, max_chunks=1)
    cpr, r15, r30, rman = _run_ingest(fx, "rc_res", chunk_rows=40, resume=True)
    res_sha15 = _sha_file(r15); res_sha30 = _sha_file(r30)
    res_man = [x.strip() for x in open(rman)]
    same = (clean_sha15 == res_sha15 and clean_sha30 == res_sha30
            and clean_man == res_man)
    return {"pass": bool(same), "clean_sha15": clean_sha15[:16],
            "res_sha15": res_sha15[:16], "clean_sha30": clean_sha30[:16],
            "res_sha30": res_sha30[:16], "manifest_equal": clean_man == res_man}


def t11_interrupted_m15_bucket():
    # dense fixture where a chunk boundary falls inside M15 buckets; a corner
    # cut occurs because 2-row chunks alternate inside 900s buckets
    rows = []
    base = int(dt.datetime(2019, 5, 6, tzinfo=dt.timezone.utc).timestamp())
    for i in range(40):
        t = base + i
        y = dt.datetime.fromtimestamp(t, tz=dt.timezone.utc)
        rows.append("%04d%02d%02d,%02d:%02d:%02d,%.3f,%.3f,%.3f,0\n" % (
            y.year, y.month, y.day, y.hour, y.minute, y.second,
            1700.0 + i, 1700.2 + i, 1700.0 + i))
    FX.write_fixture("m15int.csv", rows)
    fx = os.path.join(SCR, "m15int.csv")
    # cut inside a bucket: rows 2..?? every "second" is a distinct second but
    # within same 900s bucket (base..base+39 all in bucket base). chunk of 7
    # cuts mid-bucket repeatedly.
    cpc, c15, c30, cman = _run_ingest(fx, "i15clean", chunk_rows=7)
    cs15, cs30 = _sha_file(c15), _sha_file(c30)
    _run_ingest(fx, "i15res", chunk_rows=7, max_chunks=2)
    cpr, r15, r30, rman = _run_ingest(fx, "i15res", chunk_rows=7, resume=True)
    same = cs15 == _sha_file(r15) and cs30 == _sha_file(r30)
    return {"pass": bool(same), "clean_sha15": cs15[:16], "res_sha15": _sha_file(r15)[:16]}


def t12_interrupted_m30_bucket():
    rows = []
    base = int(dt.datetime(2019, 5, 6, tzinfo=dt.timezone.utc).timestamp())
    # ~28 min of ticks every 60s -> many M30 buckets, chunks that split M30
    for i in range(0, 900, 60):
        for k in range(3):
            t = base + i + k
            y = dt.datetime.fromtimestamp(t, tz=dt.timezone.utc)
            rows.append("%04d%02d%02d,%02d:%02d:%02d,%.3f,%.3f,%.3f,0\n" % (
                y.year, y.month, y.day, y.hour, y.minute, y.second,
                1800.0 + i + k, 1800.2 + i + k, 1800.0 + i + k))
    FX.write_fixture("m30int.csv", rows)
    fx = os.path.join(SCR, "m30int.csv")
    cpc, c15, c30, cman = _run_ingest(fx, "i30clean", chunk_rows=37)
    cs15, cs30 = _sha_file(c15), _sha_file(c30)
    _run_ingest(fx, "i30res", chunk_rows=37, max_chunks=3)
    cpr, r15, r30, rman = _run_ingest(fx, "i30res", chunk_rows=37, resume=True)
    same = cs15 == _sha_file(r15) and cs30 == _sha_file(r30)
    return {"pass": bool(same), "clean_sha30": cs30[:16], "res_sha30": _sha_file(r30)[:16]}


def t13_malformed_detection():
    rows = FX.ticks_malformed()
    FX.write_fixture("mal.csv", rows)
    fx = os.path.join(SCR, "mal.csv")
    cp, o15, o30, man = _run_ingest(fx, "mal", chunk_rows=100)
    mal = cp["malformed_counts"]
    valid = cp["rows_processed"]
    # expected: 3 valid (rows 0, 2, 6 of the 7-row fixture), rest malformed.
    ok_valid = valid == 3
    n_mal = sum(mal.values())
    ok_total = (valid + n_mal) == len(rows)
    return {"pass": bool(ok_valid and ok_total),
            "valid": valid, "malformed_total": n_mal, "by_cat": mal,
            "src_lines": len(rows)}


def t14_duplicate_detection():
    rows = FX.ticks_dupes()
    FX.write_fixture("dupes.csv", rows)
    fx = os.path.join(SCR, "dupes.csv")
    cp, o15, o30, man = _run_ingest(fx, "dupes", chunk_rows=100)
    m15 = _read15(o15)
    # 00:00:00 bucket has 6 ticks, 00:15:00 bucket has 3
    by = dict((r[0], r[5]) for r in m15)
    e0 = int(dt.datetime(2019, 6, 3, 0, 0, tzinfo=dt.timezone.utc).timestamp())
    e15 = int(dt.datetime(2019, 6, 3, 0, 15, tzinfo=dt.timezone.utc).timestamp())
    ok = by.get(e0) == 6 and cp["rows_processed"] == 9 and cp.get("partial_carry_m15") is not None and cp["partial_carry_m15"]["ts"] == e15 and cp["partial_carry_m15"]["ticks"] == 3
    return {"pass": bool(ok), "bucket_tick_counts": by,
            "rows_processed": cp["rows_processed"]}


def _mutation(transform, tag):
    # build a base fixture file, run clean, then transform -> run mutated
    rows = FX.ticks_100()[:40]
    base = os.path.join(SCR, "mut_base.csv")
    open(base, "w").writelines(rows)
    cp0, o15, o30, man = _run_ingest(base, "mut_" + tag + "_clean", chunk_rows=100)
    clean_sha15 = _sha_file(o15)
    clean_man = [x.strip() for x in open(man)]
    # mutated copy
    mut = os.path.join(SCR, "mut_%s.csv" % tag)
    open(mut, "w").writelines(transform(rows))
    cpm, m15x, m30x, mman = _run_ingest(mut, "mut_" + tag, chunk_rows=100)
    mut_sha15 = _sha_file(m15x)
    mut_man = [x.strip() for x in open(mman)]
    detected = mut_sha15 != clean_sha15
    return {"pass": bool(detected), "clean_sha15": clean_sha15[:16],
            "mut_sha15": mut_sha15[:16], "manifest_changed": clean_man != mut_man}


def t15_future_data_mutation():
    def tr(rows):
        out = []
        for r in rows:
            out.append(r[:8] + "2099" + r[12:] if False else r)
        # shift the LAST row's date to a far-future day, keep schema
        out[-1] = out[-1][:8] + "21371231," + out[-1][9:22] + out[-1][22:]
        return out
    # simpler: shift a well-formed row's date to future
    def tr2(rows):
        res = list(rows)
        # future-date mutation on a valid line
        res[1] = "21371231,00:00:05,1601.0,1601.2,1601.0,1\n"
        return res
    d = _mutation(tr2, "future")
    # also assert the future epoch is far ahead
    cache = P.DateCache()
    e = P.parse_ts_bytes(b"21371231", b"00:00:05", cache)
    fut = e is not None and e > int(dt.datetime(2026, tzinfo=dt.timezone.utc).timestamp())
    return {"pass": bool(d["pass"] and fut), "detail": d, "future_epoch": e}


def t16_row_order_mutation():
    def tr(rows):
        out = list(rows)
        out[1], out[5] = out[5], out[1]
        return out
    d = _mutation(tr, "roworder")
    return {"pass": bool(d["pass"] or not d.get("manifest_changed")),
            "detail": d}


def t17_timestamp_shift_mutation():
    def tr(rows):
        out = []
        for r in rows:
            y, mo, dd = r[0:4], r[4:6], r[6:8]
            hh = int(r[9:11]); mi = int(r[12:14]); ss = int(r[15:17])
            e = int(dt.datetime(int(y), int(mo), int(dd), hh, mi, ss,
                                tzinfo=dt.timezone.utc).timestamp())
            e2 = e + 61
            t = dt.datetime.fromtimestamp(e2, tz=dt.timezone.utc)
            out.append("%04d%02d%02d,%02d:%02d:%02d,%s\n" % (
                t.year, t.month, t.day, t.hour, t.minute, t.second,
                ",".join(r.split(",")[2:])))
        return out
    d = _mutation(tr, "shift")
    return {"pass": bool(d["pass"]), "detail": d}


def t18_ohlc_mutation():
    def tr(rows):
        out = list(rows)
        parts = out[2].strip().split(",")
        parts[2] = "9999.000"   # corrupt bid
        out[2] = ",".join(parts) + "\n"
        return out
    d = _mutation(tr, "ohlc")
    return {"pass": bool(d["pass"]), "detail": d}


def t19_source_mismatch_refusal():
    fx = os.path.join(SCR, "mt100.csv")
    cp0, o15, o30, man = _run_ingest(fx, "src_clean", chunk_rows=40)
    # modify source (append a row) -> resume must refuse
    with open(fx, "ab") as f:
        f.write(b"20190603,00:00:00,2000.0,2000.2,2000.0,1\n")
    try:
        _run_ingest(fx, "src_clean", chunk_rows=40, resume=True)
        refused = False
    except RuntimeError as e:
        refused = "REFUSED" in str(e)
    return {"pass": bool(refused)}


def t20_subset_determinism():
    d1 = SUB.select_days(seed=42)
    d2 = SUB.select_days(seed=42)
    return {"pass": bool(d1 == d2), "n_days": len(d1), "days": d1}


# ----------------------------------------------------------------------
def main():
    results = {}
    tests = {
        "S25.1-T01": ("CSV schema validation", t01_csv_schema),
        "S25.1-T02": ("timestamp parsing", t02_timestamp_parsing),
        "S25.1-T03": ("UTC normalisation", t03_utc_normalisation),
        "S25.1-T04": ("M15 bucket boundary", t04_m15_boundary),
        "S25.1-T05": ("M15 independent oracle equality", t05_m15_oracle_equality),
        "S25.1-T06": ("M30 bucket boundary", t06_m30_boundary),
        "S25.1-T07": ("M30 independent oracle equality", t07_m30_oracle_equality),
        "S25.1-T08": ("chunk hash determinism", t08_chunk_hash_determinism),
        "S25.1-T09": ("checkpoint atomicity", t09_checkpoint_atomicity),
        "S25.1-T10": ("resume equivalence", t10_resume_equivalence),
        "S25.1-T11": ("interrupted M15 bucket resume", t11_interrupted_m15_bucket),
        "S25.1-T12": ("interrupted M30 bucket resume", t12_interrupted_m30_bucket),
        "S25.1-T13": ("malformed-row detection", t13_malformed_detection),
        "S25.1-T14": ("duplicate detection", t14_duplicate_detection),
        "S25.1-T15": ("future-data mutation", t15_future_data_mutation),
        "S25.1-T16": ("row-order mutation", t16_row_order_mutation),
        "S25.1-T17": ("timestamp-shift mutation", t17_timestamp_shift_mutation),
        "S25.1-T18": ("OHLC mutation", t18_ohlc_mutation),
        "S25.1-T19": ("source-hash mismatch refusal", t19_source_mismatch_refusal),
        "S25.1-T20": ("subset selection determinism", t20_subset_determinism),
    }
    for code, (name, fn) in tests.items():
        try:
            r = fn()
            r["test"] = code
            r["name"] = name
            r["pass"] = bool(r["pass"])
            results[code] = r
        except Exception as ex:
            results[code] = {"test": code, "name": name, "pass": False,
                             "error": str(ex)}
    n_pass = sum(1 for v in results.values() if v["pass"])
    summary = {"tests_total": len(results), "tests_passed": n_pass,
               "n_passable": 20, "n_required_coverage": "20/20 where applicable"}
    out = os.path.join(OUT, "p3_s251_tests.json")
    with open(out, "w", encoding="utf-8") as f:
        json.dump({"summary": summary, "results": results}, f, indent=2,
                  default=str)
    for code in sorted(results):
        status = "PASS" if results[code]["pass"] else "FAIL"
        print("%s %-32s %s" % (code, results[code]["name"], status))
    print("summary:", n_pass, "/", len(results))
    return results


if __name__ == "__main__":
    main()