SniperGold_ML/ml/p3/smc_semantic/comparison.py

# -*- coding: utf-8 -*-
"""COMPARISON (P3-S.1) — Human vs Human, Machine vs Human Consensus (Liquidity Sweep f7).

PROTOKOL:
  1. Human A -> output/human_A_f7.csv ; Human B -> output/human_B_f7.csv (independen).
  2. Jalankan script: menghitung INTER-RATER dulu (A vs B) -> human_interrater_f7_report.json.
  3. Kasus berbeda -> adjudikasi -> output/human_adjudicated_f7.csv
     (adjudicator TIDAK melihat machine result).
  4. Jalankan lagi: machine (f7_v2) vs human consensus -> human_machine_f7_comparison.json.

OUTPUT:
  human_interrater_f7_report.json   (A vs B: agreement, reference/direction/timeframe/
                                     rejection/confidence; disagreement -> ADJUDICATION REQUIRED)
  human_machine_f7_comparison.json  (TP/FP/FN/TN, P/R/F1, level 1-3, false agreement,
                                     timeframe mismatch, rejection agreement)

METRIC DEFINITIONS:
  L1 outcome    : human YES == machine YES
  L2 structural : L1 + reference kind match + direction match + timeframe match
  L3 semantic   : L2 + rejection agreement (+ reason primitives konsisten via keyword)

Usage: python comparison.py
"""
import os
import sys
import csv
import json
import hashlib
import datetime as dt

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "output")

MACHINE_VERSION = "f7_v2"
SOURCE_COMMIT = "b519a34"


def sha256_file(p):
    h = hashlib.sha256()
    with open(p, "rb") as f:
        for blk in iter(lambda: f.read(1 << 20), b""):
            h.update(blk)
    return h.hexdigest()


def load_csv(p):
    if not os.path.exists(p):
        return None
    with open(p, encoding="utf-8-sig") as f:
        return {r["case_id"]: r for r in csv.DictReader(f)}


def norm_yes_no(v):
    v = (v or "").strip().upper()
    if v in ("YES", "YA", "TRUE", "1"):
        return "YES"
    if v in ("NO", "TIDAK", "FALSE", "0"):
        return "NO"
    if v in ("AMBIGUOUS", "AMB", "?"):
        return "AMBIGUOUS"
    return None


def norm_rej(v):
    v = (v or "").strip().upper()
    if v.startswith("YES"):
        return "YES"
    if v.startswith("NO"):
        return "NO"
    if v.startswith("AMB"):
        return "AMBIGUOUS"
    if v in ("N/A", "NA", ""):
        return "N/A"
    return v


def machine_ref_kind(m):
    rt = (m.get("reference_type") or "").upper()
    if "EQH" in rt:
        return "EQH"
    if "EQL" in rt:
        return "EQL"
    if "SWING" in rt or "INTERNAL" in rt:
        return "SWING"
    return "NONE"


def machine_dir(m):
    prim = m.get("primary_primitive", "")
    if prim == "f10_eqh":
        return "BEARISH"
    if prim == "f11_eql":
        return "BULLISH"
    if prim == "f7_grab":
        return "BEARISH" if int(m.get("f7_corrected", 0)) < 0 else "BULLISH"
    return "NONE"


def machine_rej(m):
    r = (m.get("rejection") or "").upper()
    if r.startswith("YES"):
        return "YES"
    if r.startswith("N/A"):
        return "N/A"
    return "N/A"


def human_ref_kind(h):
    r = (h.get("reference") or "").strip().upper()
    if r in ("EQH",):
        return "EQH"
    if r in ("EQL",):
        return "EQL"
    if r in ("SWING HIGH", "SWING HIGH", "SWING"):
        return "SWING"
    return r or "NONE"


def conf_matrix(yt, yp):
    tp = fp = fn = tn = 0
    for t, p in zip(yt, yp):
        if t == "YES" and p == "YES":
            tp += 1
        elif t == "NO" and p == "YES":
            fp += 1
        elif t == "YES" and p == "NO":
            fn += 1
        elif t == "NO" and p == "NO":
            tn += 1
    prec = tp / (tp + fp) if tp + fp else float("nan")
    rec = tp / (tp + fn) if tp + fn else float("nan")
    f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else float("nan")
    return {"tp": tp, "fp": fp, "fn": fn, "tn": tn,
            "precision": round(prec, 4), "recall": round(rec, 4), "f1": round(f1, 4)}


def dist(counter):
    return {k: int(v) for k, v in counter.items()}


def interrater(hA, hB, machine, rep):
    common = sorted(set(hA) & set(hB))
    res = {"n_common": len(common)}
    yn_agree = ref_agree = dir_agree = tf_agree = rej_agree = 0
    confA, confB = {}, {}
    disagree = []
    for cid in common:
        a, b = hA[cid], hB[cid]
        va, vb = norm_yes_no(a.get("liquidity_sweep")), norm_yes_no(b.get("liquidity_sweep"))
        if va == vb:
            yn_agree += 1
        else:
            disagree.append({"case_id": cid, "A": va, "B": vb})
        ra, rb = human_ref_kind(a), human_ref_kind(b)
        if ra == rb:
            ref_agree += 1
        da, db = (a.get("direction") or "").strip().upper(), (b.get("direction") or "").strip().upper()
        if da == db:
            dir_agree += 1
        ta, tb = (a.get("timeframe") or "").strip().upper(), (b.get("timeframe") or "").strip().upper()
        if ta == tb:
            tf_agree += 1
        rja, rjb = norm_rej(a.get("close_back_rejection")), norm_rej(b.get("close_back_rejection"))
        if rja == rjb:
            rej_agree += 1
        confA[a.get("confidence", "").upper()] = confA.get(a.get("confidence", "").upper(), 0) + 1
        confB[b.get("confidence", "").upper()] = confB.get(b.get("confidence", "").upper(), 0) + 1
    n = len(common)
    res.update({
        "yes_no_agreement_rate": round(yn_agree / n, 4) if n else None,
        "reference_agreement_rate": round(ref_agree / n, 4) if n else None,
        "direction_agreement_rate": round(dir_agree / n, 4) if n else None,
        "timeframe_agreement_rate": round(tf_agree / n, 4) if n else None,
        "rejection_agreement_rate": round(rej_agree / n, 4) if n else None,
        "confidence_dist_A": dist(confA),
        "confidence_dist_B": dist(confB),
        "n_disagreement": len(disagree),
        "disagreement_cases": disagree,
        "adjudication_required": len(disagree) > 0,
    })
    return res


def machine_vs_consensus(human, machine, rep):
    common = sorted(set(human) & set(machine))
    yt, yp = [], []
    amb = []
    for cid in common:
        t = norm_yes_no(human[cid].get("liquidity_sweep"))
        p = norm_yes_no(machine[cid]["machine_decision"])
        if t == "AMBIGUOUS":
            amb.append(cid)
            continue
        if t is None or p is None:
            continue
        yt.append(t)
        yp.append(p)
    cm = conf_matrix(yt, yp)
    res = {"n_common": len(common), "n_evaluated": len(yt),
           "n_ambiguous_human": len(amb), "ambiguous_cases": amb, **cm}

    # semantic levels utk kasus L1 (human YES & machine YES)
    both_yes = [c for c in common
                if norm_yes_no(human[c].get("liquidity_sweep")) == "YES" and
                machine[c]["machine_decision"] == "YES"]
    l2 = l3 = 0
    false_agree = []
    tf_mismatch = []
    rej_agree = 0
    for cid in both_yes:
        h, m = human[cid], machine[cid]
        hk, mk = human_ref_kind(h), machine_ref_kind(m)
        hd, md = (h.get("direction") or "").strip().upper(), machine_dir(m)
        ht, mt = (h.get("timeframe") or "").strip().upper(), m.get("timeframe", "M15").upper()
        hr, mr = norm_rej(h.get("close_back_rejection")), machine_rej(m)
        struct = (hk == mk and hd == md and ht == mt)
        rej_ok = (hr == mr or mr == "N/A")
        if struct:
            l2 += 1
            if rej_ok:
                l3 += 1
        if hk != mk:
            false_agree.append({"case_id": cid, "human_ref": h.get("reference"),
                                "machine_ref": m.get("reference_type")})
        if ht != mt:
            tf_mismatch.append({"case_id": cid, "human_tf": ht, "machine_tf": mt})
        if rej_ok:
            rej_agree += 1
    res["semantic_levels"] = {
        "L1_outcome_agreement": len(both_yes),
        "L2_structural_agreement": l2,
        "L3_semantic_agreement": l3,
        "L3_rate_of_L1": round(l3 / len(both_yes), 4) if both_yes else None,
    }
    res["semantic_false_agreement"] = false_agree
    res["timeframe_mismatch"] = tf_mismatch
    res["rejection_agreement_both_yes"] = {"agree": rej_agree,
                                           "n_both_yes": len(both_yes)}
    # rejection summary semua kasus
    rej_table = []
    for cid in common:
        hr = norm_rej(human[cid].get("close_back_rejection"))
        mr = machine_rej(machine[cid])
        rej_table.append({"case_id": cid, "human_rejection": hr, "machine_rejection": mr,
                          "agree": hr == mr or mr == "N/A"})
    res["rejection_table"] = rej_table
    # timeframe mismatch grouping
    grp = {}
    for x in tf_mismatch:
        key = f"{x['human_tf']} vs {x['machine_tf']}"
        grp[key] = grp.get(key, 0) + 1
    res["timeframe_mismatch_grouped"] = grp
    return res


def main():
    hA = load_csv(os.path.join(OUT, "human_A_f7.csv"))
    hB = load_csv(os.path.join(OUT, "human_B_f7.csv"))
    hAdj = load_csv(os.path.join(OUT, "human_adjudicated_f7.csv"))
    machine = load_csv(os.path.join(OUT, "machine_annotations_f7_v2.csv"))
    if machine is None:
        raise SystemExit("machine_annotations_f7_v2.csv belum ada — jalankan machine_annotator.py")

    rep = {"generated_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
           "machine_version": MACHINE_VERSION, "source_commit": SOURCE_COMMIT,
           "machine_annotation_sha": sha256_file(os.path.join(OUT, "machine_annotations_f7_v2.csv")),
           "case_set_sha": sha256_file(os.path.join(OUT, "cases_meta.json")),
           "human_A_sha": sha256_file(os.path.join(OUT, "human_A_f7.csv")) if hA else None,
           "human_B_sha": sha256_file(os.path.join(OUT, "human_B_f7.csv")) if hB else None,
           "adjudicated_sha": sha256_file(os.path.join(OUT, "human_adjudicated_f7.csv")) if hAdj else None}

    # ---------- INTER-RATER (Human A vs Human B) ----------
    if hA and hB:
        rep["human_vs_human"] = interrater(hA, hB, machine, rep)
        with open(os.path.join(OUT, "human_interrater_f7_report.json"), "w",
                  encoding="utf-8") as f:
            json.dump(rep["human_vs_human"], f, indent=2, default=str)
        print("HUMAN A vs HUMAN B (inter-rater):")
        hv = rep["human_vs_human"]
        for k in ("yes_no_agreement_rate", "reference_agreement_rate",
                  "direction_agreement_rate", "timeframe_agreement_rate",
                  "rejection_agreement_rate", "n_disagreement"):
            print(f"  {k}: {hv.get(k)}")
        if hv["disagreement_cases"]:
            print("  ADJUDICATION REQUIRED:", [d["case_id"] for d in hv["disagreement_cases"]])
    else:
        rep["human_vs_human"] = {"note": "human_A_f7.csv / human_B_f7.csv belum ada"}

    # ---------- MACHINE vs CONSENSUS ----------
    if hAdj:
        consensus = hAdj
        rep["consensus_source"] = "human_adjudicated_f7.csv"
    elif hA:
        consensus = hA
        rep["consensus_source"] = "human_A_f7.csv (NO adjudication yet)"
    else:
        consensus = None
        rep["consensus_source"] = None

    if consensus:
        rep["machine_vs_human"] = machine_vs_consensus(consensus, machine, rep)
        with open(os.path.join(OUT, "human_machine_f7_comparison.json"), "w",
                  encoding="utf-8") as f:
            json.dump(rep["machine_vs_human"], f, indent=2, default=str)
        mv = rep["machine_vs_human"]
        print("\nMACHINE v2 vs HUMAN CONSENSUS:")
        for k in ("n_evaluated", "tp", "fp", "fn", "tn", "precision", "recall", "f1"):
            print(f"  {k}: {mv.get(k)}")
        print("  levels:", mv.get("semantic_levels"))
        print("  false agreement:", len(mv.get("semantic_false_agreement", [])))
        print("  timeframe mismatch:", mv.get("timeframe_mismatch_grouped"))
    else:
        rep["machine_vs_human"] = {"note": "menunggu human annotation"}

    print("\n[saved] human_interrater_f7_report.json + human_machine_f7_comparison.json "
          "(jika human ada)")
    print("STATUS: " + ("MENUNGGU HUMAN ANNOTATION" if consensus is None else "COMPARISON DONE"))


if __name__ == "__main__":
    main()