323 lines
12 KiB
Python
323 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""COMPARISON (P3-S.1) — Human vs Human, Machine vs Human Consensus (Liquidity Sweep f7).
|
|
|
|
PROTOKOL:
|
|
1. Human A -> output/human_A_f7.csv ; Human B -> output/human_B_f7.csv (independen).
|
|
2. Jalankan script: menghitung INTER-RATER dulu (A vs B) -> human_interrater_f7_report.json.
|
|
3. Kasus berbeda -> adjudikasi -> output/human_adjudicated_f7.csv
|
|
(adjudicator TIDAK melihat machine result).
|
|
4. Jalankan lagi: machine (f7_v2) vs human consensus -> human_machine_f7_comparison.json.
|
|
|
|
OUTPUT:
|
|
human_interrater_f7_report.json (A vs B: agreement, reference/direction/timeframe/
|
|
rejection/confidence; disagreement -> ADJUDICATION REQUIRED)
|
|
human_machine_f7_comparison.json (TP/FP/FN/TN, P/R/F1, level 1-3, false agreement,
|
|
timeframe mismatch, rejection agreement)
|
|
|
|
METRIC DEFINITIONS:
|
|
L1 outcome : human YES == machine YES
|
|
L2 structural : L1 + reference kind match + direction match + timeframe match
|
|
L3 semantic : L2 + rejection agreement (+ reason primitives konsisten via keyword)
|
|
|
|
Usage: python comparison.py
|
|
"""
|
|
import os
|
|
import sys
|
|
import csv
|
|
import json
|
|
import hashlib
|
|
import datetime as dt
|
|
|
|
import numpy as np
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
OUT = os.path.join(HERE, "output")
|
|
|
|
MACHINE_VERSION = "f7_v2"
|
|
SOURCE_COMMIT = "b519a34"
|
|
|
|
|
|
def sha256_file(p):
|
|
h = hashlib.sha256()
|
|
with open(p, "rb") as f:
|
|
for blk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(blk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load_csv(p):
|
|
if not os.path.exists(p):
|
|
return None
|
|
with open(p, encoding="utf-8-sig") as f:
|
|
return {r["case_id"]: r for r in csv.DictReader(f)}
|
|
|
|
|
|
def norm_yes_no(v):
|
|
v = (v or "").strip().upper()
|
|
if v in ("YES", "YA", "TRUE", "1"):
|
|
return "YES"
|
|
if v in ("NO", "TIDAK", "FALSE", "0"):
|
|
return "NO"
|
|
if v in ("AMBIGUOUS", "AMB", "?"):
|
|
return "AMBIGUOUS"
|
|
return None
|
|
|
|
|
|
def norm_rej(v):
|
|
v = (v or "").strip().upper()
|
|
if v.startswith("YES"):
|
|
return "YES"
|
|
if v.startswith("NO"):
|
|
return "NO"
|
|
if v.startswith("AMB"):
|
|
return "AMBIGUOUS"
|
|
if v in ("N/A", "NA", ""):
|
|
return "N/A"
|
|
return v
|
|
|
|
|
|
def machine_ref_kind(m):
|
|
rt = (m.get("reference_type") or "").upper()
|
|
if "EQH" in rt:
|
|
return "EQH"
|
|
if "EQL" in rt:
|
|
return "EQL"
|
|
if "SWING" in rt or "INTERNAL" in rt:
|
|
return "SWING"
|
|
return "NONE"
|
|
|
|
|
|
def machine_dir(m):
|
|
prim = m.get("primary_primitive", "")
|
|
if prim == "f10_eqh":
|
|
return "BEARISH"
|
|
if prim == "f11_eql":
|
|
return "BULLISH"
|
|
if prim == "f7_grab":
|
|
return "BEARISH" if int(m.get("f7_corrected", 0)) < 0 else "BULLISH"
|
|
return "NONE"
|
|
|
|
|
|
def machine_rej(m):
|
|
r = (m.get("rejection") or "").upper()
|
|
if r.startswith("YES"):
|
|
return "YES"
|
|
if r.startswith("N/A"):
|
|
return "N/A"
|
|
return "N/A"
|
|
|
|
|
|
def human_ref_kind(h):
|
|
r = (h.get("reference") or "").strip().upper()
|
|
if r in ("EQH",):
|
|
return "EQH"
|
|
if r in ("EQL",):
|
|
return "EQL"
|
|
if r in ("SWING HIGH", "SWING HIGH", "SWING"):
|
|
return "SWING"
|
|
return r or "NONE"
|
|
|
|
|
|
def conf_matrix(yt, yp):
|
|
tp = fp = fn = tn = 0
|
|
for t, p in zip(yt, yp):
|
|
if t == "YES" and p == "YES":
|
|
tp += 1
|
|
elif t == "NO" and p == "YES":
|
|
fp += 1
|
|
elif t == "YES" and p == "NO":
|
|
fn += 1
|
|
elif t == "NO" and p == "NO":
|
|
tn += 1
|
|
prec = tp / (tp + fp) if tp + fp else float("nan")
|
|
rec = tp / (tp + fn) if tp + fn else float("nan")
|
|
f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else float("nan")
|
|
return {"tp": tp, "fp": fp, "fn": fn, "tn": tn,
|
|
"precision": round(prec, 4), "recall": round(rec, 4), "f1": round(f1, 4)}
|
|
|
|
|
|
def dist(counter):
|
|
return {k: int(v) for k, v in counter.items()}
|
|
|
|
|
|
def interrater(hA, hB, machine, rep):
|
|
common = sorted(set(hA) & set(hB))
|
|
res = {"n_common": len(common)}
|
|
yn_agree = ref_agree = dir_agree = tf_agree = rej_agree = 0
|
|
confA, confB = {}, {}
|
|
disagree = []
|
|
for cid in common:
|
|
a, b = hA[cid], hB[cid]
|
|
va, vb = norm_yes_no(a.get("liquidity_sweep")), norm_yes_no(b.get("liquidity_sweep"))
|
|
if va == vb:
|
|
yn_agree += 1
|
|
else:
|
|
disagree.append({"case_id": cid, "A": va, "B": vb})
|
|
ra, rb = human_ref_kind(a), human_ref_kind(b)
|
|
if ra == rb:
|
|
ref_agree += 1
|
|
da, db = (a.get("direction") or "").strip().upper(), (b.get("direction") or "").strip().upper()
|
|
if da == db:
|
|
dir_agree += 1
|
|
ta, tb = (a.get("timeframe") or "").strip().upper(), (b.get("timeframe") or "").strip().upper()
|
|
if ta == tb:
|
|
tf_agree += 1
|
|
rja, rjb = norm_rej(a.get("close_back_rejection")), norm_rej(b.get("close_back_rejection"))
|
|
if rja == rjb:
|
|
rej_agree += 1
|
|
confA[a.get("confidence", "").upper()] = confA.get(a.get("confidence", "").upper(), 0) + 1
|
|
confB[b.get("confidence", "").upper()] = confB.get(b.get("confidence", "").upper(), 0) + 1
|
|
n = len(common)
|
|
res.update({
|
|
"yes_no_agreement_rate": round(yn_agree / n, 4) if n else None,
|
|
"reference_agreement_rate": round(ref_agree / n, 4) if n else None,
|
|
"direction_agreement_rate": round(dir_agree / n, 4) if n else None,
|
|
"timeframe_agreement_rate": round(tf_agree / n, 4) if n else None,
|
|
"rejection_agreement_rate": round(rej_agree / n, 4) if n else None,
|
|
"confidence_dist_A": dist(confA),
|
|
"confidence_dist_B": dist(confB),
|
|
"n_disagreement": len(disagree),
|
|
"disagreement_cases": disagree,
|
|
"adjudication_required": len(disagree) > 0,
|
|
})
|
|
return res
|
|
|
|
|
|
def machine_vs_consensus(human, machine, rep):
|
|
common = sorted(set(human) & set(machine))
|
|
yt, yp = [], []
|
|
amb = []
|
|
for cid in common:
|
|
t = norm_yes_no(human[cid].get("liquidity_sweep"))
|
|
p = norm_yes_no(machine[cid]["machine_decision"])
|
|
if t == "AMBIGUOUS":
|
|
amb.append(cid)
|
|
continue
|
|
if t is None or p is None:
|
|
continue
|
|
yt.append(t)
|
|
yp.append(p)
|
|
cm = conf_matrix(yt, yp)
|
|
res = {"n_common": len(common), "n_evaluated": len(yt),
|
|
"n_ambiguous_human": len(amb), "ambiguous_cases": amb, **cm}
|
|
|
|
# semantic levels utk kasus L1 (human YES & machine YES)
|
|
both_yes = [c for c in common
|
|
if norm_yes_no(human[c].get("liquidity_sweep")) == "YES" and
|
|
machine[c]["machine_decision"] == "YES"]
|
|
l2 = l3 = 0
|
|
false_agree = []
|
|
tf_mismatch = []
|
|
rej_agree = 0
|
|
for cid in both_yes:
|
|
h, m = human[cid], machine[cid]
|
|
hk, mk = human_ref_kind(h), machine_ref_kind(m)
|
|
hd, md = (h.get("direction") or "").strip().upper(), machine_dir(m)
|
|
ht, mt = (h.get("timeframe") or "").strip().upper(), m.get("timeframe", "M15").upper()
|
|
hr, mr = norm_rej(h.get("close_back_rejection")), machine_rej(m)
|
|
struct = (hk == mk and hd == md and ht == mt)
|
|
rej_ok = (hr == mr or mr == "N/A")
|
|
if struct:
|
|
l2 += 1
|
|
if rej_ok:
|
|
l3 += 1
|
|
if hk != mk:
|
|
false_agree.append({"case_id": cid, "human_ref": h.get("reference"),
|
|
"machine_ref": m.get("reference_type")})
|
|
if ht != mt:
|
|
tf_mismatch.append({"case_id": cid, "human_tf": ht, "machine_tf": mt})
|
|
if rej_ok:
|
|
rej_agree += 1
|
|
res["semantic_levels"] = {
|
|
"L1_outcome_agreement": len(both_yes),
|
|
"L2_structural_agreement": l2,
|
|
"L3_semantic_agreement": l3,
|
|
"L3_rate_of_L1": round(l3 / len(both_yes), 4) if both_yes else None,
|
|
}
|
|
res["semantic_false_agreement"] = false_agree
|
|
res["timeframe_mismatch"] = tf_mismatch
|
|
res["rejection_agreement_both_yes"] = {"agree": rej_agree,
|
|
"n_both_yes": len(both_yes)}
|
|
# rejection summary semua kasus
|
|
rej_table = []
|
|
for cid in common:
|
|
hr = norm_rej(human[cid].get("close_back_rejection"))
|
|
mr = machine_rej(machine[cid])
|
|
rej_table.append({"case_id": cid, "human_rejection": hr, "machine_rejection": mr,
|
|
"agree": hr == mr or mr == "N/A"})
|
|
res["rejection_table"] = rej_table
|
|
# timeframe mismatch grouping
|
|
grp = {}
|
|
for x in tf_mismatch:
|
|
key = f"{x['human_tf']} vs {x['machine_tf']}"
|
|
grp[key] = grp.get(key, 0) + 1
|
|
res["timeframe_mismatch_grouped"] = grp
|
|
return res
|
|
|
|
|
|
def main():
|
|
hA = load_csv(os.path.join(OUT, "human_A_f7.csv"))
|
|
hB = load_csv(os.path.join(OUT, "human_B_f7.csv"))
|
|
hAdj = load_csv(os.path.join(OUT, "human_adjudicated_f7.csv"))
|
|
machine = load_csv(os.path.join(OUT, "machine_annotations_f7_v2.csv"))
|
|
if machine is None:
|
|
raise SystemExit("machine_annotations_f7_v2.csv belum ada — jalankan machine_annotator.py")
|
|
|
|
rep = {"generated_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"machine_version": MACHINE_VERSION, "source_commit": SOURCE_COMMIT,
|
|
"machine_annotation_sha": sha256_file(os.path.join(OUT, "machine_annotations_f7_v2.csv")),
|
|
"case_set_sha": sha256_file(os.path.join(OUT, "cases_meta.json")),
|
|
"human_A_sha": sha256_file(os.path.join(OUT, "human_A_f7.csv")) if hA else None,
|
|
"human_B_sha": sha256_file(os.path.join(OUT, "human_B_f7.csv")) if hB else None,
|
|
"adjudicated_sha": sha256_file(os.path.join(OUT, "human_adjudicated_f7.csv")) if hAdj else None}
|
|
|
|
# ---------- INTER-RATER (Human A vs Human B) ----------
|
|
if hA and hB:
|
|
rep["human_vs_human"] = interrater(hA, hB, machine, rep)
|
|
with open(os.path.join(OUT, "human_interrater_f7_report.json"), "w",
|
|
encoding="utf-8") as f:
|
|
json.dump(rep["human_vs_human"], f, indent=2, default=str)
|
|
print("HUMAN A vs HUMAN B (inter-rater):")
|
|
hv = rep["human_vs_human"]
|
|
for k in ("yes_no_agreement_rate", "reference_agreement_rate",
|
|
"direction_agreement_rate", "timeframe_agreement_rate",
|
|
"rejection_agreement_rate", "n_disagreement"):
|
|
print(f" {k}: {hv.get(k)}")
|
|
if hv["disagreement_cases"]:
|
|
print(" ADJUDICATION REQUIRED:", [d["case_id"] for d in hv["disagreement_cases"]])
|
|
else:
|
|
rep["human_vs_human"] = {"note": "human_A_f7.csv / human_B_f7.csv belum ada"}
|
|
|
|
# ---------- MACHINE vs CONSENSUS ----------
|
|
if hAdj:
|
|
consensus = hAdj
|
|
rep["consensus_source"] = "human_adjudicated_f7.csv"
|
|
elif hA:
|
|
consensus = hA
|
|
rep["consensus_source"] = "human_A_f7.csv (NO adjudication yet)"
|
|
else:
|
|
consensus = None
|
|
rep["consensus_source"] = None
|
|
|
|
if consensus:
|
|
rep["machine_vs_human"] = machine_vs_consensus(consensus, machine, rep)
|
|
with open(os.path.join(OUT, "human_machine_f7_comparison.json"), "w",
|
|
encoding="utf-8") as f:
|
|
json.dump(rep["machine_vs_human"], f, indent=2, default=str)
|
|
mv = rep["machine_vs_human"]
|
|
print("\nMACHINE v2 vs HUMAN CONSENSUS:")
|
|
for k in ("n_evaluated", "tp", "fp", "fn", "tn", "precision", "recall", "f1"):
|
|
print(f" {k}: {mv.get(k)}")
|
|
print(" levels:", mv.get("semantic_levels"))
|
|
print(" false agreement:", len(mv.get("semantic_false_agreement", [])))
|
|
print(" timeframe mismatch:", mv.get("timeframe_mismatch_grouped"))
|
|
else:
|
|
rep["machine_vs_human"] = {"note": "menunggu human annotation"}
|
|
|
|
print("\n[saved] human_interrater_f7_report.json + human_machine_f7_comparison.json "
|
|
"(jika human ada)")
|
|
print("STATUS: " + ("MENUNGGU HUMAN ANNOTATION" if consensus is None else "COMPARISON DONE"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|