# Coder agreement for the collection codes (each plan's §5.4 or §5: "Cohen's kappa per family"): model coder A
# (code, p1) against blind model coder B (code2, p1_2), per family and for p1, with observed agreement and every
# disagreement listed for adjudication. Unweighted (nominal) kappa; kappa is undefined (printed n/a) when both coders
# used one and the same code throughout. Standard library only.
#   python -I docs/evidence/research-programme-2026-q4/kappa.py <codes-day1.json>
#   python -I docs/evidence/research-programme-2026-q4/kappa.py --selftest
import json
import sys
from collections import Counter, defaultdict


def cohen_kappa(pairs):
    """Returns (kappa or None, observed agreement, n) for a list of (a, b) labels."""
    n = len(pairs)
    if not n:
        return None, None, 0
    po = sum(1 for a, b in pairs if a == b) / n
    ca, cb = Counter(a for a, _ in pairs), Counter(b for _, b in pairs)
    pe = sum(ca[k] * cb[k] for k in set(ca) | set(cb)) / (n * n)
    if pe == 1:
        return None, po, n
    return (po - pe) / (1 - pe), po, n


def run(doc):
    rows = [r for r in doc["answers"] if r.get("mode", "collect") == "collect"]
    print(f"study {doc['study']}: {len(rows)} answers, coder A (primary) against blind coder B")
    print("\n| Family | n | Agreement | Cohen's kappa | A codes | B codes |")
    print("|---|---|---|---|---|---|")
    by = defaultdict(list)
    for r in rows:
        by[r["family"]].append(r)
    for fam in sorted(by):
        rs = by[fam]
        k, po, n = cohen_kappa([(r["code"], r["code2"]) for r in rs])
        ks = "n/a" if k is None else f"{k:.3f}"
        ca = " ".join(f"{c}{v}" for c, v in sorted(Counter(r["code"] for r in rs).items()))
        cb = " ".join(f"{c}{v}" for c, v in sorted(Counter(r["code2"] for r in rs).items()))
        print(f"| {fam} | {n} | {po:.3f} | {ks} | {ca} | {cb} |")
    k, po, n = cohen_kappa([(r["code"], r["code2"]) for r in rows])
    print(f"| all | {n} | {po:.3f} | {'n/a' if k is None else f'{k:.3f}'} | | |")

    p1rows = [r for r in rows if r.get("p1") is not None or r.get("p1_2") is not None]
    if p1rows:
        k, po, n = cohen_kappa([(str(r.get("p1")), str(r.get("p1_2"))) for r in p1rows])
        print(f"\np1 (registration), answers either coder coded: n {n}, agreement {po:.3f}, kappa {'n/a' if k is None else f'{k:.3f}'}")

    dis = [r for r in rows if r["code"] != r["code2"] or r.get("p1") != r.get("p1_2")]
    print(f"\nDisagreements for adjudication: {len(dis)}")
    for r in sorted(dis, key=lambda r: (r["family"], r["wordingId"], r["engine"], r["day"], r["round"])):
        p = "" if r.get("p1") == r.get("p1_2") else f"  p1 {r.get('p1')} v {r.get('p1_2')}"
        print(f"  {r['codingId']}  {r['family']}/{r['wordingId']}  {r['engine']}  d{r['day']}r{r['round']}  A {r['code']} v B {r['code2']}{p}")


def selftest():
    k, po, n = cohen_kappa([("F", "F")] * 20 + [("C", "C")] * 15 + [("F", "C")] * 5 + [("C", "F")] * 10)
    # Worked by hand: po = 35/50 = 0.7; pe = (25*30 + 25*20) / 2500 = 0.5; kappa = (0.7 - 0.5) / 0.5 = 0.4.
    assert abs(po - 0.7) < 1e-12 and abs(k - 0.4) < 1e-12, (k, po)
    assert cohen_kappa([("C", "C")] * 5)[0] is None
    assert cohen_kappa([])[2] == 0
    print("selftest ok")


if __name__ == "__main__":
    if sys.argv[1:] == ["--selftest"]:
        selftest()
    else:
        run(json.load(open(sys.argv[1], encoding="utf-8")))