neuron/tools/retrieval-eval/compare.py

#!/usr/bin/env python3
"""
compare.py — diff two run_eval.py result files, WITH a noise threshold.

WHY THE STATISTICS ARE NOT OPTIONAL
    With ~35 scored queries, one query is ~2.9 percentage points. A harness that
    reports "hit@5 improved 2.9%" without saying that is one query is a harness
    that will approve noise. So this file refuses to call anything an
    improvement on the strength of the headline number alone. It reports:

      1. The DISCORDANT PAIRS. Two configurations scored on the same queries are
         paired data, so the only queries carrying information are the ones
         where they disagree: b = fixed by B, c = broken by B. Queries both got
         right, or both got wrong, tell you nothing about which is better.

      2. McNEMAR'S EXACT TEST on (b, c). Under the null "the change is a coin
         flip", the discordant outcomes are Binomial(b+c, 0.5). The two-sided
         exact p-value is computed here with no scipy dependency.

      3. The MINIMUM DETECTABLE SWING for this gold set: the smallest number of
         net-changed queries that would reach p < 0.05 if every discordant pair
         fell the same way. Anything smaller is inside the noise band, and the
         verdict line says so in those words.

    Repeat-run variance is the other half of honesty. Spreading activation is a
    stateful read (it reinforces what it touches), so identical inputs need not
    give identical outputs. Pass --repeats to fold several runs of the same
    config into an observed variance band; a delta inside that band is not real
    either, however good its p-value looks.

usage:
  python3 compare.py --baseline results-main.json --candidate results-act.json
  python3 compare.py --baseline a.json --candidate b.json \
      --repeats-baseline a2.json a3.json --repeats-candidate b2.json b3.json
"""
import argparse
import json
from math import comb


def binom_two_sided(b, c):
    """Two-sided exact binomial p for b successes in n=b+c at p=0.5."""
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n)
    return min(1.0, 2 * tail)


def min_detectable_swing(n_scored, alpha=0.05):
    """Smallest all-one-way discordant count reaching p < alpha.

    If every query that changes changes in the same direction, the p-value is
    2 * 0.5**n. Solve for the smallest n where that drops under alpha. This is
    the FLOOR: any real change will have some discordance both ways, so the true
    requirement is larger. Reporting the floor is the conservative move — it is
    the most generous threshold we would ever accept.
    """
    n = 1
    while n <= n_scored:
        if 2 * (0.5 ** n) < alpha:
            return n
        n += 1
    return n_scored


def load(path):
    with open(path, encoding="utf-8") as fh:
        return json.load(fh)


def row_map(doc):
    return {r["id"]: r for r in doc["rows"]}


def outcome(r):
    """Binary per-query outcome used for the paired test.

    hit@5 for scored queries; 'returned nothing' for the nonsense controls;
    'correction outranks the stale node' for the superseded queries. One number
    per query, so every query votes exactly once.
    """
    if "clean" in r:
        return 1.0 if r["clean"] else 0.0
    if "outranks" in r:
        return 1.0 if r["outranks"] else 0.0
    return r.get("hit@5") or 0.0


def band(values):
    return (min(values), max(values))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--baseline", required=True)
    ap.add_argument("--candidate", required=True)
    ap.add_argument("--repeats-baseline", nargs="*", default=[])
    ap.add_argument("--repeats-candidate", nargs="*", default=[])
    ap.add_argument("--out", default=None)
    args = ap.parse_args()

    A, B = load(args.baseline), load(args.candidate)
    ra, rb = row_map(A), row_map(B)
    ids = [q for q in ra if q in rb]
    n = len(ids)

    aa, ab = A["aggregate"], B["aggregate"]
    print(f"baseline  {A['label']:14} soul={A['soul_md5'][:12]}  {n} shared queries")
    print(f"candidate {B['label']:14} soul={B['soul_md5'][:12]}")
    print(f"corpus    {A['corpus_nodes']} nodes / {A['corpus_edges']} edges "
          f"(identical copy for both runs)\n")

    metrics = [("hit@5", 1), ("recall@5", 1), ("recall@10", 1),
               ("precision@5", 1), ("mrr@10", 0)]
    print(f"  {'metric':14} {'baseline':>10} {'candidate':>10} {'delta':>10}")
    for m, as_pct in metrics:
        x, y = aa[m], ab[m]
        if as_pct:
            print(f"  {m:14} {100*x:>9.1f}% {100*y:>9.1f}% {100*(y-x):>+9.1f}pp")
        else:
            print(f"  {m:14} {x:>10.3f} {y:>10.3f} {y-x:>+10.3f}")
    for m in ("latency_ms_p50", "latency_ms_p95"):
        x, y = aa[m], ab[m]
        ratio = f"{y/x:.2f}x" if x else "n/a"
        print(f"  {m:14} {x:>9.0f}ms {y:>9.0f}ms {ratio:>10}")
    print(f"  {'nonsense':14} {aa['nonsense_clean']:>10} {ab['nonsense_clean']:>10}")
    print(f"  {'outranks':14} {aa['superseded_outranks']:>10} {ab['superseded_outranks']:>10}")

    print(f"\n  {'category':14} {'n':>3} {'base hit@5':>11} {'cand hit@5':>11} {'delta':>9}")
    for c in sorted(set(aa["by_category"]) & set(ab["by_category"])):
        ea, eb = aa["by_category"][c], ab["by_category"][c]
        if c == "nonsense":
            print(f"  {c:14} {ea['n']:>3} {'clean ' + str(ea['clean']):>11} "
                  f"{'clean ' + str(eb['clean']):>11}")
        else:
            print(f"  {c:14} {ea['n']:>3} {100*ea['hit@5']:>10.1f}% {100*eb['hit@5']:>10.1f}% "
                  f"{100*(eb['hit@5']-ea['hit@5']):>+8.1f}pp")

    # ---- paired significance -------------------------------------------------
    fixed, broken = [], []
    for q in ids:
        oa, ob = outcome(ra[q]), outcome(rb[q])
        if ob > oa:
            fixed.append(q)
        elif ob < oa:
            broken.append(q)
    b, c = len(fixed), len(broken)
    p = binom_two_sided(b, c)
    mds = min_detectable_swing(n)

    print(f"\n== paired comparison over {n} queries ==")
    print(f"  fixed by candidate : {b}  {[ra[q]['category'] + ':' + q for q in fixed]}")
    print(f"  broken by candidate: {c}  {[ra[q]['category'] + ':' + q for q in broken]}")
    print(f"  discordant pairs   : {b + c}   net {b - c:+d} queries")
    print(f"  McNemar exact p    : {p:.4f}")
    print(f"  noise threshold    : a difference needs at least {mds} queries moving the "
          f"same way to clear p<0.05 on this {n}-query set")

    # ---- repeat-run variance -------------------------------------------------
    var = {}
    for name, paths, first in (("baseline", args.repeats_baseline, A),
                               ("candidate", args.repeats_candidate, B)):
        docs = [first] + [load(p) for p in paths]
        if len(docs) > 1:
            hits = [d["aggregate"]["hit@5"] for d in docs]
            lo, hi = band(hits)
            spread_q = round((hi - lo) * first["aggregate"]["n_scored"])
            var[name] = {"runs": len(docs), "hit@5_min": lo, "hit@5_max": hi,
                         "spread_queries": spread_q}
            print(f"  {name} repeat runs ({len(docs)}): hit@5 {100*lo:.1f}%..{100*hi:.1f}% "
                  f"= {spread_q} query of run-to-run drift")

    drift = max([v["spread_queries"] for v in var.values()], default=0)
    floor = max(mds, drift + 1)

    print("\n== VERDICT ==")
    net = b - c
    if abs(net) < floor:
        print(f"  NO MEASURABLE DIFFERENCE. Net {net:+d} queries is inside the noise band "
              f"(needs |net| >= {floor}: {mds} for significance, {drift} observed run-to-run drift).")
    elif net > 0:
        print(f"  CANDIDATE BETTER by {net} queries (p={p:.4f}), outside the noise band "
              f"(>= {floor}).")
    else:
        print(f"  CANDIDATE WORSE by {abs(net)} queries (p={p:.4f}), outside the noise band "
              f"(>= {floor}).")

    if args.out:
        with open(args.out, "w", encoding="utf-8") as fh:
            json.dump({
                "baseline": A["label"], "candidate": B["label"],
                "n_shared_queries": n,
                "fixed_by_candidate": fixed, "broken_by_candidate": broken,
                "discordant": b + c, "net_queries": net,
                "mcnemar_exact_p": p,
                "min_detectable_swing_queries": mds,
                "observed_run_to_run_drift_queries": drift,
                "noise_floor_queries": floor,
                "verdict": ("no measurable difference" if abs(net) < floor
                            else ("candidate better" if net > 0 else "candidate worse")),
                "baseline_aggregate": aa, "candidate_aggregate": ab,
                "repeat_variance": var,
            }, fh, indent=1)
        print(f"\nwrote {args.out}")


if __name__ == "__main__":
    main()