#!/usr/bin/env python3 """ compare.py — diff two run_eval.py result files, WITH a noise threshold. WHY THE STATISTICS ARE NOT OPTIONAL With ~35 scored queries, one query is ~2.9 percentage points. A harness that reports "hit@5 improved 2.9%" without saying that is one query is a harness that will approve noise. So this file refuses to call anything an improvement on the strength of the headline number alone. It reports: 1. The DISCORDANT PAIRS. Two configurations scored on the same queries are paired data, so the only queries carrying information are the ones where they disagree: b = fixed by B, c = broken by B. Queries both got right, or both got wrong, tell you nothing about which is better. 2. McNEMAR'S EXACT TEST on (b, c). Under the null "the change is a coin flip", the discordant outcomes are Binomial(b+c, 0.5). The two-sided exact p-value is computed here with no scipy dependency. 3. The MINIMUM DETECTABLE SWING for this gold set: the smallest number of net-changed queries that would reach p < 0.05 if every discordant pair fell the same way. Anything smaller is inside the noise band, and the verdict line says so in those words. Repeat-run variance is the other half of honesty. Spreading activation is a stateful read (it reinforces what it touches), so identical inputs need not give identical outputs. Pass --repeats to fold several runs of the same config into an observed variance band; a delta inside that band is not real either, however good its p-value looks. usage: python3 compare.py --baseline results-main.json --candidate results-act.json python3 compare.py --baseline a.json --candidate b.json \ --repeats-baseline a2.json a3.json --repeats-candidate b2.json b3.json """ import argparse import json from math import comb def binom_two_sided(b, c): """Two-sided exact binomial p for b successes in n=b+c at p=0.5.""" n = b + c if n == 0: return 1.0 k = min(b, c) tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n) return min(1.0, 2 * tail) def min_detectable_swing(n_scored, alpha=0.05): """Smallest all-one-way discordant count reaching p < alpha. If every query that changes changes in the same direction, the p-value is 2 * 0.5**n. Solve for the smallest n where that drops under alpha. This is the FLOOR: any real change will have some discordance both ways, so the true requirement is larger. Reporting the floor is the conservative move — it is the most generous threshold we would ever accept. """ n = 1 while n <= n_scored: if 2 * (0.5 ** n) < alpha: return n n += 1 return n_scored def load(path): with open(path, encoding="utf-8") as fh: return json.load(fh) def row_map(doc): return {r["id"]: r for r in doc["rows"]} def outcome(r): """Binary per-query outcome used for the paired test. hit@5 for scored queries; 'returned nothing' for the nonsense controls; 'correction outranks the stale node' for the superseded queries. One number per query, so every query votes exactly once. """ if "clean" in r: return 1.0 if r["clean"] else 0.0 if "outranks" in r: return 1.0 if r["outranks"] else 0.0 return r.get("hit@5") or 0.0 def band(values): return (min(values), max(values)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--baseline", required=True) ap.add_argument("--candidate", required=True) ap.add_argument("--repeats-baseline", nargs="*", default=[]) ap.add_argument("--repeats-candidate", nargs="*", default=[]) ap.add_argument("--out", default=None) args = ap.parse_args() A, B = load(args.baseline), load(args.candidate) ra, rb = row_map(A), row_map(B) ids = [q for q in ra if q in rb] n = len(ids) aa, ab = A["aggregate"], B["aggregate"] print(f"baseline {A['label']:14} soul={A['soul_md5'][:12]} {n} shared queries") print(f"candidate {B['label']:14} soul={B['soul_md5'][:12]}") print(f"corpus {A['corpus_nodes']} nodes / {A['corpus_edges']} edges " f"(identical copy for both runs)\n") metrics = [("hit@5", 1), ("recall@5", 1), ("recall@10", 1), ("precision@5", 1), ("mrr@10", 0)] print(f" {'metric':14} {'baseline':>10} {'candidate':>10} {'delta':>10}") for m, as_pct in metrics: x, y = aa[m], ab[m] if as_pct: print(f" {m:14} {100*x:>9.1f}% {100*y:>9.1f}% {100*(y-x):>+9.1f}pp") else: print(f" {m:14} {x:>10.3f} {y:>10.3f} {y-x:>+10.3f}") for m in ("latency_ms_p50", "latency_ms_p95"): x, y = aa[m], ab[m] ratio = f"{y/x:.2f}x" if x else "n/a" print(f" {m:14} {x:>9.0f}ms {y:>9.0f}ms {ratio:>10}") print(f" {'nonsense':14} {aa['nonsense_clean']:>10} {ab['nonsense_clean']:>10}") print(f" {'outranks':14} {aa['superseded_outranks']:>10} {ab['superseded_outranks']:>10}") print(f"\n {'category':14} {'n':>3} {'base hit@5':>11} {'cand hit@5':>11} {'delta':>9}") for c in sorted(set(aa["by_category"]) & set(ab["by_category"])): ea, eb = aa["by_category"][c], ab["by_category"][c] if c == "nonsense": print(f" {c:14} {ea['n']:>3} {'clean ' + str(ea['clean']):>11} " f"{'clean ' + str(eb['clean']):>11}") else: print(f" {c:14} {ea['n']:>3} {100*ea['hit@5']:>10.1f}% {100*eb['hit@5']:>10.1f}% " f"{100*(eb['hit@5']-ea['hit@5']):>+8.1f}pp") # ---- paired significance ------------------------------------------------- fixed, broken = [], [] for q in ids: oa, ob = outcome(ra[q]), outcome(rb[q]) if ob > oa: fixed.append(q) elif ob < oa: broken.append(q) b, c = len(fixed), len(broken) p = binom_two_sided(b, c) mds = min_detectable_swing(n) print(f"\n== paired comparison over {n} queries ==") print(f" fixed by candidate : {b} {[ra[q]['category'] + ':' + q for q in fixed]}") print(f" broken by candidate: {c} {[ra[q]['category'] + ':' + q for q in broken]}") print(f" discordant pairs : {b + c} net {b - c:+d} queries") print(f" McNemar exact p : {p:.4f}") print(f" noise threshold : a difference needs at least {mds} queries moving the " f"same way to clear p<0.05 on this {n}-query set") # ---- repeat-run variance ------------------------------------------------- var = {} for name, paths, first in (("baseline", args.repeats_baseline, A), ("candidate", args.repeats_candidate, B)): docs = [first] + [load(p) for p in paths] if len(docs) > 1: hits = [d["aggregate"]["hit@5"] for d in docs] lo, hi = band(hits) spread_q = round((hi - lo) * first["aggregate"]["n_scored"]) var[name] = {"runs": len(docs), "hit@5_min": lo, "hit@5_max": hi, "spread_queries": spread_q} print(f" {name} repeat runs ({len(docs)}): hit@5 {100*lo:.1f}%..{100*hi:.1f}% " f"= {spread_q} query of run-to-run drift") drift = max([v["spread_queries"] for v in var.values()], default=0) floor = max(mds, drift + 1) print("\n== VERDICT ==") net = b - c if abs(net) < floor: print(f" NO MEASURABLE DIFFERENCE. Net {net:+d} queries is inside the noise band " f"(needs |net| >= {floor}: {mds} for significance, {drift} observed run-to-run drift).") elif net > 0: print(f" CANDIDATE BETTER by {net} queries (p={p:.4f}), outside the noise band " f"(>= {floor}).") else: print(f" CANDIDATE WORSE by {abs(net)} queries (p={p:.4f}), outside the noise band " f"(>= {floor}).") if args.out: with open(args.out, "w", encoding="utf-8") as fh: json.dump({ "baseline": A["label"], "candidate": B["label"], "n_shared_queries": n, "fixed_by_candidate": fixed, "broken_by_candidate": broken, "discordant": b + c, "net_queries": net, "mcnemar_exact_p": p, "min_detectable_swing_queries": mds, "observed_run_to_run_drift_queries": drift, "noise_floor_queries": floor, "verdict": ("no measurable difference" if abs(net) < floor else ("candidate better" if net > 0 else "candidate worse")), "baseline_aggregate": aa, "candidate_aggregate": ab, "repeat_variance": var, }, fh, indent=1) print(f"\nwrote {args.out}") if __name__ == "__main__": main()