{
 "baseline": "unfloor-clean",
 "candidate": "semsub",
 "n_shared_queries": 75,
 "fixed_by_candidate": [
  "q24",
  "q39",
  "q42"
 ],
 "broken_by_candidate": [
  "q18",
  "q19",
  "q22",
  "q31",
  "q43",
  "q44",
  "q52",
  "q63"
 ],
 "discordant": 11,
 "net_queries": -5,
 "mcnemar_exact_p": 0.2265625,
 "min_detectable_swing_queries": 6,
 "observed_run_to_run_drift_queries": 0,
 "noise_floor_queries": 6,
 "verdict": "no measurable difference",
 "baseline_aggregate": {
  "n_queries": 75,
  "n_scored": 65,
  "hit@5": 0.5384615384615384,
  "recall@5": 0.44907176157176154,
  "recall@10": 0.5380300255300255,
  "precision@5": 0.13230769230769232,
  "mrr@10": 0.32437728937728944,
  "nonsense_clean": "10/10",
  "superseded_outranks": "2/3",
  "latency_ms_p50": 632.5,
  "latency_ms_p95": 992.5,
  "latency_ms_max": 1177.8,
  "errors": 0,
  "by_category": {
   "associative": {
    "n": 6,
    "hit@5": 0.5,
    "recall@5": 0.07575757575757576,
    "recall@10": 0.13636363636363635,
    "mrr@10": 0.23214285714285712
   },
   "exact_rare": {
    "n": 6,
    "hit@5": 1.0,
    "recall@5": 1.0,
    "recall@10": 1.0,
    "mrr@10": 1.0
   },
   "heldout_paraphrase": {
    "n": 30,
    "hit@5": 0.3,
    "recall@5": 0.3,
    "recall@10": 0.4,
    "mrr@10": 0.11638888888888889
   },
   "nonsense": {
    "n": 10,
    "clean": 10,
    "avg_false_positives": 0.0
   },
   "paraphrase": {
    "n": 13,
    "hit@5": 0.6923076923076923,
    "recall@5": 0.6923076923076923,
    "recall@10": 0.7692307692307693,
    "mrr@10": 0.29423076923076924
   },
   "phrase": {
    "n": 7,
    "hit@5": 1.0,
    "recall@5": 0.5335884353741497,
    "recall@10": 0.5933956916099773,
    "mrr@10": 0.8214285714285714
   },
   "superseded": {
    "n": 3,
    "hit@5": 0.3333333333333333,
    "recall@5": 0.3333333333333333,
    "recall@10": 0.6666666666666666,
    "mrr@10": 0.20833333333333334,
    "outranks": 2
   }
  }
 },
 "candidate_aggregate": {
  "n_queries": 75,
  "n_scored": 65,
  "hit@5": 0.46153846153846156,
  "recall@5": 0.3889430014430015,
  "recall@10": 0.4887681762681762,
  "precision@5": 0.12307692307692313,
  "mrr@10": 0.30181318681318675,
  "nonsense_clean": "10/10",
  "superseded_outranks": "2/3",
  "latency_ms_p50": 634.4,
  "latency_ms_p95": 988.3,
  "latency_ms_max": 1184.5,
  "errors": 0,
  "by_category": {
   "associative": {
    "n": 6,
    "hit@5": 0.3333333333333333,
    "recall@5": 0.06060606060606061,
    "recall@10": 0.12121212121212122,
    "mrr@10": 0.19047619047619047
   },
   "exact_rare": {
    "n": 6,
    "hit@5": 1.0,
    "recall@5": 1.0,
    "recall@10": 1.0,
    "mrr@10": 1.0
   },
   "heldout_paraphrase": {
    "n": 30,
    "hit@5": 0.23333333333333334,
    "recall@5": 0.23333333333333334,
    "recall@10": 0.3,
    "mrr@10": 0.08925925925925927
   },
   "nonsense": {
    "n": 10,
    "clean": 10,
    "avg_false_positives": 0.0
   },
   "paraphrase": {
    "n": 13,
    "hit@5": 0.5384615384615384,
    "recall@5": 0.5384615384615384,
    "recall@10": 0.7692307692307693,
    "mrr@10": 0.26324786324786326
   },
   "phrase": {
    "n": 7,
    "hit@5": 1.0,
    "recall@5": 0.5596655328798186,
    "recall@10": 0.5775226757369615,
    "mrr@10": 0.8214285714285714
   },
   "superseded": {
    "n": 3,
    "hit@5": 0.3333333333333333,
    "recall@5": 0.3333333333333333,
    "recall@10": 0.6666666666666666,
    "mrr@10": 0.20833333333333334,
    "outranks": 2
   }
  }
 },
 "repeat_variance": {}
}