Tim Lingo
21d3516426
gains q14,q25 (gold at global cosine rank 1, previously discarded by the 0.60 floor); losses q15,q28 (the semantic leg was EMPTY on those queries under the floor, so filling it turns a 2-leg rotation into a 3-leg one and halves the associative leg's share of the top 5). Guards held: exact_rare 6/6, phrase 7/7, nonsense 2/3, superseded 2/3 outranks. recall@10 61.8->65.4pp, latency 0.99x. Baseline reproduced from source (results-bm25base-rerun.json is byte-identical to the committed results-bm25lex.json), candidate deterministic across 2 runs.
150 lines
3.2 KiB
JSON
150 lines
3.2 KiB
JSON
{
|
|
"baseline": "bm25lex",
|
|
"candidate": "claim24",
|
|
"n_shared_queries": 38,
|
|
"fixed_by_candidate": [
|
|
"q14",
|
|
"q25"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q15",
|
|
"q28"
|
|
],
|
|
"discordant": 4,
|
|
"net_queries": 0,
|
|
"mcnemar_exact_p": 1.0,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.7428571428571429,
|
|
"recall@5": 0.5536485340056769,
|
|
"recall@10": 0.6175677497106068,
|
|
"precision@5": 0.20000000000000007,
|
|
"mrr@10": 0.5021428571428571,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1184.4,
|
|
"latency_ms_p95": 1620.0,
|
|
"latency_ms_max": 1655.4,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.7428571428571429,
|
|
"recall@5": 0.5768475572047,
|
|
"recall@10": 0.6537440733869305,
|
|
"precision@5": 0.19428571428571437,
|
|
"mrr@10": 0.5021428571428572,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1173.7,
|
|
"latency_ms_p95": 1623.0,
|
|
"latency_ms_max": 1647.9,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.5,
|
|
"recall@5": 0.07575757575757576,
|
|
"recall@10": 0.12121212121212122,
|
|
"mrr@10": 0.22916666666666666
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6923076923076923,
|
|
"recall@5": 0.6923076923076923,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.29423076923076924
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5335884353741497,
|
|
"recall@10": 0.5933956916099773,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {
|
|
"baseline": {
|
|
"runs": 3,
|
|
"hit@5_min": 0.7428571428571429,
|
|
"hit@5_max": 0.7428571428571429,
|
|
"spread_queries": 0
|
|
},
|
|
"candidate": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.7428571428571429,
|
|
"hit@5_max": 0.7428571428571429,
|
|
"spread_queries": 0
|
|
}
|
|
}
|
|
} |