Tim Lingo
55f9ee3cb0
hit@5 68.6% -> 74.3%, phrase 71.4% -> 100%, MRR@10 0.461 -> 0.502, latency p50 0.97x. Zero losses, zero run-to-run drift on both sides. 2 queries moved against a 6-query noise floor: NO MEASURABLE DIFFERENCE by the harness's own test (McNemar exact p=0.50). Unaddressable records in returned slots: 57 -> 0. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
147 lines
3.2 KiB
JSON
147 lines
3.2 KiB
JSON
{
|
|
"baseline": "semseed",
|
|
"candidate": "bm25lex",
|
|
"n_shared_queries": 38,
|
|
"fixed_by_candidate": [
|
|
"q10",
|
|
"q11"
|
|
],
|
|
"broken_by_candidate": [],
|
|
"discordant": 2,
|
|
"net_queries": 2,
|
|
"mcnemar_exact_p": 0.5,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.6857142857142857,
|
|
"recall@5": 0.5213459159887731,
|
|
"recall@10": 0.6027048348476919,
|
|
"precision@5": 0.18285714285714294,
|
|
"mrr@10": 0.4608730158730158,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1227.1,
|
|
"latency_ms_p95": 1692.6,
|
|
"latency_ms_max": 1710.4,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.07342657342657344,
|
|
"recall@10": 0.24825174825174823,
|
|
"mrr@10": 0.20833333333333334
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 0.7142857142857143,
|
|
"recall@5": 0.40093537414965985,
|
|
"recall@10": 0.5150226757369615,
|
|
"mrr@10": 0.6507936507936508
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.7428571428571429,
|
|
"recall@5": 0.5536485340056769,
|
|
"recall@10": 0.6175677497106068,
|
|
"precision@5": 0.20000000000000007,
|
|
"mrr@10": 0.5021428571428571,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1184.4,
|
|
"latency_ms_p95": 1620.0,
|
|
"latency_ms_max": 1655.4,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {
|
|
"baseline": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.6857142857142857,
|
|
"hit@5_max": 0.6857142857142857,
|
|
"spread_queries": 0
|
|
},
|
|
"candidate": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.7428571428571429,
|
|
"hit@5_max": 0.7428571428571429,
|
|
"spread_queries": 0
|
|
}
|
|
}
|
|
} |