Tim Lingo
4eb4c9e287
The retrieval match test is a raw substring scan, so a query token matches anywhere INSIDE a corpus word: "throom" matches "bathroom". Measured over the 38-query gold set on this corpus that is not a rare accident - q28's lexical leg is 36,954 records of which only 13 contain a query token at a word start (99.96% mid-word noise), six further queries carry ~20,500 mid-word-only records each, and the nonsense control q35 returns 7 records ALL of which match only mid-word. istr_contains_wordstart() anchors a token to a word start (preceding char not alphanumeric) while still matching suffixes, so "value" still hits "values". That empties the lexical leg for gibberish, and the nhits==0 corpus-vocabulary gate (iteration 6's mechanism, feat/claim24-unfloored-semantic) then makes the whole query decline rather than let the semantic leg answer it. Measured vs feat/bm25-lexical-leg on the embedded corpus, 2 runs each, 0 queries of run-to-run drift on both sides: net +1 (nonsense:q35), 0 losses, McNemar p=1.0 -> NOT-SHOWN (floor is 6) nonsense clean 2/3 -> 3/3; exact_rare 100%, phrase 100%, paraphrase 61.5%, associative 66.7%, superseded 2/3 all UNCHANGED latency p50 1184 -> 543 ms (0.46x) Iteration 6 called q35 "a DEFECTIVE CONTROL ... cannot be cleaned without breaking the lexical leg". It can: the defect was the match primitive, and cleaning it cost nothing. Also committed: results-wsclaim24.json + cmp-nogate.json, a measured negative for bundling the claim-24 unfloored semantic leg on top (gains q14/q25, breaks q15/q28/q33/q34, net -2) - it independently reproduces iteration 6's q15/q28 losses and shows unflooring REQUIRES the vocabulary gate. Reproducers: legs.py (leg-level replica, reproduces baseline hit@5 exactly on all 38 queries), policy2.py, ceiling.py, wb2.py.
146 lines
3.1 KiB
JSON
146 lines
3.1 KiB
JSON
{
|
|
"baseline": "bm25lex",
|
|
"candidate": "wsclaim24",
|
|
"n_shared_queries": 38,
|
|
"fixed_by_candidate": [
|
|
"q14",
|
|
"q25"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q15",
|
|
"q28",
|
|
"q33",
|
|
"q34"
|
|
],
|
|
"discordant": 6,
|
|
"net_queries": -2,
|
|
"mcnemar_exact_p": 0.6875,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.7428571428571429,
|
|
"recall@5": 0.5536485340056769,
|
|
"recall@10": 0.6175677497106068,
|
|
"precision@5": 0.20000000000000007,
|
|
"mrr@10": 0.5021428571428571,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1184.4,
|
|
"latency_ms_p95": 1620.0,
|
|
"latency_ms_max": 1655.4,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.7428571428571429,
|
|
"recall@5": 0.5768475572047,
|
|
"recall@10": 0.6563414759843332,
|
|
"precision@5": 0.19428571428571437,
|
|
"mrr@10": 0.5026530612244898,
|
|
"nonsense_clean": "0/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 524.8,
|
|
"latency_ms_p95": 738.7,
|
|
"latency_ms_max": 755.8,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.5,
|
|
"recall@5": 0.07575757575757576,
|
|
"recall@10": 0.13636363636363635,
|
|
"mrr@10": 0.23214285714285712
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 0,
|
|
"avg_false_positives": 10.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6923076923076923,
|
|
"recall@5": 0.6923076923076923,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.29423076923076924
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5335884353741497,
|
|
"recall@10": 0.5933956916099773,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {
|
|
"baseline": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.7428571428571429,
|
|
"hit@5_max": 0.7428571428571429,
|
|
"spread_queries": 0
|
|
}
|
|
}
|
|
} |