Tim Lingo
9717a4eeaf
Measured on the 75-query extended gold set (iteration 8's held-out extension) against the certified stack baseline results-stack-ext.json, on the embedded corpus. Three runs of the candidate, zero drift. A. CLAIM 24 WITHOUT THE THRESHOLD - net +4, NOT-SHOWN, kept in the tree. fixed : q14, q25 (in-sample paraphrase), q43, q52, q63, q67 (held-out) broken : q15 (paraphrase), q28 (associative) 8 discordant, McNemar exact p = 0.2891, floor is 6. heldout_paraphrase 16.7% -> 30.0%, paraphrase 61.5% -> 69.2%. Every regression guard held: exact_rare 6/6, phrase 7/7, nonsense 10/10, superseded 2/3. Latency FLAT: p50 641 -> 632 ms. The in-sample half (+q14 +q25 -q15 -q28 = 0) was already on record in iteration 7's cmp-nogate.json, so only the held-out +4 is new. B. ASYMMETRIC TASK PREFIXES ON THE EMBEDDER - net -5, REVERTED in this commit. Rationale was sound and the prediction was wrong, which is why it was worth measuring: nomic-embed-text is an asymmetric retrieval encoder and this file embedded query and document bare on both sides. Prefixing does exactly what the model card implies for the far-away cases - it rescued q42 (gold at GLOBAL COSINE RANK 25,564) and q39 - but it re-ranks the whole space and broke more than it fixed: fixed : q24, q39, q42 broken : q18, q19, q22, q31, q43, q44, q52, q63 heldout_paraphrase 30.0% -> 23.3%, paraphrase 69.2% -> 53.8%. The corpus and the reproducer are kept (embed-corpus-prefixed.py, snapshot-pre-repair-20260806-embedded-prefixed.json) so nobody re-runs it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
158 lines
3.4 KiB
JSON
158 lines
3.4 KiB
JSON
{
|
|
"baseline": "unfloor-clean",
|
|
"candidate": "semsub",
|
|
"n_shared_queries": 75,
|
|
"fixed_by_candidate": [
|
|
"q24",
|
|
"q39",
|
|
"q42"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q18",
|
|
"q19",
|
|
"q22",
|
|
"q31",
|
|
"q43",
|
|
"q44",
|
|
"q52",
|
|
"q63"
|
|
],
|
|
"discordant": 11,
|
|
"net_queries": -5,
|
|
"mcnemar_exact_p": 0.2265625,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.5384615384615384,
|
|
"recall@5": 0.44907176157176154,
|
|
"recall@10": 0.5380300255300255,
|
|
"precision@5": 0.13230769230769232,
|
|
"mrr@10": 0.32437728937728944,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 632.5,
|
|
"latency_ms_p95": 992.5,
|
|
"latency_ms_max": 1177.8,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.5,
|
|
"recall@5": 0.07575757575757576,
|
|
"recall@10": 0.13636363636363635,
|
|
"mrr@10": 0.23214285714285712
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.3,
|
|
"recall@5": 0.3,
|
|
"recall@10": 0.4,
|
|
"mrr@10": 0.11638888888888889
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6923076923076923,
|
|
"recall@5": 0.6923076923076923,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.29423076923076924
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5335884353741497,
|
|
"recall@10": 0.5933956916099773,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.46153846153846156,
|
|
"recall@5": 0.3889430014430015,
|
|
"recall@10": 0.4887681762681762,
|
|
"precision@5": 0.12307692307692313,
|
|
"mrr@10": 0.30181318681318675,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 634.4,
|
|
"latency_ms_p95": 988.3,
|
|
"latency_ms_max": 1184.5,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.06060606060606061,
|
|
"recall@10": 0.12121212121212122,
|
|
"mrr@10": 0.19047619047619047
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.23333333333333334,
|
|
"recall@5": 0.23333333333333334,
|
|
"recall@10": 0.3,
|
|
"mrr@10": 0.08925925925925927
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.5384615384615384,
|
|
"recall@5": 0.5384615384615384,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.26324786324786326
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5596655328798186,
|
|
"recall@10": 0.5775226757369615,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {}
|
|
} |