Tim Lingo
9717a4eeaf
Measured on the 75-query extended gold set (iteration 8's held-out extension) against the certified stack baseline results-stack-ext.json, on the embedded corpus. Three runs of the candidate, zero drift. A. CLAIM 24 WITHOUT THE THRESHOLD - net +4, NOT-SHOWN, kept in the tree. fixed : q14, q25 (in-sample paraphrase), q43, q52, q63, q67 (held-out) broken : q15 (paraphrase), q28 (associative) 8 discordant, McNemar exact p = 0.2891, floor is 6. heldout_paraphrase 16.7% -> 30.0%, paraphrase 61.5% -> 69.2%. Every regression guard held: exact_rare 6/6, phrase 7/7, nonsense 10/10, superseded 2/3. Latency FLAT: p50 641 -> 632 ms. The in-sample half (+q14 +q25 -q15 -q28 = 0) was already on record in iteration 7's cmp-nogate.json, so only the held-out +4 is new. B. ASYMMETRIC TASK PREFIXES ON THE EMBEDDER - net -5, REVERTED in this commit. Rationale was sound and the prediction was wrong, which is why it was worth measuring: nomic-embed-text is an asymmetric retrieval encoder and this file embedded query and document bare on both sides. Prefixing does exactly what the model card implies for the far-away cases - it rescued q42 (gold at GLOBAL COSINE RANK 25,564) and q39 - but it re-ranks the whole space and broke more than it fixed: fixed : q24, q39, q42 broken : q18, q19, q22, q31, q43, q44, q52, q63 heldout_paraphrase 30.0% -> 23.3%, paraphrase 69.2% -> 53.8%. The corpus and the reproducer are kept (embed-corpus-prefixed.py, snapshot-pre-repair-20260806-embedded-prefixed.json) so nobody re-runs it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
155 lines
3.3 KiB
JSON
155 lines
3.3 KiB
JSON
{
|
|
"baseline": "stack-ext",
|
|
"candidate": "unfloor-clean",
|
|
"n_shared_queries": 75,
|
|
"fixed_by_candidate": [
|
|
"q14",
|
|
"q25",
|
|
"q43",
|
|
"q52",
|
|
"q63",
|
|
"q67"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q15",
|
|
"q28"
|
|
],
|
|
"discordant": 8,
|
|
"net_queries": 4,
|
|
"mcnemar_exact_p": 0.2890625,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.47692307692307695,
|
|
"recall@5": 0.3750415183107491,
|
|
"recall@10": 0.45561340369032677,
|
|
"precision@5": 0.12307692307692313,
|
|
"mrr@10": 0.3055555555555555,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 640.9,
|
|
"latency_ms_p95": 1011.1,
|
|
"latency_ms_max": 1190.3,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.16666666666666666,
|
|
"recall@5": 0.16666666666666666,
|
|
"recall@10": 0.26666666666666666,
|
|
"mrr@10": 0.0762037037037037
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.5384615384615384,
|
|
"recall@5": 0.44907176157176154,
|
|
"recall@10": 0.5380300255300255,
|
|
"precision@5": 0.13230769230769232,
|
|
"mrr@10": 0.32437728937728944,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 632.5,
|
|
"latency_ms_p95": 992.5,
|
|
"latency_ms_max": 1177.8,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.5,
|
|
"recall@5": 0.07575757575757576,
|
|
"recall@10": 0.13636363636363635,
|
|
"mrr@10": 0.23214285714285712
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.3,
|
|
"recall@5": 0.3,
|
|
"recall@10": 0.4,
|
|
"mrr@10": 0.11638888888888889
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6923076923076923,
|
|
"recall@5": 0.6923076923076923,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.29423076923076924
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5335884353741497,
|
|
"recall@10": 0.5933956916099773,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {}
|
|
} |