measure: claim 24 unflooring is +4 (NOT-SHOWN); asymmetric embedding prefixes are -5 (discarded)

Measured on the 75-query extended gold set (iteration 8's held-out extension)
against the certified stack baseline results-stack-ext.json, on the embedded
corpus. Three runs of the candidate, zero drift.

A. CLAIM 24 WITHOUT THE THRESHOLD - net +4, NOT-SHOWN, kept in the tree.
   fixed  : q14, q25 (in-sample paraphrase), q43, q52, q63, q67 (held-out)
   broken : q15 (paraphrase), q28 (associative)
   8 discordant, McNemar exact p = 0.2891, floor is 6.
   heldout_paraphrase 16.7% -> 30.0%, paraphrase 61.5% -> 69.2%.
   Every regression guard held: exact_rare 6/6, phrase 7/7, nonsense 10/10,
   superseded 2/3. Latency FLAT: p50 641 -> 632 ms.
   The in-sample half (+q14 +q25 -q15 -q28 = 0) was already on record in
   iteration 7's cmp-nogate.json, so only the held-out +4 is new.

B. ASYMMETRIC TASK PREFIXES ON THE EMBEDDER - net -5, REVERTED in this commit.
   Rationale was sound and the prediction was wrong, which is why it was worth
   measuring: nomic-embed-text is an asymmetric retrieval encoder and this file
   embedded query and document bare on both sides. Prefixing does exactly what
   the model card implies for the far-away cases - it rescued q42 (gold at
   GLOBAL COSINE RANK 25,564) and q39 - but it re-ranks the whole space and
   broke more than it fixed:
   fixed  : q24, q39, q42
   broken : q18, q19, q22, q31, q43, q44, q52, q63
   heldout_paraphrase 30.0% -> 23.3%, paraphrase 69.2% -> 53.8%.
   The corpus and the reproducer are kept (embed-corpus-prefixed.py,
   snapshot-pre-repair-20260806-embedded-prefixed.json) so nobody re-runs it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tim Lingo
2026-08-07 17:45:05 -05:00
parent 9790d9342d
commit 9717a4eeaf
6 changed files with 5646 additions and 31 deletions
@@ -1,6 +1,6 @@
{
"baseline": "stack-ext",
"candidate": "unfloor-ext",
"candidate": "unfloor-clean",
"n_shared_queries": 75,
"fixed_by_candidate": [
"q14",
@@ -96,9 +96,9 @@
"mrr@10": 0.32437728937728944,
"nonsense_clean": "10/10",
"superseded_outranks": "2/3",
"latency_ms_p50": 993.0,
"latency_ms_p95": 1544.7,
"latency_ms_max": 1809.0,
"latency_ms_p50": 632.5,
"latency_ms_p95": 992.5,
"latency_ms_max": 1177.8,
"errors": 0,
"by_category": {
"associative": {
@@ -0,0 +1,158 @@
{
"baseline": "unfloor-clean",
"candidate": "semsub",
"n_shared_queries": 75,
"fixed_by_candidate": [
"q24",
"q39",
"q42"
],
"broken_by_candidate": [
"q18",
"q19",
"q22",
"q31",
"q43",
"q44",
"q52",
"q63"
],
"discordant": 11,
"net_queries": -5,
"mcnemar_exact_p": 0.2265625,
"min_detectable_swing_queries": 6,
"observed_run_to_run_drift_queries": 0,
"noise_floor_queries": 6,
"verdict": "no measurable difference",
"baseline_aggregate": {
"n_queries": 75,
"n_scored": 65,
"hit@5": 0.5384615384615384,
"recall@5": 0.44907176157176154,
"recall@10": 0.5380300255300255,
"precision@5": 0.13230769230769232,
"mrr@10": 0.32437728937728944,
"nonsense_clean": "10/10",
"superseded_outranks": "2/3",
"latency_ms_p50": 632.5,
"latency_ms_p95": 992.5,
"latency_ms_max": 1177.8,
"errors": 0,
"by_category": {
"associative": {
"n": 6,
"hit@5": 0.5,
"recall@5": 0.07575757575757576,
"recall@10": 0.13636363636363635,
"mrr@10": 0.23214285714285712
},
"exact_rare": {
"n": 6,
"hit@5": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"mrr@10": 1.0
},
"heldout_paraphrase": {
"n": 30,
"hit@5": 0.3,
"recall@5": 0.3,
"recall@10": 0.4,
"mrr@10": 0.11638888888888889
},
"nonsense": {
"n": 10,
"clean": 10,
"avg_false_positives": 0.0
},
"paraphrase": {
"n": 13,
"hit@5": 0.6923076923076923,
"recall@5": 0.6923076923076923,
"recall@10": 0.7692307692307693,
"mrr@10": 0.29423076923076924
},
"phrase": {
"n": 7,
"hit@5": 1.0,
"recall@5": 0.5335884353741497,
"recall@10": 0.5933956916099773,
"mrr@10": 0.8214285714285714
},
"superseded": {
"n": 3,
"hit@5": 0.3333333333333333,
"recall@5": 0.3333333333333333,
"recall@10": 0.6666666666666666,
"mrr@10": 0.20833333333333334,
"outranks": 2
}
}
},
"candidate_aggregate": {
"n_queries": 75,
"n_scored": 65,
"hit@5": 0.46153846153846156,
"recall@5": 0.3889430014430015,
"recall@10": 0.4887681762681762,
"precision@5": 0.12307692307692313,
"mrr@10": 0.30181318681318675,
"nonsense_clean": "10/10",
"superseded_outranks": "2/3",
"latency_ms_p50": 634.4,
"latency_ms_p95": 988.3,
"latency_ms_max": 1184.5,
"errors": 0,
"by_category": {
"associative": {
"n": 6,
"hit@5": 0.3333333333333333,
"recall@5": 0.06060606060606061,
"recall@10": 0.12121212121212122,
"mrr@10": 0.19047619047619047
},
"exact_rare": {
"n": 6,
"hit@5": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"mrr@10": 1.0
},
"heldout_paraphrase": {
"n": 30,
"hit@5": 0.23333333333333334,
"recall@5": 0.23333333333333334,
"recall@10": 0.3,
"mrr@10": 0.08925925925925927
},
"nonsense": {
"n": 10,
"clean": 10,
"avg_false_positives": 0.0
},
"paraphrase": {
"n": 13,
"hit@5": 0.5384615384615384,
"recall@5": 0.5384615384615384,
"recall@10": 0.7692307692307693,
"mrr@10": 0.26324786324786326
},
"phrase": {
"n": 7,
"hit@5": 1.0,
"recall@5": 0.5596655328798186,
"recall@10": 0.5775226757369615,
"mrr@10": 0.8214285714285714
},
"superseded": {
"n": 3,
"hit@5": 0.3333333333333333,
"recall@5": 0.3333333333333333,
"recall@10": 0.6666666666666666,
"mrr@10": 0.20833333333333334,
"outranks": 2
}
}
},
"repeat_variance": {}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff