Tim Lingo
77e3c5fa26
Iteration 7 measured this harness's own ceiling at +3 gross / +1 net against a decision floor of 6. An instrument whose ceiling sits below its floor cannot certify or refute anything, so the gold set — not the retriever — was the blocker. This iteration builds no retrieval mechanism; it fixes the instrument and uses it once. extend_gold_set.py appends 37 queries (q39-q75) WITHOUT touching q01-q38, so every committed baseline and per-query id stays comparable. 30 held-out paraphrases: targets sampled mechanically (seed 8080) from addressable 500-2600 char nodes outside the original answer space and outside any duplicate cluster; queries authored from the node body alone, before any retrieval was run, and each one re-proved at build time to share ZERO content words with its target. 7 extra nonsense controls, fully mechanical. Why this was needed: the original 13 paraphrase and 6 associative queries share ONE answer space — the 13 `Self - Values (grounded)` children. 19 of 35 scored queries tested retrieval against a single 13-node neighbourhood. FIRST OUT-OF-SAMPLE RESULT (main vs the accumulated stack, embedded corpus): held-out only : +5 / -0, p=0.0625 — one query short of the floor, NOT-SHOWN original 38 : +15 / -0 full 75 : +20 / -0, p=0.0000, latency 0.46x In-sample paraphrase 61.5% vs out-of-sample 16.7%: generalisation is real, directional and 3.7x weaker than the headline number suggested. Also recorded: 47.4% of this corpus is redundant and ONE record accounts for 46.6% of all 78,768 nodes (36,737 byte-identical copies under distinct ids). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
166 lines
3.3 KiB
JSON
166 lines
3.3 KiB
JSON
{
|
|
"baseline": "main-ext",
|
|
"candidate": "stack-ext",
|
|
"n_shared_queries": 75,
|
|
"fixed_by_candidate": [
|
|
"q10",
|
|
"q15",
|
|
"q16",
|
|
"q18",
|
|
"q19",
|
|
"q20",
|
|
"q21",
|
|
"q22",
|
|
"q26",
|
|
"q27",
|
|
"q28",
|
|
"q29",
|
|
"q31",
|
|
"q35",
|
|
"q37",
|
|
"q40",
|
|
"q44",
|
|
"q48",
|
|
"q49",
|
|
"q50"
|
|
],
|
|
"broken_by_candidate": [],
|
|
"discordant": 20,
|
|
"net_queries": 20,
|
|
"mcnemar_exact_p": 1.9073486328125e-06,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "candidate better",
|
|
"baseline_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.18461538461538463,
|
|
"recall@5": 0.14510073260073258,
|
|
"recall@10": 0.1794871794871795,
|
|
"precision@5": 0.06461538461538462,
|
|
"mrr@10": 0.15847985347985344,
|
|
"nonsense_clean": "9/10",
|
|
"superseded_outranks": "1/3",
|
|
"latency_ms_p50": 1380.2,
|
|
"latency_ms_p95": 2293.8,
|
|
"latency_ms_max": 2879.1,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.0,
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.0,
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 9,
|
|
"avg_false_positives": 1.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.0,
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 0.8571428571428571,
|
|
"recall@5": 0.4902210884353741,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.5965986394557822
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.0,
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.3333333333333333,
|
|
"mrr@10": 0.041666666666666664,
|
|
"outranks": 1
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.47692307692307695,
|
|
"recall@5": 0.3750415183107491,
|
|
"recall@10": 0.45561340369032677,
|
|
"precision@5": 0.12307692307692313,
|
|
"mrr@10": 0.3055555555555555,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 640.9,
|
|
"latency_ms_p95": 1011.1,
|
|
"latency_ms_max": 1190.3,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.16666666666666666,
|
|
"recall@5": 0.16666666666666666,
|
|
"recall@10": 0.26666666666666666,
|
|
"mrr@10": 0.0762037037037037
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {}
|
|
} |