Tim Lingo
9790d9342d
Two changes to the ONE leg that generalises. Iteration 8 measured that out of sample the semantic leg contributes 100% of the stack's gain and the graph leg contributes nothing, so this is where the remaining headroom is. 1. THE 0.60 FLOOR IS A PER-QUERY LOTTERY, AND CLAIM 24 HAS NO THRESHOLD IN IT. 06-claims.md l.148: "respond to embedding search queries by returning the node records whose embedding vectors have the HIGHEST COSINE SIMILARITY to a query vector, independently of the spreading activation traversal." A ranking. ENGRAM_EMBED_SEED_MIN is defined at el_runtime.c l.6094 as the HippoRAG seed-JOIN threshold and l.6102 admits the read-path leg merely "reuses" it. Measured on the 30 held-out paraphrases: the query's own top-1 cosine ranges 0.564-0.680, so the constant keeps a rank-1 answer for one query and discards a rank-1 answer for the next. Six golds sit at global cosine rank 1-2 scoring 0.564-0.589 - discarded by nothing but the constant. What holds the nonsense controls is the corpus-vocabulary gate (nhits == 0), not this floor. Cosine clamped to [0,1] per 05-detailed-description l.69. 2. THE VECTORS THEMSELVES ANSWER THE WRONG QUESTION. EL_EMBED_MODEL defaults to nomic-embed-text, an ASYMMETRIC retrieval encoder trained with task prefixes. Embedding query and document bare - as this file did on both sides - measures topical similarity rather than answer-hood. eg_embed_fetch now takes the task prefix: EL_EMBED_QUERY_PREFIX on the three query call sites, EL_EMBED_DOC_PREFIX on the two backfill sites. Restores no claim, and says so: Will specifies only "computed by an embedding model over the node's content" (l.17), so the model is his and its correct use is ours. It is the substrate under claim 24 - the index is only as good as the vectors in it. Reproducer for the derived corpus: tools/retrieval-eval/embed-corpus-prefixed.py. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
155 lines
3.3 KiB
JSON
155 lines
3.3 KiB
JSON
{
|
|
"baseline": "stack-ext",
|
|
"candidate": "unfloor-ext",
|
|
"n_shared_queries": 75,
|
|
"fixed_by_candidate": [
|
|
"q14",
|
|
"q25",
|
|
"q43",
|
|
"q52",
|
|
"q63",
|
|
"q67"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q15",
|
|
"q28"
|
|
],
|
|
"discordant": 8,
|
|
"net_queries": 4,
|
|
"mcnemar_exact_p": 0.2890625,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.47692307692307695,
|
|
"recall@5": 0.3750415183107491,
|
|
"recall@10": 0.45561340369032677,
|
|
"precision@5": 0.12307692307692313,
|
|
"mrr@10": 0.3055555555555555,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 640.9,
|
|
"latency_ms_p95": 1011.1,
|
|
"latency_ms_max": 1190.3,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.08857808857808858,
|
|
"recall@10": 0.23310023310023312,
|
|
"mrr@10": 0.25
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.16666666666666666,
|
|
"recall@5": 0.16666666666666666,
|
|
"recall@10": 0.26666666666666666,
|
|
"mrr@10": 0.0762037037037037
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5494614512471656,
|
|
"recall@10": 0.6023242630385487,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 75,
|
|
"n_scored": 65,
|
|
"hit@5": 0.5384615384615384,
|
|
"recall@5": 0.44907176157176154,
|
|
"recall@10": 0.5380300255300255,
|
|
"precision@5": 0.13230769230769232,
|
|
"mrr@10": 0.32437728937728944,
|
|
"nonsense_clean": "10/10",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 993.0,
|
|
"latency_ms_p95": 1544.7,
|
|
"latency_ms_max": 1809.0,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.5,
|
|
"recall@5": 0.07575757575757576,
|
|
"recall@10": 0.13636363636363635,
|
|
"mrr@10": 0.23214285714285712
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"heldout_paraphrase": {
|
|
"n": 30,
|
|
"hit@5": 0.3,
|
|
"recall@5": 0.3,
|
|
"recall@10": 0.4,
|
|
"mrr@10": 0.11638888888888889
|
|
},
|
|
"nonsense": {
|
|
"n": 10,
|
|
"clean": 10,
|
|
"avg_false_positives": 0.0
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6923076923076923,
|
|
"recall@5": 0.6923076923076923,
|
|
"recall@10": 0.7692307692307693,
|
|
"mrr@10": 0.29423076923076924
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 1.0,
|
|
"recall@5": 0.5335884353741497,
|
|
"recall@10": 0.5933956916099773,
|
|
"mrr@10": 0.8214285714285714
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {}
|
|
} |