Tim Lingo
6f3a048f36
engram_assoc_leg previously took its seeds only from the top-3 LEXICAL hits. For a paraphrase query the lexical hits are noise by construction, so the walk never reached the neighbourhood that holds the answer. This adds the seeding pass Will documents at el_runtime.c l.6082 — "Semantic seeding (HippoRAG pattern, use similarity twice): the query is embedded, the top-K nodes by cosine join the seed set" — using his own ENGRAM_EMBED_SEED_K (8). Similarity is now used twice, coherently: cosine picks where to STAND in the graph, the structural-relation walk decides what is REACHABLE, and cosine orders what was reached (iteration 2's finding, unchanged). The seed list is deliberately NOT floored at ENGRAM_EMBED_SEED_MIN. Measured over all 38 gold queries: true paraphrase targets score cosine 0.46-0.66 and the three nonsense controls' own nearest neighbours score 0.55/0.60/0.62 — the distributions OVERLAP, so no absolute cosine floor separates signal from gibberish. The gate that works is reachability: gibberish's nearest neighbours carry no structural edge, so its graph leg is empty and the controls hold. The raw top-K is selected inside the existing scoring pass, so the cosine is computed exactly once per node: no extra corpus pass, no extra embed round-trip, latency flat (p50 1220 -> 1227 ms, 1.01x). Measured vs the certified baseline feat/hybrid-semantic-recall, embedded corpus, 2 runs each, zero run-to-run drift on both sides: hit@5 51.4% -> 68.6% MRR@10 0.387 -> 0.461 paraphrase 38.5% -> 61.5% associative 0% -> 66.7% exact_rare 100% held, nonsense 2/3 held, superseded 2/3 held phrase 85.7% -> 71.4% (q11, the known rank-5 rotation tax) net +6 queries (7 fixed / 1 broken), McNemar p=0.0703
150 lines
3.3 KiB
JSON
150 lines
3.3 KiB
JSON
{
|
|
"baseline": "assoc-leg",
|
|
"candidate": "semseed",
|
|
"n_shared_queries": 38,
|
|
"fixed_by_candidate": [
|
|
"q18",
|
|
"q19",
|
|
"q22"
|
|
],
|
|
"broken_by_candidate": [
|
|
"q11"
|
|
],
|
|
"discordant": 4,
|
|
"net_queries": 2,
|
|
"mcnemar_exact_p": 0.625,
|
|
"min_detectable_swing_queries": 6,
|
|
"observed_run_to_run_drift_queries": 0,
|
|
"noise_floor_queries": 6,
|
|
"verdict": "no measurable difference",
|
|
"baseline_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.6285714285714286,
|
|
"recall@5": 0.45309194773480493,
|
|
"recall@10": 0.5405733155733157,
|
|
"precision@5": 0.17714285714285719,
|
|
"mrr@10": 0.42650793650793645,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1228.5,
|
|
"latency_ms_p95": 1681.8,
|
|
"latency_ms_max": 1718.6,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.07342657342657342,
|
|
"recall@10": 0.24825174825174826,
|
|
"mrr@10": 0.22777777777777777
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.38461538461538464,
|
|
"recall@5": 0.38461538461538464,
|
|
"recall@10": 0.38461538461538464,
|
|
"mrr@10": 0.17307692307692307
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 0.8571428571428571,
|
|
"recall@5": 0.4882369614512472,
|
|
"recall@10": 0.6329365079365079,
|
|
"mrr@10": 0.6634920634920636
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.2222222222222222,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"candidate_aggregate": {
|
|
"n_queries": 38,
|
|
"n_scored": 35,
|
|
"hit@5": 0.6857142857142857,
|
|
"recall@5": 0.5213459159887731,
|
|
"recall@10": 0.6027048348476919,
|
|
"precision@5": 0.18285714285714294,
|
|
"mrr@10": 0.4608730158730158,
|
|
"nonsense_clean": "2/3",
|
|
"superseded_outranks": "2/3",
|
|
"latency_ms_p50": 1227.1,
|
|
"latency_ms_p95": 1692.6,
|
|
"latency_ms_max": 1710.4,
|
|
"errors": 0,
|
|
"by_category": {
|
|
"associative": {
|
|
"n": 6,
|
|
"hit@5": 0.6666666666666666,
|
|
"recall@5": 0.07342657342657344,
|
|
"recall@10": 0.24825174825174823,
|
|
"mrr@10": 0.20833333333333334
|
|
},
|
|
"exact_rare": {
|
|
"n": 6,
|
|
"hit@5": 1.0,
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0
|
|
},
|
|
"nonsense": {
|
|
"n": 3,
|
|
"clean": 2,
|
|
"avg_false_positives": 3.3333333333333335
|
|
},
|
|
"paraphrase": {
|
|
"n": 13,
|
|
"hit@5": 0.6153846153846154,
|
|
"recall@5": 0.6153846153846154,
|
|
"recall@10": 0.6153846153846154,
|
|
"mrr@10": 0.2846153846153846
|
|
},
|
|
"phrase": {
|
|
"n": 7,
|
|
"hit@5": 0.7142857142857143,
|
|
"recall@5": 0.40093537414965985,
|
|
"recall@10": 0.5150226757369615,
|
|
"mrr@10": 0.6507936507936508
|
|
},
|
|
"superseded": {
|
|
"n": 3,
|
|
"hit@5": 0.3333333333333333,
|
|
"recall@5": 0.3333333333333333,
|
|
"recall@10": 0.6666666666666666,
|
|
"mrr@10": 0.20833333333333334,
|
|
"outranks": 2
|
|
}
|
|
}
|
|
},
|
|
"repeat_variance": {
|
|
"baseline": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.6285714285714286,
|
|
"hit@5_max": 0.6285714285714286,
|
|
"spread_queries": 0
|
|
},
|
|
"candidate": {
|
|
"runs": 2,
|
|
"hit@5_min": 0.6857142857142857,
|
|
"hit@5_max": 0.6857142857142857,
|
|
"spread_queries": 0
|
|
}
|
|
}
|
|
} |