diff --git a/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json b/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json new file mode 100644 index 0000000..6f13286 --- /dev/null +++ b/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json @@ -0,0 +1,145 @@ +{ + "baseline": "baseline-embcorpus", + "candidate": "hybrid-semantic", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q15", + "q16", + "q20", + "q21", + "q26", + "q37" + ], + "broken_by_candidate": [], + "discordant": 6, + "net_queries": 6, + "mcnemar_exact_p": 0.03125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.34285714285714286, + "recall@5": 0.26947278911564626, + "recall@10": 0.3333333333333333, + "precision@5": 0.12000000000000001, + "mrr@10": 0.2943197278911564, + "nonsense_clean": "2/3", + "superseded_outranks": "1/3", + "latency_ms_p50": 1145.9, + "latency_ms_p95": 1574.3, + "latency_ms_max": 1634.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "repeat_variance": { + "candidate": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/embed-corpus.py b/tools/retrieval-eval/embed-corpus.py new file mode 100644 index 0000000..6dbef64 --- /dev/null +++ b/tools/retrieval-eval/embed-corpus.py @@ -0,0 +1,43 @@ +import json,sys,time,urllib.request,threading,queue +SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json" +OUT=sys.argv[1] +URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text" +MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does +d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape')) +tasks=[] +for n in d["nodes"]: + c=n.get("content") or ""; t=n.get("node_type") or "" + if len(c)<8: continue # eg_embed_eligible + if t in ("InternalStateEvent","Tag"): continue + b=c.encode('utf-8',errors='surrogateescape')[:MAXB] + tasks.append((n.get("id") or "", b.decode('utf-8',errors='replace'))) +del d +print("tasks",len(tasks),flush=True) +q=queue.Queue(); [q.put(t) for t in tasks] +lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0] +def work(): + while True: + try: nid,txt=q.get_nowait() + except queue.Empty: return + v=None + for attempt in range(3): + try: + body=json.dumps({"model":MODEL,"prompt":txt}).encode() + r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"}) + with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"] + break + except Exception as e: + if attempt==2: + with lock: fails[0]+=1 + time.sleep(0.5) + with lock: + if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n") + done[0]+=1 + if done[0]%2000==0: + el=time.time()-t0 + print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True) + f.flush() +ths=[threading.Thread(target=work) for _ in range(8)] +[t.start() for t in ths]; [t.join() for t in ths] +f.close() +print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True) diff --git a/tools/retrieval-eval/merge-corpus.py b/tools/retrieval-eval/merge-corpus.py new file mode 100644 index 0000000..f19bbfa --- /dev/null +++ b/tools/retrieval-eval/merge-corpus.py @@ -0,0 +1,17 @@ +import json,sys +SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json" +TSV,OUT=sys.argv[1],sys.argv[2] +emb={} +for line in open(TSV,encoding='utf-8',errors='surrogateescape'): + p=line.rstrip("\n").rsplit("\t",1) + if len(p)==2 and p[1].count(",")>100: emb[p[0]]=p[1] +print("vectors",len(emb),flush=True) +d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape')) +hit=0 +for n in d["nodes"]: + v=emb.get(n.get("id") or "") + if v: n["emb"]=v; hit+=1 +print("attached",hit,"of",len(d["nodes"]),flush=True) +with open(OUT,"w",encoding='utf-8',errors='surrogateescape') as f: + json.dump(d,f,ensure_ascii=False) +print("wrote",OUT,flush=True) diff --git a/tools/retrieval-eval/results-baseline-embcorpus.json b/tools/retrieval-eval/results-baseline-embcorpus.json new file mode 100644 index 0000000..e585152 --- /dev/null +++ b/tools/retrieval-eval/results-baseline-embcorpus.json @@ -0,0 +1,945 @@ +{ + "label": "baseline-embcorpus", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-baseline", + "soul_md5": "5cc9521734907cf2da30f0af94498c06", + "corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json", + "limit": 10, + "port": 7893, + "wall_clock_s": 48.4, + "child_pid": 85995, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.34285714285714286, + "recall@5": 0.26947278911564626, + "recall@10": 0.3333333333333333, + "precision@5": 0.12000000000000001, + "mrr@10": 0.2943197278911564, + "nonsense_clean": "2/3", + "superseded_outranks": "1/3", + "latency_ms_p50": 1145.9, + "latency_ms_p95": 1574.3, + "latency_ms_max": 1634.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 229.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5" + ], + "n_returned": 1, + "latency_ms": 262.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 230.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 229.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848" + ], + "n_returned": 1, + "latency_ms": 250.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 227.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "?ǚ?7??????", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "????7???Ջ3", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b", + "7?e?7???\f3?" + ], + "n_returned": 10, + "latency_ms": 532.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "Z[? 28.6%). Semantics reorders WITHIN and BELOW the - * full-lexical-match band, and admits high-cosine nodes the lexical pass - * missed entirely. */ -#define ENGRAM_SEM_WEIGHT 0.90 +/* The read-path semantic leg (engram claim 24) reuses ENGRAM_EMBED_SEED_MIN + * above as its admission floor: a node joins the embedding ranking only if its + * query cosine clears the same bar that lets it join the seed set. No new + * tuning constant is introduced, and the floor is load-bearing rather than + * cosmetic — it is what keeps a query with no real match (the gold set's + * nonsense controls) from being answered with its nearest neighbours. */ #define ENGRAM_EMBED_MAX_CHARS 2000 #define ENGRAM_EMBED_TIMEOUT_MS 4000L #define ENGRAM_EMBED_BREAKER_LIMIT 3 @@ -7411,29 +7402,67 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) { return t > 1.0 ? 1.0 : t; } -/* Fused rank score: distinct-token coverage in [0,1] plus at most - * ENGRAM_SEM_WEIGHT of semantic similarity. With no query embedding this is - * sc/ntok, a strictly monotone map of the old integer score, so the ordering - * is bit-identical to the pre-semantic ranking. */ -static double eg_fused_score(int sc, int ntok, double sem) { - double f = (double)sc / (double)(ntok > 0 ? ntok : 1); - return f + ENGRAM_SEM_WEIGHT * sem; -} - -/* Rank entry: fused lexical+semantic score (primary, desc) then salience - * (tiebreak, desc). `score` retains the raw distinct-token count for callers - * that want the lexical signal on its own. */ -typedef struct { int64_t idx; int score; double fused; double salience; } EngramRankEntry; +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was — + * the semantic leg is a second ranking merged beside it, never a reweighting + * of this one. */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; static int engram_rank_cmp(const void* a, const void* b) { const EngramRankEntry* ea = (const EngramRankEntry*)a; const EngramRankEntry* eb = (const EngramRankEntry*)b; - if (ea->fused < eb->fused) return 1; /* desc */ - if (ea->fused > eb->fused) return -1; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ if (ea->salience < eb->salience) return 1; if (ea->salience > eb->salience) return -1; return 0; } +/* Semantic rank entry: node index and its renormalized query similarity, + * ordered by similarity desc. This is the claim-24 "embedding search" + * ranking, computed independently of the lexical one. */ +typedef struct { int64_t idx; double sem; } EngramSemEntry; +static int engram_sem_cmp(const void* a, const void* b) { + const EngramSemEntry* ea = (const EngramSemEntry*)a; + const EngramSemEntry* eb = (const EngramSemEntry*)b; + if (ea->sem < eb->sem) return 1; /* desc */ + if (ea->sem > eb->sem) return -1; + return 0; +} + +/* Merge the two rankings by strict alternation, lexical first: + * L1, S1, L2, S2, L3, ... deduplicated by node index, capped at lim. + * + * Rank fusion, not score fusion. nomic's cosine scale is compressed (real + * matches land ~0.55-0.70 while unrelated pairs sit ~0.35-0.50), so any + * additive blend of a cosine onto a token-coverage score is dominated by + * whichever leg happens to have the wider spread. Alternation is invariant to + * both scales: it asks each leg for its next best answer in turn. + * + * Position 1 is always the top lexical hit, so a query whose answer the + * lexical leg already ranks first cannot be displaced — exact-token retrieval + * is structurally safe. The cost is bounded and explicit: a lexical hit at + * rank r lands at output position 2r-1. */ +static int64_t engram_interleave(const EngramRankEntry* L, int64_t nL, + const EngramSemEntry* S, int64_t nS, + int64_t lim, int64_t* out) { + int64_t no = 0, li = 0, si = 0; + while (no < lim && (li < nL || si < nS)) { + if (li < nL) { + int dup = 0; + for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; } + if (!dup) out[no++] = L[li].idx; + li++; + } + if (no >= lim) break; + if (si < nS) { + int dup = 0; + for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; } + if (!dup) out[no++] = S[si].idx; + si++; + } + } + return no; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -7450,6 +7479,8 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * NULL (embedder down / circuit breaker open) => pure lexical, as before. */ int32_t qdim = 0; float* qv = eg_embed_fetch(q, &qdim); + EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; + int64_t nsem = 0; int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; @@ -7459,25 +7490,29 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - double sem = eg_sem_term(n, qv, qdim); - /* Union, not replacement: a node enters the candidate set on EITHER - * leg. sc == 0 && sem > 0 is the embedding-search half of claim 24 — - * nodes the lexical pass cannot see at all. */ - if (sc > 0 || sem > 0.0) { + if (sc > 0) { hits[nhits].idx = i; hits[nhits].score = sc; - hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } + if (sem) { + double sv = eg_sem_term(n, qv, qdim); + if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; } + } } - /* Rank by fused score (desc) then salience (desc), then cap. */ + /* Rank each leg independently, then alternate between them. */ qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); - int64_t end = nhits < lim ? nhits : lim; - for (int64_t k = 0; k < end; k++) { - lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp); + int64_t* order = malloc((size_t)lim * sizeof(int64_t)); + if (order) { + int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order); + for (int64_t k = 0; k < no; k++) + lst = el_list_append(lst, engram_node_to_map(&g->nodes[order[k]])); + free(order); } free(hits); + free(sem); free(qv); return lst; } @@ -9316,29 +9351,39 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { * to be observable to the MCP wrapper and the app. */ int32_t qdim = 0; float* qv = eg_embed_fetch(q, &qdim); + EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; + int64_t nsem = 0; int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - double sem = eg_sem_term(n, qv, qdim); - if (sc > 0 || sem > 0.0) { + if (sc > 0) { hits[nhits].idx = i; hits[nhits].score = sc; - hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } + if (sem) { + double sv = eg_sem_term(n, qv, qdim); + if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; } + } } qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); - int64_t end = nhits < lim ? nhits : lim; - for (int64_t k = 0; k < end; k++) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0); - first = 0; + if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp); + int64_t* order = malloc((size_t)lim * sizeof(int64_t)); + if (order) { + int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order); + for (int64_t k = 0; k < no; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[order[k]], 0); + first = 0; + } + free(order); } free(hits); + free(sem); free(qv); } }