From 4eb4c9e287cdc5818afb4229acb0040e9a4fc5a9 Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 7 Aug 2026 16:58:34 -0500 Subject: [PATCH] feat(engram): word-start match primitive + corpus-vocabulary gate on recall The retrieval match test is a raw substring scan, so a query token matches anywhere INSIDE a corpus word: "throom" matches "bathroom". Measured over the 38-query gold set on this corpus that is not a rare accident - q28's lexical leg is 36,954 records of which only 13 contain a query token at a word start (99.96% mid-word noise), six further queries carry ~20,500 mid-word-only records each, and the nonsense control q35 returns 7 records ALL of which match only mid-word. istr_contains_wordstart() anchors a token to a word start (preceding char not alphanumeric) while still matching suffixes, so "value" still hits "values". That empties the lexical leg for gibberish, and the nhits==0 corpus-vocabulary gate (iteration 6's mechanism, feat/claim24-unfloored-semantic) then makes the whole query decline rather than let the semantic leg answer it. Measured vs feat/bm25-lexical-leg on the embedded corpus, 2 runs each, 0 queries of run-to-run drift on both sides: net +1 (nonsense:q35), 0 losses, McNemar p=1.0 -> NOT-SHOWN (floor is 6) nonsense clean 2/3 -> 3/3; exact_rare 100%, phrase 100%, paraphrase 61.5%, associative 66.7%, superseded 2/3 all UNCHANGED latency p50 1184 -> 543 ms (0.46x) Iteration 6 called q35 "a DEFECTIVE CONTROL ... cannot be cleaned without breaking the lexical leg". It can: the defect was the match primitive, and cleaning it cost nothing. Also committed: results-wsclaim24.json + cmp-nogate.json, a measured negative for bundling the claim-24 unfloored semantic leg on top (gains q14/q25, breaks q15/q28/q33/q34, net -2) - it independently reproduces iteration 6's q15/q28 losses and shows unflooring REQUIRES the vocabulary gate. Reproducers: legs.py (leg-level replica, reproduces baseline hit@5 exactly on all 38 queries), policy2.py, ceiling.py, wb2.py. --- tools/retrieval-eval/ceiling.py | 43 + tools/retrieval-eval/cmp-nogate.json | 146 +++ .../comparison-bm25lex-vs-wordstart.json | 146 +++ tools/retrieval-eval/legs.py | 117 ++ tools/retrieval-eval/policy2.py | 102 ++ .../retrieval-eval/results-wordstart-r2.json | 948 +++++++++++++++ tools/retrieval-eval/results-wordstart.json | 948 +++++++++++++++ tools/retrieval-eval/results-wsclaim24.json | 1024 +++++++++++++++++ tools/retrieval-eval/wb2.py | 92 ++ .../el-runtime/v1.0.0-20260501/el_runtime.c | 59 +- 10 files changed, 3618 insertions(+), 7 deletions(-) create mode 100644 tools/retrieval-eval/ceiling.py create mode 100644 tools/retrieval-eval/cmp-nogate.json create mode 100644 tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json create mode 100644 tools/retrieval-eval/legs.py create mode 100644 tools/retrieval-eval/policy2.py create mode 100644 tools/retrieval-eval/results-wordstart-r2.json create mode 100644 tools/retrieval-eval/results-wordstart.json create mode 100644 tools/retrieval-eval/results-wsclaim24.json create mode 100644 tools/retrieval-eval/wb2.py diff --git a/tools/retrieval-eval/ceiling.py b/tools/retrieval-eval/ceiling.py new file mode 100644 index 0000000..4078e39 --- /dev/null +++ b/tools/retrieval-eval/ceiling.py @@ -0,0 +1,43 @@ +import json,sys,pickle,numpy as np,itertools +sys.path.insert(0,'.') +from policy2 import legs3,outcome,G,NODES,merge +# cache per-query leg id-lists, floored and unfloored +cache={} +for q in G['queries']: + Lf,Sf,Af=legs3(q['query']) + Lu,Su,Au=legs3(q['query'],unfloor=True) + cache[q['id']]=dict(L=Lf,Sf=Sf,A=Af,Su=Su,Au=Au) +pickle.dump(cache,open('ceil.pkl','wb')) +def mrg(pattern,L,S,A,lim=10): + out=[];p={'L':0,'S':0,'A':0};src={'L':L,'S':S,'A':A} + i=0 + while len(out)=lim: return out + if not prog: break + return out +def ev(pattern,unfl): + res={} + for q in G['queries']: + c=cache[q['id']] + S=c['Su'] if unfl else c['Sf'] + ids=[NODES[i]['id'] for i in mrg(pattern,c['L'],S,c['A'],10)] + res[q['id']]=outcome(q,ids) + return res +base=ev('LSA',False) +print("baseline",sum(base.values())) +best=[] +pats=['LSA','LAS','SLA','ALS','SAL','ASL','LSSA','LSASA','LSAA','LSSAA','LSAS','SSLA','LLSA','SALSA','LSAAS'] +for unfl in (False,True): + for p in pats: + r=ev(p,unfl) + g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k]) + best.append((len(g)-len(l),p,unfl,g,l)) +best.sort(reverse=True) +for n,p,u,g,l in best[:10]: + print("net=%+d pat=%-6s unfloor=%s gains=%s losses=%s"%(n,p,u,g,l)) diff --git a/tools/retrieval-eval/cmp-nogate.json b/tools/retrieval-eval/cmp-nogate.json new file mode 100644 index 0000000..e680068 --- /dev/null +++ b/tools/retrieval-eval/cmp-nogate.json @@ -0,0 +1,146 @@ +{ + "baseline": "bm25lex", + "candidate": "wsclaim24", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q14", + "q25" + ], + "broken_by_candidate": [ + "q15", + "q28", + "q33", + "q34" + ], + "discordant": 6, + "net_queries": -2, + "mcnemar_exact_p": 0.6875, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5768475572047, + "recall@10": 0.6563414759843332, + "precision@5": 0.19428571428571437, + "mrr@10": 0.5026530612244898, + "nonsense_clean": "0/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 524.8, + "latency_ms_p95": 738.7, + "latency_ms_max": 755.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 0, + "avg_false_positives": 10.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json b/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json new file mode 100644 index 0000000..661e2db --- /dev/null +++ b/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json @@ -0,0 +1,146 @@ +{ + "baseline": "bm25lex", + "candidate": "wordstart", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q35" + ], + "broken_by_candidate": [], + "discordant": 1, + "net_queries": 1, + "mcnemar_exact_p": 1.0, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "3/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 542.6, + "latency_ms_p95": 741.3, + "latency_ms_max": 758.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 3, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/legs.py b/tools/retrieval-eval/legs.py new file mode 100644 index 0000000..2b98534 --- /dev/null +++ b/tools/retrieval-eval/legs.py @@ -0,0 +1,117 @@ +import json,pickle,os,math,urllib.request,numpy as np +S='/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/sim/' +C=pickle.load(open(S+'corpus.pkl','rb')) +NODES=C['nodes']; EDGES=C['edges']; N=len(NODES) +E=np.load(S+'emb.npy'); HAVE=np.load(S+'have.npy') +En=E/np.maximum(np.linalg.norm(E,axis=1,keepdims=True),1e-12) +LAYERS={int(l['layer_id']):l for l in (C['layers'] or [])} if C['layers'] else {} +TRANS=set(i for i,l in LAYERS.items() if l.get('transparent')) +def addressable(s): + if not s: return False + return all(0x20<=ord(ch)<=0x7e for ch in s) +ADDR=np.array([addressable(n['id']) for n in NODES]) +OK=np.array([ (n['layer_id'] not in TRANS) and ADDR[i] for i,n in enumerate(NODES)]) +SAL=np.array([n['salience'] for n in NODES]) +LOW=[ (n['content']+'\x00'+n['label']+'\x00'+n['tags']).lower() for n in NODES] +DL=np.array([float(len(n['content'])+len(n['label'])+len(n['tags'])) for n in NODES]) +IDX={} +for i,n in enumerate(NODES): + IDX.setdefault(n['id'],i) +STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"} +ADJ_F=[[] for _ in range(N)]; ADJ_T=[[] for _ in range(N)] +for e in EDGES: + a=IDX.get(e['from']); b=IDX.get(e['to']) + if a is None or b is None: continue + ADJ_F[a].append((e,b)); ADJ_T[b].append((e,a)) +EXCL=np.array([n['node_type'] in ('Tag','InternalStateEvent') for n in NODES]) +avgdl_all=None +def tokenize(q): + out=[] + for t in q.split(): + if not any(t.lower()==x.lower() for x in out): out.append(t) + return out +_qcache={} +def qemb(q): + if q in _qcache: return _qcache[q] + body=json.dumps({"model":"nomic-embed-text","prompt":q}).encode() + r=urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=body,headers={"Content-Type":"application/json"}),timeout=30) + v=np.array(json.loads(r.read())["embedding"],dtype=np.float32) + v=v/np.linalg.norm(v); _qcache[q]=v; return v +K1,B=1.2,0.75 +SEED_MIN=0.60; SEED_K=8; ASSOC_SEEDS=3; DEPTH=2; FIRE=0.02; AMAX=64; DECAY=0.7 +def legs(query): + toks=tokenize(query) + masks=[]; + hit_idx=[]; hit_mask=[] + df=[0]*len(toks) + lt=[t.lower() for t in toks] + for i in range(N): + if not OK[i]: continue + s=LOW[i]; m=0 + for t,tok in enumerate(lt): + if tok in s: m|=(1<>t&1: df[t]+=1 + dl_n=int(OK.sum()); avgdl=float(DL[OK].sum()/max(dl_n,1)) + idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df] + L=[] + for j,i in enumerate(hit_idx): + norm=1.0-B+B*(DL[i]/avgdl); w=0.0 + for t in range(len(toks)): + if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm) + L.append((i,w,SAL[i])) + L.sort(key=lambda x:(-x[1],-x[2])) + qv=qemb(query) + cos=En@qv + cos=np.where(HAVE&OK,cos,-2.0) + order=np.argsort(-cos) + semfull=[(int(i),float(cos[i])) for i in order[:400]] + Sleg=[(i,(c-SEED_MIN)/(1-SEED_MIN)) for i,c in semfull if c>SEED_MIN] + semseed=[i for i,c in semfull[:SEED_K] if c>0.0] + # assoc + act={}; seen={}; qq=[] + for i,_,_ in L[:ASSOC_SEEDS]: + act[i]=1.0; seen[i]=2; qq.append((i,0)) + for i in semseed: + if i in seen: continue + act[i]=1.0; seen[i]=2; qq.append((i,0)) + qh=0 + while qh=DEPTH: continue + parent=act[cur] + for e,oi in ADJ_F[cur]+ADJ_T[cur]: + if e['rel'] not in STRUCT: continue + if EXCL[oi]: continue + na=parent*e['w']*DECAY*SAL[oi] + if na=lim: break + if si=lim: break + if ai>t&1: df[t]+=1 + dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1)) + idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df] + L=[] + for j,i in enumerate(hit_idx): + norm=1.0-B+B*(DL[i]/avgdl);w=0.0 + for t in range(len(toks)): + if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm) + L.append((i,w,SAL[i])) + L.sort(key=lambda x:(-x[1],-x[2])) + if not L: return [],[],[] + qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0) + order=np.argsort(-cos)[:600] + cand=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)] + key=(lambda i:(SAL[i] if sem_sal else 1.0)*float(cos[i])) + Sl=sorted(cand,key=lambda i:-key(i)) + if sem_cap: Sl=Sl[:sem_cap] + semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0] + act={};seen={};qq=[] + for i,_,_ in L[:ASSOC_SEEDS]: + act[i]=1.0;seen[i]=2;qq.append((i,0)) + for i in semseed: + if i in seen: continue + act[i]=1.0;seen[i]=2;qq.append((i,0)) + qh=0 + while qh=DEPTH: continue + parent=act[cur] + for e,oi in ADJ_F[cur]+ADJ_T[cur]: + if e['rel'] not in STRUCT: continue + if EXCL[oi]: continue + na=parent*e['w']*DECAY*SAL[oi] + if na=lim: break + if si=lim: break + if ai>t&1: df[t]+=1 + dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1)) + idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df] + L=[] + for j,i in enumerate(hit_idx): + norm=1.0-B+B*(DL[i]/avgdl);w=0.0 + for t in range(len(toks)): + if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm) + L.append((i,w,SAL[i])) + L.sort(key=lambda x:(-x[1],-x[2])) + if not L: return [],[],[] + qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0) + order=np.argsort(-cos)[:600] + Sl=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)] + semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0] + act={};seen={};qq=[] + for i,_,_ in L[:ASSOC_SEEDS]: + act[i]=1.0;seen[i]=2;qq.append((i,0)) + for i in semseed: + if i in seen: continue + act[i]=1.0;seen[i]=2;qq.append((i,0)) + qh=0 + while qh=DEPTH: continue + parent=act[cur] + for e,oi in ADJ_F[cur]+ADJ_T[cur]: + if e['rel'] not in STRUCT or EXCL[oi]: continue + na=parent*e['w']*DECAY*SAL[oi] + if na0.0],key=lambda x:-x[1])[:AMAX] + return [i for i,_,_ in L],Sl,[i for i,_ in A] +def merge(L,S,A,lim=10): + out=[];li=si=ai=0 + while len(out)=lim: break + if si=lim: break + if aicontent, toks[t]) || - istr_contains(n->label, toks[t]) || - istr_contains(n->tags, toks[t])) + if (istr_contains_wordstart(n->content, toks[t]) || + istr_contains_wordstart(n->label, toks[t]) || + istr_contains_wordstart(n->tags, toks[t])) score++; } return score; @@ -7396,9 +7429,9 @@ static uint32_t engram_node_match_mask(const EngramNode* n, char toks[][ENGRAM_QTOK_LEN], int ntok) { uint32_t m = 0; for (int t = 0; t < ntok && t < 32; t++) { - if (istr_contains(n->content, toks[t]) || - istr_contains(n->label, toks[t]) || - istr_contains(n->tags, toks[t])) + if (istr_contains_wordstart(n->content, toks[t]) || + istr_contains_wordstart(n->label, toks[t]) || + istr_contains_wordstart(n->tags, toks[t])) m |= (uint32_t)1u << t; } return m; @@ -9712,7 +9745,19 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { ? engram_assoc_leg(g, hits, nhits, semseed, nsemseed, qv, qdim, assoc, ENGRAM_ASSOC_MAX) : 0; - int64_t* order = malloc((size_t)lim * sizeof(int64_t)); + /* Corpus-vocabulary gate. If no stored record contains ANY + * query token in its content, label or tags, the query is + * outside this graph's vocabulary: there are no seeds, and + * 05-detailed-description l.64 makes retrieval downstream of + * seeds ("the caller provides one or more seed node UUIDs + * representing the current active context"). No seeds, no + * retrieval — the graph declines rather than confabulating a + * nearest neighbour for gibberish. The mechanism is iteration + * 6's (feat/claim24-unfloored-semantic); it is required here + * because word-start matching empties the lexical leg for + * q35-style queries whose only "hits" were mid-word, and the + * semantic leg would otherwise answer them anyway. */ + int64_t* order = (nhits > 0) ? malloc((size_t)lim * sizeof(int64_t)) : NULL; if (order) { int64_t no = engram_interleave3(hits, nhits, sem, nsem, assoc, nassoc, lim, order);