feat(engram): semantically seed the graph leg (Will's HippoRAG pass, SEED_K=8)
engram_assoc_leg previously took its seeds only from the top-3 LEXICAL hits. For a paraphrase query the lexical hits are noise by construction, so the walk never reached the neighbourhood that holds the answer. This adds the seeding pass Will documents at el_runtime.c l.6082 — "Semantic seeding (HippoRAG pattern, use similarity twice): the query is embedded, the top-K nodes by cosine join the seed set" — using his own ENGRAM_EMBED_SEED_K (8). Similarity is now used twice, coherently: cosine picks where to STAND in the graph, the structural-relation walk decides what is REACHABLE, and cosine orders what was reached (iteration 2's finding, unchanged). The seed list is deliberately NOT floored at ENGRAM_EMBED_SEED_MIN. Measured over all 38 gold queries: true paraphrase targets score cosine 0.46-0.66 and the three nonsense controls' own nearest neighbours score 0.55/0.60/0.62 — the distributions OVERLAP, so no absolute cosine floor separates signal from gibberish. The gate that works is reachability: gibberish's nearest neighbours carry no structural edge, so its graph leg is empty and the controls hold. The raw top-K is selected inside the existing scoring pass, so the cosine is computed exactly once per node: no extra corpus pass, no extra embed round-trip, latency flat (p50 1220 -> 1227 ms, 1.01x). Measured vs the certified baseline feat/hybrid-semantic-recall, embedded corpus, 2 runs each, zero run-to-run drift on both sides: hit@5 51.4% -> 68.6% MRR@10 0.387 -> 0.461 paraphrase 38.5% -> 61.5% associative 0% -> 66.7% exact_rare 100% held, nonsense 2/3 held, superseded 2/3 held phrase 85.7% -> 71.4% (q11, the known rank-5 rotation tax) net +6 queries (7 fixed / 1 broken), McNemar p=0.0703
This commit is contained in:
Tim Lingo
@@ -0,0 +1,123 @@
|
||||
import numpy as np, json, urllib.request, collections, sys
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
N={n['id']:n for n in d['nodes']}
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list); hasstruct=set()
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
hasstruct.add(e['from_id']); hasstruct.add(e['to_id'])
|
||||
del d
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
LEX={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"))['rows']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
def inter3(L,S,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def run(mode, K=0):
|
||||
res={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if s[j]>SEED_MIN]
|
||||
A=[]
|
||||
if mode!='hybrid':
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
if mode=='semseed':
|
||||
seeds=seeds+[eids[j] for j in ordr[:K] if eids[j] in N and eids[j] not in seeds]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
res[qid]=inter3(L,S,A)
|
||||
return res
|
||||
def score(res,label):
|
||||
hits=0; det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok = (len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
rel=q['relevant']; must=q.get('must_outrank') or {}
|
||||
ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok = any(r in out for r in rel)
|
||||
else: ok = any(r in out for r in q['relevant'])
|
||||
det[qid]=ok; hits+=ok
|
||||
print("%-22s outcome-true=%d/38" % (label,hits))
|
||||
return det
|
||||
print("gold sample keys:", list(list(gold.values())[0].keys()))
|
||||
mk=[q for q in gold.values() if q['category']=='superseded'][0]
|
||||
print("superseded fields:", {k:v for k,v in mk.items() if k!='derivation'})
|
||||
a=score(run('hybrid'),'sim hybrid(L+S)')
|
||||
b=score(run('lexseed'),'sim assoc(lex seeds)')
|
||||
for K in (3,5,10):
|
||||
c=score(run('semseed',K),'sim assoc(+sem K=%d)'%K)
|
||||
d=[q for q in gold if c[q]!=b[q]]
|
||||
print(" vs lexseed: moved=%d gains=%s losses=%s"%(len(d),[q for q in d if c[q]],[q for q in d if not c[q]]))
|
||||
e=[q for q in gold if c[q]!=a[q]]
|
||||
print(" vs hybrid : moved=%d gains=%s losses=%s"%(len(e),[q for q in e if c[q]],[q for q in e if not c[q]]))
|
||||
|
||||
print("\n=== Will's own constant ENGRAM_EMBED_SEED_K = 8 ===")
|
||||
c=score(run('semseed',8),'sim assoc(+sem K=8)')
|
||||
for base,lab in ((b,'lexseed(iter2)'),(a,'hybrid(iter1 KEEP)')):
|
||||
dd=[q for q in gold if c[q]!=base[q]]
|
||||
print(" vs %-18s moved=%d gains=%s losses=%s"%(lab,len(dd),[q for q in dd if c[q]],[q for q in dd if not c[q]]))
|
||||
# diagnostic: what is assoc rank-1 for each paraphrase query at K=8
|
||||
print("\nassoc leg head at K=8 (paraphrase):")
|
||||
for qid,q in gold.items():
|
||||
if q['category'] not in ('paraphrase','nonsense'): continue
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]; ordr=np.argsort(-s)
|
||||
seeds=[x for x in L[:3] if x in N]+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in L[:3]]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
rel=set(q['relevant']); gr=next((i+1 for i,x in enumerate(A) if x in rel),None)
|
||||
print(" %-4s %-11s |A|=%-4d goldAssocRank=%-5s head=%s"%(qid,q['category'],len(A),gr,
|
||||
[ (N[x].get('label') or x)[:26] for x in A[:3] ]))
|
||||
Reference in New Issue
Block a user