From 21d3516426293e638f0f9908de204a2df9fb6c7c Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 7 Aug 2026 16:33:50 -0500 Subject: [PATCH] measure: claim-24 unfloored semantic leg vs bm25lex baseline - net +0, NOT-SHOWN gains q14,q25 (gold at global cosine rank 1, previously discarded by the 0.60 floor); losses q15,q28 (the semantic leg was EMPTY on those queries under the floor, so filling it turns a 2-leg rotation into a 3-leg one and halves the associative leg's share of the top 5). Guards held: exact_rare 6/6, phrase 7/7, nonsense 2/3, superseded 2/3 outranks. recall@10 61.8->65.4pp, latency 0.99x. Baseline reproduced from source (results-bm25base-rerun.json is byte-identical to the committed results-bm25lex.json), candidate deterministic across 2 runs. --- tools/retrieval-eval/cmp-bm25-vs-claim24.json | 150 +++ tools/retrieval-eval/diag6.py | 110 ++ .../results-bm25base-rerun.json | 959 ++++++++++++++++ tools/retrieval-eval/results-claim24-r2.json | 1002 +++++++++++++++++ tools/retrieval-eval/results-claim24.json | 1002 +++++++++++++++++ tools/retrieval-eval/sim6.py | 135 +++ tools/retrieval-eval/sim6b.py | 32 + tools/retrieval-eval/sim6c.py | 30 + 8 files changed, 3420 insertions(+) create mode 100644 tools/retrieval-eval/cmp-bm25-vs-claim24.json create mode 100644 tools/retrieval-eval/diag6.py create mode 100644 tools/retrieval-eval/results-bm25base-rerun.json create mode 100644 tools/retrieval-eval/results-claim24-r2.json create mode 100644 tools/retrieval-eval/results-claim24.json create mode 100644 tools/retrieval-eval/sim6.py create mode 100644 tools/retrieval-eval/sim6b.py create mode 100644 tools/retrieval-eval/sim6c.py diff --git a/tools/retrieval-eval/cmp-bm25-vs-claim24.json b/tools/retrieval-eval/cmp-bm25-vs-claim24.json new file mode 100644 index 0000000..52149a3 --- /dev/null +++ b/tools/retrieval-eval/cmp-bm25-vs-claim24.json @@ -0,0 +1,150 @@ +{ + "baseline": "bm25lex", + "candidate": "claim24", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q14", + "q25" + ], + "broken_by_candidate": [ + "q15", + "q28" + ], + "discordant": 4, + "net_queries": 0, + "mcnemar_exact_p": 1.0, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5768475572047, + "recall@10": 0.6537440733869305, + "precision@5": 0.19428571428571437, + "mrr@10": 0.5021428571428572, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1173.7, + "latency_ms_p95": 1623.0, + "latency_ms_max": 1647.9, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.12121212121212122, + "mrr@10": 0.22916666666666666 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 3, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/diag6.py b/tools/retrieval-eval/diag6.py new file mode 100644 index 0000000..a164816 --- /dev/null +++ b/tools/retrieval-eval/diag6.py @@ -0,0 +1,110 @@ +import numpy as np, json, urllib.request, collections, math, re, sys, time, pickle, os +SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad" +EV="/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/" +np.seterr(all='ignore') +t0=time.time() +M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n') +eidx={k:i for i,k in enumerate(eids)} +d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape')) +STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"} +adj=collections.defaultdict(list) +for e in d['edges']: + if e.get('relation') not in STRUCT: continue + w=float(e.get('weight') or 0.0) + adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w)) +nodes=d['nodes'] +N={n['id']:n for n in nodes} +PRINT=re.compile(r'^[\x20-\x7e]+$') +ids=[]; hay=[]; dl=[]; sal=[]; addressable=[] +for n in nodes: + i=n.get('id') or '' + h=((n.get('content') or '')+'\x00'+(n.get('label') or '')+'\x00'+(n.get('tags') or '')).lower() + ids.append(i); hay.append(h); dl.append(len(h)); sal.append(float(n.get('salience') or 0.0)) + addressable.append(bool(PRINT.match(i))) +del d +NN=len(ids); avgdl=sum(dl)/NN +print("nodes=%d avgdl=%.0f %.1fs"%(NN,avgdl,time.time()-t0),file=sys.stderr) +gold={q['id']:q for q in json.load(open(EV+"gold_set.json"),)['queries']} +CACHE={} +def emb(t): + if t in CACHE: return CACHE[t] + b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode() + r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"}) + v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32) + v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v +K1,B=1.2,0.75 +def lexleg(query, lim=10): + toks=[] + for w in query.split(): + wl=w.lower() + if wl not in toks: toks.append(wl) + nt=len(toks) + masks=[]; df=[0]*nt + for i in range(NN): + if not addressable[i]: continue + h=hay[i]; m=0; sc=0 + for t in range(nt): + if toks[t] in h: m|=(1<=DEPTH: continue + p=act[cur] + for oid,w in adj.get(cur,()): + n=N.get(oid) + if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue + na=p*w*DECAY*float(n.get('salience') or 0.0) + if na=DEPTH: continue + p=act[cur] + for oid,w in adj.get(cur,()): + n=N.get(oid) + if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue + c=1.0 + if use_cos: + j=eidx.get(oid) + c=max(0.0,float(s[j])) if j is not None else 0.0 + na=p*w*DECAY*float(n.get('salience') or 0.0)*c + if na=lim: break + return out +def run(floor, vocabgate, use_cos, order): + res={}; legs={} + for qid,q in gold.items(): + v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1 + L,nmatch=lexleg(qid,q['query']) + if vocabgate and nmatch==0: + res[qid]=[]; legs[qid]=([],[],[]); continue + ordr=np.argsort(-s) + S=[eids[j] for j in ordr[:10] if PRINT.match(eids[j] or '') and (not floor or s[j]>SEED_MIN)] + seeds=[x for x in L[:3] if x in N] + seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')] + A=assoc(seeds,s,use_cos,order) if seeds else [] + res[qid]=inter([L,S,A]); legs[qid]=(L,S,A) + return res,legs +def score(res,label,base=None): + det={} + for qid,q in gold.items(): + out=res[qid][:5] + if q['category']=='nonsense': ok=(len(res[qid])==0) + elif q['category']=='superseded': + must=q.get('must_outrank') or {}; ok=False + for good,bad in (must.items() if isinstance(must,dict) else []): + ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)=1 structural edge:",len(HASSTRUCT),file=sys.stderr) +def run2(sfilter, seedout, lim=10): + res={} + for qid,q in gold.items(): + v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1 + L,nmatch=lexleg(qid,q['query']) + if nmatch==0: res[qid]=[]; continue + ordr=np.argsort(-s) + cand=[eids[j] for j in ordr[:200] if PRINT.match(eids[j] or '')] + S=[x for x in cand if (not sfilter or x in HASSTRUCT)][:10] + seeds=[x for x in L[:3] if x in N] + semseeds=[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')] + seeds=seeds+semseeds + A=assoc(seeds,s,False,'cos') if seeds else [] + if seedout: + extra=[(float(s[eidx[x]]),x) for x in semseeds if x in HASSTRUCT and x in eidx] + merged=[(float(s[eidx[x]]),x) for x in A if x in eidx]+extra + merged.sort(reverse=True) + seen=set(); A=[] + for c,x in merged: + if x in seen: continue + seen.add(x); A.append(x) + A=A[:ASSOC_MAX] + res[qid]=inter([L,S,A]) + return res +b,_=run(True,False,False,'cos'); base=score(b,'BASE bm25lex replica') +a,_=run(False,True,False,'cos'); score(a,'A floor-off+vocabgate',base) +score(run2(False,True),'E A+struct-seeds-in-graphleg',base) +score(run2(True,False),'F A+S-restricted-to-graph',base) +score(run2(True,True),'G E+F',base) diff --git a/tools/retrieval-eval/sim6c.py b/tools/retrieval-eval/sim6c.py new file mode 100644 index 0000000..1979b05 --- /dev/null +++ b/tools/retrieval-eval/sim6c.py @@ -0,0 +1,30 @@ +exec(open('sim6.py').read().split('if __name__')[0]) +HASSTRUCT=set(adj.keys()) +import json as _j +d2=_j.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape')) +ANYEDGE=set() +for e in d2['edges']: ANYEDGE.add(e['from_id']); ANYEDGE.add(e['to_id']) +del d2 +print("struct=%d anyedge=%d"%(len(HASSTRUCT),len(ANYEDGE)),file=sys.stderr) +def run4(pool, nlegs, lim=10): + P = HASSTRUCT if pool=='struct' else ANYEDGE + res={} + for qid,q in gold.items(): + v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1 + L,nmatch=lexleg(qid,q['query']) + if nmatch==0: res[qid]=[]; continue + ordr=np.argsort(-s) + cand=[eids[j] for j in ordr[:3000] if PRINT.match(eids[j] or '')] + S=cand[:10] + G=[x for x in cand if x in P][:10] + seeds=[x for x in L[:3] if x in N] + seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')] + A=assoc(seeds,s,False,'cos') if seeds else [] + legs=[L,S,G,A] if nlegs==4 else [L,G,A] + res[qid]=inter(legs) + return res +b,_=run(True,False,False,'cos'); base=score(b,'BASE bm25lex replica') +score(run4('struct',4),'I 4leg L,S,G(struct),A',base) +score(run4('any',4), 'J 4leg L,S,G(anyedge),A',base) +score(run4('struct',3),'K 3leg L,G(struct),A',base) +score(run4('any',3), 'L 3leg L,G(anyedge),A',base)