diff --git a/tools/retrieval-eval/bestval.py b/tools/retrieval-eval/bestval.py new file mode 100644 index 0000000..473c742 --- /dev/null +++ b/tools/retrieval-eval/bestval.py @@ -0,0 +1,21 @@ +import numpy as np, json, urllib.request +SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad" +np.seterr(all='ignore') +M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n') +eidx={k:i for i,k in enumerate(eids)} +gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries'] +VALS=sorted({r for q in gold if q['category']=='paraphrase' for r in q['relevant']}) +VI=[eidx[v] for v in VALS] +def emb(t): + b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode() + r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"}) + v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32) + return v/(np.linalg.norm(v)+1e-9) +print("qid cat bestValueNodeGlobalRank goldGlobalRank goldSiblingRank") +for q in gold: + if q['category']!='paraphrase': continue + v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1 + ranks=sorted(int((s>s[j]).sum())+1 for j in VI) + g=eidx[q['relevant'][0]]; gr=int((s>s[g]).sum())+1 + sv=np.array([s[j] for j in VI]); sib=int((sv>s[g]).sum())+1 + print("%-4s %-11s best=%-5d (top3 val ranks %s) gold=%-5d sib=%d" % (q['id'],q['category'],ranks[0],ranks[:3],gr,sib)) diff --git a/tools/retrieval-eval/comparison-assoc-vs-semseed.json b/tools/retrieval-eval/comparison-assoc-vs-semseed.json new file mode 100644 index 0000000..7fe83ec --- /dev/null +++ b/tools/retrieval-eval/comparison-assoc-vs-semseed.json @@ -0,0 +1,150 @@ +{ + "baseline": "assoc-leg", + "candidate": "semseed", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q18", + "q19", + "q22" + ], + "broken_by_candidate": [ + "q11" + ], + "discordant": 4, + "net_queries": 2, + "mcnemar_exact_p": 0.625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1228.5, + "latency_ms_p95": 1681.8, + "latency_ms_max": 1718.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.6285714285714286, + "hit@5_max": 0.6285714285714286, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-hybrid-vs-semseed.json b/tools/retrieval-eval/comparison-hybrid-vs-semseed.json new file mode 100644 index 0000000..a49a008 --- /dev/null +++ b/tools/retrieval-eval/comparison-hybrid-vs-semseed.json @@ -0,0 +1,154 @@ +{ + "baseline": "hybrid-semantic", + "candidate": "semseed", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q18", + "q19", + "q22", + "q27", + "q28", + "q29", + "q31" + ], + "broken_by_candidate": [ + "q11" + ], + "discordant": 8, + "net_queries": 6, + "mcnemar_exact_p": 0.0703125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/fullsim.py b/tools/retrieval-eval/fullsim.py new file mode 100644 index 0000000..82c3f50 --- /dev/null +++ b/tools/retrieval-eval/fullsim.py @@ -0,0 +1,123 @@ +import numpy as np, json, urllib.request, collections, sys +SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad" +EV="/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/" +np.seterr(all='ignore') +M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n') +eidx={k:i for i,k in enumerate(eids)} +d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape')) +N={n['id']:n for n in d['nodes']} +STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"} +adj=collections.defaultdict(list); hasstruct=set() +for e in d['edges']: + if e.get('relation') not in STRUCT: continue + w=float(e.get('weight') or 0.0) + adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w)) + hasstruct.add(e['from_id']); hasstruct.add(e['to_id']) +del d +gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']} +LEX={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"))['rows']} +CACHE={} +def emb(t): + if t in CACHE: return CACHE[t] + b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode() + r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"}) + v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32) + v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v +FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64 +def assoc(seeds, s): + act={x:1.0 for x in seeds}; seen={x:2 for x in seeds} + Q=[(x,0) for x in seeds]; h=0 + while h=DEPTH: continue + p=act[cur] + for oid,w in adj.get(cur,()): + n=N.get(oid) + if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue + na=p*w*DECAY*float(n.get('salience') or 0.0) + if na=lim: break + if si=lim: break + if aiSEED_MIN] + A=[] + if mode!='hybrid': + seeds=[x for x in L[:3] if x in N] + if mode=='semseed': + seeds=seeds+[eids[j] for j in ordr[:K] if eids[j] in N and eids[j] not in seeds] + A=assoc(seeds,s) if seeds else [] + res[qid]=inter3(L,S,A) + return res +def score(res,label): + hits=0; det={} + for qid,q in gold.items(): + out=res[qid][:5] + if q['category']=='nonsense': ok = (len(res[qid])==0) + elif q['category']=='superseded': + rel=q['relevant']; must=q.get('must_outrank') or {} + ok=False + for good,bad in (must.items() if isinstance(must,dict) else []): + ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good) (M@v)[j]).sum())+1, round(float((M@v)[j]),3))) + grank.sort() + out[q['id']]=dict(cat=q['category'],mu=round(mu,3),sd=round(sd,4),top1=round(float(top[0]),3), + z1=round(z[0],2),z3=round(z[2],2),z5=round(z[4],2),gold=grank[:1]) + print("%s %-11s mu=%.3f sd=%.4f top1=%.3f z1=%5.2f z3=%5.2f z5=%5.2f gold=%s"%( + q['id'],q['category'],mu,sd,top[0],z[0],z[2],z[4],grank[:1])) +json.dump(out,open(SP+'/zprobe.json','w'),indent=1) diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 3fe3fad..32c5a74 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -7531,8 +7531,30 @@ static int eg_assoc_excluded(const EngramNode* n) { * function of hop count and ranks the relay hub above all of its own * children. Composing the two is mine, not Will's — the description ranks the * activation result set by strength (l.78). */ +/* Semantic seeding of the graph leg — the HippoRAG pass Will documents at + * l.6082: "the query is embedded, the top-K nodes by cosine >= SEED_MIN join + * the seed set", using his own ENGRAM_EMBED_SEED_K (8). It is "similarity used + * twice, coherently": cosine picks where to STAND in the graph, the structural + * walk decides what is REACHABLE from there, and cosine then ORDERS what was + * reached (iteration-2's finding, kept intact). + * + * Why the seed list is NOT floored at ENGRAM_EMBED_SEED_MIN here. That + * constant is calibrated for a cosine scale this corpus does not have: with + * nomic-embed-text every true paraphrase target measures 0.46-0.66, and the + * three nonsense controls' own nearest neighbours measure 0.55/0.60/0.62 — + * they OVERLAP, so no absolute cosine floor separates signal from gibberish + * (measured, all 38 queries). Iteration 2 established the same thing one step + * later in the pipeline: applying the floor to graph CANDIDATES removed every + * gain, because within a structurally-reached neighbourhood relative cosine + * still discriminates below the absolute threshold. The gate that actually + * works is reachability — eg_rel_is_structural() plus the firing threshold. + * A semantically-near node with no structural attachment expands to nothing + * and contributes nothing, which is exactly what happens to gibberish: the + * nearest neighbours of q33/q34 are unattached, so their graph leg is empty. + */ static int64_t engram_assoc_leg(EngramStore* g, const EngramRankEntry* L, int64_t nL, + const int64_t* semseed, int64_t nsemseed, const float* qv, int32_t qdim, EngramSemEntry* out, int64_t out_cap) { if (!g || nL <= 0 || !qv || qdim <= 0 || out_cap <= 0) return 0; @@ -7553,6 +7575,14 @@ static int64_t engram_assoc_leg(EngramStore* g, act[idx] = 1.0; seen[idx] = 2; /* 2 = seed: never a result */ if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; } } + /* ...and the semantic seeds, on the same footing (strength 1.0, hop 0). */ + for (int64_t s = 0; s < nsemseed; s++) { + int64_t idx = semseed[s]; + if (idx < 0 || idx >= g->node_count) continue; + if (seen[idx]) continue; + act[idx] = 1.0; seen[idx] = 2; + if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; } + } const double SPREAD_DECAY = 0.7; while (qh < qt) { @@ -9524,6 +9554,13 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; int64_t nsem = 0; int64_t nhits = 0; + /* Raw (unfloored) cosine top-K, kept for the graph leg's + * semantic seeds. Selected in THIS pass so the cosine is + * computed exactly once per node — the seeding costs no extra + * pass over the corpus and no extra embed round-trip. */ + int64_t semseed[ENGRAM_EMBED_SEED_K]; + double semseedc[ENGRAM_EMBED_SEED_K]; + int64_t nsemseed = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ @@ -9535,9 +9572,29 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { hits[nhits].salience = n->salience; nhits++; } - if (sem) { - double sv = eg_sem_term(n, qv, qdim); - if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; } + if (sem && n->emb && n->emb_dim == qdim) { + double c = eg_cosine(n->emb, qv, qdim); + /* Semantic leg: identical to eg_sem_term(), which is + * left in place and still used by engram_search(). + * Inlined here only so one cosine serves both uses. */ + if (c > ENGRAM_EMBED_SEED_MIN) { + double sv = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN); + if (sv > 1.0) sv = 1.0; + sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; + } + /* Graph seeds: top-K by RAW cosine, insertion-ordered. */ + if (c > 0.0 && (nsemseed < ENGRAM_EMBED_SEED_K + || c > semseedc[nsemseed - 1])) { + int64_t p = nsemseed < ENGRAM_EMBED_SEED_K + ? nsemseed : ENGRAM_EMBED_SEED_K - 1; + while (p > 0 && semseedc[p - 1] < c) { + semseedc[p] = semseedc[p - 1]; + semseed[p] = semseed[p - 1]; + p--; + } + semseedc[p] = c; semseed[p] = i; + if (nsemseed < ENGRAM_EMBED_SEED_K) nsemseed++; + } } } qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); @@ -9549,7 +9606,8 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { * lexical ordering safe. */ EngramSemEntry* assoc = qv ? malloc((size_t)ENGRAM_ASSOC_MAX * sizeof(EngramSemEntry)) : NULL; int64_t nassoc = assoc - ? engram_assoc_leg(g, hits, nhits, qv, qdim, assoc, ENGRAM_ASSOC_MAX) + ? engram_assoc_leg(g, hits, nhits, semseed, nsemseed, + qv, qdim, assoc, ENGRAM_ASSOC_MAX) : 0; int64_t* order = malloc((size_t)lim * sizeof(int64_t)); if (order) {