Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Tim Lingo
|
cc4c9345f2 |
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "execfilter",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q16",
|
||||
"q20",
|
||||
"q21",
|
||||
"q26",
|
||||
"q37"
|
||||
],
|
||||
"discordant": 6,
|
||||
"net_queries": -6,
|
||||
"mcnemar_exact_p": 0.03125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate worse",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6,
|
||||
"recall@5": 0.41518699554413835,
|
||||
"recall@10": 0.4869109065537637,
|
||||
"precision@5": 0.18285714285714288,
|
||||
"mrr@10": 0.4493197278911565,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1183.3,
|
||||
"latency_ms_p95": 1637.5,
|
||||
"latency_ms_max": 1672.5,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.11421911421911422,
|
||||
"recall@10": 0.3146853146853147,
|
||||
"mrr@10": 0.2916666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.23076923076923078,
|
||||
"recall@5": 0.23076923076923078,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"mrr@10": 0.12637362637362637
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8333333333333334
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
+22
-41
@@ -1,16 +1,10 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "claim24",
|
||||
"candidate": "execfilter2",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q14",
|
||||
"q25"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q28"
|
||||
],
|
||||
"discordant": 4,
|
||||
"fixed_by_candidate": [],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 0,
|
||||
"net_queries": 0,
|
||||
"mcnemar_exact_p": 1.0,
|
||||
"min_detectable_swing_queries": 6,
|
||||
@@ -79,23 +73,23 @@
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5768475572047,
|
||||
"recall@10": 0.6537440733869305,
|
||||
"precision@5": 0.19428571428571437,
|
||||
"mrr@10": 0.5021428571428572,
|
||||
"recall@5": 0.5580441384012813,
|
||||
"recall@10": 0.631553763696621,
|
||||
"precision@5": 0.2114285714285715,
|
||||
"mrr@10": 0.5235714285714286,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1173.7,
|
||||
"latency_ms_p95": 1623.0,
|
||||
"latency_ms_max": 1647.9,
|
||||
"latency_ms_p50": 1198.3,
|
||||
"latency_ms_p95": 1632.6,
|
||||
"latency_ms_max": 1664.0,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.12121212121212122,
|
||||
"mrr@10": 0.22916666666666666
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.11421911421911422,
|
||||
"recall@10": 0.3146853146853147,
|
||||
"mrr@10": 0.2916666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
@@ -111,16 +105,16 @@
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.3230769230769231
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
@@ -133,18 +127,5 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 3,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"baseline": "semseed",
|
||||
"candidate": "execfilter2",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q11"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 2,
|
||||
"net_queries": 2,
|
||||
"mcnemar_exact_p": 0.5,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5580441384012813,
|
||||
"recall@10": 0.631553763696621,
|
||||
"precision@5": 0.2114285714285715,
|
||||
"mrr@10": 0.5235714285714286,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1198.3,
|
||||
"latency_ms_p95": 1632.6,
|
||||
"latency_ms_max": 1664.0,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.11421911421911422,
|
||||
"recall@10": 0.3146853146853147,
|
||||
"mrr@10": 0.2916666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.3230769230769231
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
import numpy as np, json, urllib.request, collections, math, re, sys, time, pickle, os
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
t0=time.time()
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list)
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
nodes=d['nodes']
|
||||
N={n['id']:n for n in nodes}
|
||||
PRINT=re.compile(r'^[\x20-\x7e]+$')
|
||||
ids=[]; hay=[]; dl=[]; sal=[]; addressable=[]
|
||||
for n in nodes:
|
||||
i=n.get('id') or ''
|
||||
h=((n.get('content') or '')+'\x00'+(n.get('label') or '')+'\x00'+(n.get('tags') or '')).lower()
|
||||
ids.append(i); hay.append(h); dl.append(len(h)); sal.append(float(n.get('salience') or 0.0))
|
||||
addressable.append(bool(PRINT.match(i)))
|
||||
del d
|
||||
NN=len(ids); avgdl=sum(dl)/NN
|
||||
print("nodes=%d avgdl=%.0f %.1fs"%(NN,avgdl,time.time()-t0),file=sys.stderr)
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"),)['queries']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
def lexleg(query, lim=10):
|
||||
toks=[]
|
||||
for w in query.split():
|
||||
wl=w.lower()
|
||||
if wl not in toks: toks.append(wl)
|
||||
nt=len(toks)
|
||||
masks=[]; df=[0]*nt
|
||||
for i in range(NN):
|
||||
if not addressable[i]: continue
|
||||
h=hay[i]; m=0; sc=0
|
||||
for t in range(nt):
|
||||
if toks[t] in h: m|=(1<<t); sc+=1; df[t]+=1
|
||||
if sc: masks.append((i,m,sc))
|
||||
idf=[math.log(1.0+(NN-df[t]+0.5)/(df[t]+0.5)) for t in range(nt)]
|
||||
scored=[]
|
||||
for i,m,sc in masks:
|
||||
norm=1.0-B+B*dl[i]/avgdl
|
||||
s=0.0
|
||||
for t in range(nt):
|
||||
if m&(1<<t): s+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
scored.append((s,i))
|
||||
scored.sort(key=lambda x:(-x[0], -sal[x[1]]))
|
||||
return [ids[i] for s,i in scored[:lim]], len(masks), sum(df)
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
|
||||
LEGS={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L,nmatch,dfsum=lexleg(q['query'])
|
||||
ordr=np.argsort(-s)
|
||||
Sall=[eids[j] for j in ordr[:40] if PRINT.match(eids[j] or '')]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
A=[x for x in A if PRINT.match(x or '')]
|
||||
LEGS[qid]=dict(L=L,Sall=Sall,A=A,scos={x:float(s[eidx[x]]) for x in set(Sall[:20]+A[:20]+list(q.get('relevant') or [])) if x in eidx},nmatch=nmatch)
|
||||
pickle.dump(LEGS,open(SP+'/legs6.pkl','wb'))
|
||||
|
||||
FOCUS=['q14','q17','q23','q24','q25','q30','q32','q33','q34','q35','q36','q37','q38']
|
||||
for qid in FOCUS:
|
||||
q=gold[qid]; g=LEGS[qid]; rel=set(q.get('relevant') or [])
|
||||
def rk(lst):
|
||||
for i,x in enumerate(lst):
|
||||
if x in rel: return i+1
|
||||
return None
|
||||
print("%s %-12s nmatch=%-6d Lrank=%s Srank=%s Arank=%s |A|=%d"%(
|
||||
qid,q['category'],g['nmatch'],rk(g['L']),rk(g['Sall']),rk(g['A']),len(g['A'])))
|
||||
for r in list(rel)[:2]:
|
||||
print(" rel cos=%.3f"%(g['scos'].get(r,-9)))
|
||||
print(" topS cos:", ["%.3f"%g['scos'].get(x,-9) for x in g['Sall'][:3]])
|
||||
print("elapsed %.1fs"%(time.time()-t0),file=sys.stderr)
|
||||
File diff suppressed because it is too large
Load Diff
+333
-376
File diff suppressed because it is too large
Load Diff
+113
-113
@@ -1,37 +1,37 @@
|
||||
{
|
||||
"label": "bm25base-rerun",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25base",
|
||||
"soul_md5": "36c8dfa09c073b85fe7e00b02904d0ed",
|
||||
"label": "execfilter2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-execfilter2",
|
||||
"soul_md5": "df2c75b74717fa40de2dd17a5bff46b5",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"port": 7932,
|
||||
"wall_clock_s": 50.6,
|
||||
"child_pid": 93451,
|
||||
"child_pid": 91409,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"recall@5": 0.5580441384012813,
|
||||
"recall@10": 0.631553763696621,
|
||||
"precision@5": 0.2114285714285715,
|
||||
"mrr@10": 0.5235714285714286,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1186.1,
|
||||
"latency_ms_p95": 1632.5,
|
||||
"latency_ms_max": 1669.8,
|
||||
"latency_ms_p50": 1198.3,
|
||||
"latency_ms_p95": 1632.6,
|
||||
"latency_ms_max": 1664.0,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
"recall@5": 0.11421911421911422,
|
||||
"recall@10": 0.3146853146853147,
|
||||
"mrr@10": 0.2916666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
@@ -50,7 +50,7 @@
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
"mrr@10": 0.3230769230769231
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
@@ -78,7 +78,7 @@
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 302.9,
|
||||
"latency_ms": 289.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -103,7 +103,7 @@
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 336.7,
|
||||
"latency_ms": 324.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -119,7 +119,7 @@
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 290.2,
|
||||
"latency_ms": 291.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -135,7 +135,7 @@
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 307.3,
|
||||
"latency_ms": 288.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -153,7 +153,7 @@
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 331.2,
|
||||
"latency_ms": 310.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -169,7 +169,7 @@
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 302.2,
|
||||
"latency_ms": 307.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -194,7 +194,7 @@
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 595.3,
|
||||
"latency_ms": 591.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -219,7 +219,7 @@
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 526.1,
|
||||
"latency_ms": 521.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
@@ -244,7 +244,7 @@
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 516.8,
|
||||
"latency_ms": 537.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
@@ -269,7 +269,7 @@
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 558.8,
|
||||
"latency_ms": 503.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
@@ -294,7 +294,7 @@
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 513.4,
|
||||
"latency_ms": 517.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -319,7 +319,7 @@
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 867.9,
|
||||
"latency_ms": 869.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
@@ -344,7 +344,7 @@
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 752.9,
|
||||
"latency_ms": 732.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -361,15 +361,15 @@
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"66c63082-b4da-4aa1-8fee-848db8a83210",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"ctx-4a41"
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-2265c223-9e19-47b5-b7ba-5e9c2ce1f22c",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1591.4,
|
||||
"latency_ms": 1594.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -382,25 +382,25 @@
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"f0eb6b13-909c-4674-91ef-23301d3abc8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"30a44d10-2487-420e-bf61-3892e4343c92",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"bfb5809e-d19a-4d3f-8c1a-796db622ad9d"
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1660.1,
|
||||
"latency_ms": 1664.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
@@ -410,16 +410,16 @@
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"tag-fiction",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-6fae4d4b-dbe8-45c5-8bd4-21ffd5caa240",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92"
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1345.5,
|
||||
"latency_ms": 1348.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -432,19 +432,19 @@
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"kn-2b961d24-7fb9-47c7-9515-e45a24dce39d",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"bl-164b520b-c503-49db-89f9-bd2fdf4215f5",
|
||||
"kn-0710e5b4-799d-4a0e-afd3-62d43b38ea37",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"1219277c-1b95-45ec-95a2-07b4a47a4d92",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"kn-c72bb6db-bd1e-4d37-bded-0399194496f6",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1070.7,
|
||||
"latency_ms": 1092.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -469,7 +469,7 @@
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1395.8,
|
||||
"latency_ms": 1410.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -482,19 +482,19 @@
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"d6b12ecf-702b-4101-b1bb-09ed9b220b29",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"c608a095-c98b-4bfa-bfe1-1611c1320290",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"451ae007-4219-4096-89fe-fa2e045fbeb1",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1243.4,
|
||||
"latency_ms": 1264.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -519,7 +519,7 @@
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1598.7,
|
||||
"latency_ms": 1604.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -544,7 +544,7 @@
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1379.6,
|
||||
"latency_ms": 1374.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -565,11 +565,11 @@
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"kn-ac3fa61f-5689-4fbe-8a42-eae016d524c2",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1632.5,
|
||||
"latency_ms": 1632.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -594,7 +594,7 @@
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1491.9,
|
||||
"latency_ms": 1481.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -607,19 +607,19 @@
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-da21c52c-04a5-4f92-8fba-f10aac47e027",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"75e036d3-c170-4e3f-acc2-e456a6850ee2",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1535.5,
|
||||
"latency_ms": 1525.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -638,13 +638,13 @@
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-759e78ca-5394-4244-aa39-1c1468bc5f3e",
|
||||
"mem-154c3ec1-061c-4314-9e5f-50dc9b9422bc",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1249.8,
|
||||
"latency_ms": 1233.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -666,10 +666,10 @@
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"tag-childhood"
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1351.0,
|
||||
"latency_ms": 1344.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -688,17 +688,17 @@
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"3499d5da-0e9c-4de4-9bc4-8941b14e0b1f",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1438.4,
|
||||
"latency_ms": 1443.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"recall@10": 0.5454545454545454,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
@@ -708,24 +708,24 @@
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1110.1,
|
||||
"latency_ms": 1127.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
"recall@5": 0.23076923076923078,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
@@ -736,19 +736,19 @@
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1186.1,
|
||||
"latency_ms": 1198.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"recall@10": 0.5454545454545454,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
@@ -769,7 +769,7 @@
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1193.1,
|
||||
"latency_ms": 1205.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -789,12 +789,12 @@
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1201.5,
|
||||
"latency_ms": 1216.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
@@ -819,7 +819,7 @@
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1424.8,
|
||||
"latency_ms": 1439.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -833,7 +833,7 @@
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 705.7,
|
||||
"latency_ms": 723.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
@@ -844,7 +844,7 @@
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 484.4,
|
||||
"latency_ms": 505.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
@@ -860,13 +860,13 @@
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7"
|
||||
"?m?\\}Q??6??",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 740.2,
|
||||
"latency_ms": 741.2,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
@@ -888,7 +888,7 @@
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1307.7,
|
||||
"latency_ms": 1319.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
@@ -909,14 +909,14 @@
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"7ac62daa-2eac-4c7a-a97e-e4203fc1b57b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1669.8,
|
||||
"latency_ms": 1660.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -944,7 +944,7 @@
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1018.2,
|
||||
"latency_ms": 1020.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
@@ -1,135 +0,0 @@
|
||||
import numpy as np, json, urllib.request, collections, math, re, sys, time
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore'); t0=time.time()
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list)
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
nodes=d['nodes']; N={n['id']:n for n in nodes}
|
||||
PRINT=re.compile(r'^[\x20-\x7e]+$')
|
||||
ids=[];hay=[];dl=[];sal=[];addr=[]
|
||||
for n in nodes:
|
||||
i=n.get('id') or ''
|
||||
h=((n.get('content') or '')+'\x00'+(n.get('label') or '')+'\x00'+(n.get('tags') or '')).lower()
|
||||
ids.append(i);hay.append(h);dl.append(len(h));sal.append(float(n.get('salience') or 0.0));addr.append(bool(PRINT.match(i)))
|
||||
del d
|
||||
NN=len(ids); avgdl=sum(dl)/NN
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
LEXC={}
|
||||
def lexleg(qid,query,lim=10):
|
||||
if qid in LEXC: return LEXC[qid]
|
||||
toks=[]
|
||||
for w in query.split():
|
||||
wl=w.lower()
|
||||
if wl not in toks: toks.append(wl)
|
||||
nt=len(toks); masks=[]; df=[0]*nt
|
||||
for i in range(NN):
|
||||
if not addr[i]: continue
|
||||
h=hay[i]; m=0; sc=0
|
||||
for t in range(nt):
|
||||
if toks[t] in h: m|=(1<<t); sc+=1; df[t]+=1
|
||||
if sc: masks.append((i,m))
|
||||
idf=[math.log(1.0+(NN-df[t]+0.5)/(df[t]+0.5)) for t in range(nt)]
|
||||
scored=[]
|
||||
for i,m in masks:
|
||||
norm=1.0-B+B*dl[i]/avgdl; s=0.0
|
||||
for t in range(nt):
|
||||
if m&(1<<t): s+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
scored.append((s,i))
|
||||
scored.sort(key=lambda x:(-x[0],-sal[x[1]]))
|
||||
LEXC[qid]=([ids[i] for s,i in scored[:lim]], len(masks))
|
||||
return LEXC[qid]
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s, use_cos, order):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
c=1.0
|
||||
if use_cos:
|
||||
j=eidx.get(oid)
|
||||
c=max(0.0,float(s[j])) if j is not None else 0.0
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)*c
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((act[k] if order=='act' else c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX] if PRINT.match(k or '')]
|
||||
def inter(legs,lim=10):
|
||||
out=[];idx=[0]*len(legs)
|
||||
while len(out)<lim and any(idx[i]<len(legs[i]) for i in range(len(legs))):
|
||||
for i in range(len(legs)):
|
||||
if idx[i]<len(legs[i]):
|
||||
if legs[i][idx[i]] not in out: out.append(legs[i][idx[i]])
|
||||
idx[i]+=1
|
||||
if len(out)>=lim: break
|
||||
return out
|
||||
def run(floor, vocabgate, use_cos, order):
|
||||
res={}; legs={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L,nmatch=lexleg(qid,q['query'])
|
||||
if vocabgate and nmatch==0:
|
||||
res[qid]=[]; legs[qid]=([],[],[]); continue
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if PRINT.match(eids[j] or '') and (not floor or s[j]>SEED_MIN)]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')]
|
||||
A=assoc(seeds,s,use_cos,order) if seeds else []
|
||||
res[qid]=inter([L,S,A]); legs[qid]=(L,S,A)
|
||||
return res,legs
|
||||
def score(res,label,base=None):
|
||||
det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok=(len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
must=q.get('must_outrank') or {}; ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok=any(r in out for r in q['relevant'])
|
||||
else: ok=any(r in out for r in q['relevant'])
|
||||
det[qid]=ok
|
||||
line="%-34s true=%d/38"%(label,sum(det.values()))
|
||||
if base is not None:
|
||||
dd=[q for q in sorted(gold) if det[q]!=base[q]]
|
||||
line+=" moved=%d gains=%s losses=%s"%(len(dd),[q for q in dd if det[q]],[q for q in dd if not det[q]])
|
||||
print(line, flush=True)
|
||||
return det
|
||||
if __name__=="__main__":
|
||||
b,_=run(True,False,False,'cos'); base=score(b,'BASE bm25lex replica')
|
||||
for lab,args in [
|
||||
("A floor-off+vocabgate", (False,True,False,'cos')),
|
||||
("B A+cos-in-traversal", (False,True,True ,'cos')),
|
||||
("C A+cos-trav+act-order", (False,True,True ,'act')),
|
||||
("D floor-off NO gate", (False,False,False,'cos')),
|
||||
]:
|
||||
r,_=run(*args); score(r,lab,base)
|
||||
print("elapsed %.1fs"%(time.time()-t0),file=sys.stderr)
|
||||
@@ -1,32 +0,0 @@
|
||||
exec(open('sim6.py').read().split('if __name__')[0])
|
||||
HASSTRUCT=set(adj.keys())
|
||||
print("nodes with >=1 structural edge:",len(HASSTRUCT),file=sys.stderr)
|
||||
def run2(sfilter, seedout, lim=10):
|
||||
res={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L,nmatch=lexleg(qid,q['query'])
|
||||
if nmatch==0: res[qid]=[]; continue
|
||||
ordr=np.argsort(-s)
|
||||
cand=[eids[j] for j in ordr[:200] if PRINT.match(eids[j] or '')]
|
||||
S=[x for x in cand if (not sfilter or x in HASSTRUCT)][:10]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
semseeds=[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')]
|
||||
seeds=seeds+semseeds
|
||||
A=assoc(seeds,s,False,'cos') if seeds else []
|
||||
if seedout:
|
||||
extra=[(float(s[eidx[x]]),x) for x in semseeds if x in HASSTRUCT and x in eidx]
|
||||
merged=[(float(s[eidx[x]]),x) for x in A if x in eidx]+extra
|
||||
merged.sort(reverse=True)
|
||||
seen=set(); A=[]
|
||||
for c,x in merged:
|
||||
if x in seen: continue
|
||||
seen.add(x); A.append(x)
|
||||
A=A[:ASSOC_MAX]
|
||||
res[qid]=inter([L,S,A])
|
||||
return res
|
||||
b,_=run(True,False,False,'cos'); base=score(b,'BASE bm25lex replica')
|
||||
a,_=run(False,True,False,'cos'); score(a,'A floor-off+vocabgate',base)
|
||||
score(run2(False,True),'E A+struct-seeds-in-graphleg',base)
|
||||
score(run2(True,False),'F A+S-restricted-to-graph',base)
|
||||
score(run2(True,True),'G E+F',base)
|
||||
@@ -1,30 +0,0 @@
|
||||
exec(open('sim6.py').read().split('if __name__')[0])
|
||||
HASSTRUCT=set(adj.keys())
|
||||
import json as _j
|
||||
d2=_j.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
ANYEDGE=set()
|
||||
for e in d2['edges']: ANYEDGE.add(e['from_id']); ANYEDGE.add(e['to_id'])
|
||||
del d2
|
||||
print("struct=%d anyedge=%d"%(len(HASSTRUCT),len(ANYEDGE)),file=sys.stderr)
|
||||
def run4(pool, nlegs, lim=10):
|
||||
P = HASSTRUCT if pool=='struct' else ANYEDGE
|
||||
res={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L,nmatch=lexleg(qid,q['query'])
|
||||
if nmatch==0: res[qid]=[]; continue
|
||||
ordr=np.argsort(-s)
|
||||
cand=[eids[j] for j in ordr[:3000] if PRINT.match(eids[j] or '')]
|
||||
S=cand[:10]
|
||||
G=[x for x in cand if x in P][:10]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and PRINT.match(eids[j] or '')]
|
||||
A=assoc(seeds,s,False,'cos') if seeds else []
|
||||
legs=[L,S,G,A] if nlegs==4 else [L,G,A]
|
||||
res[qid]=inter(legs)
|
||||
return res
|
||||
b,_=run(True,False,False,'cos'); base=score(b,'BASE bm25lex replica')
|
||||
score(run4('struct',4),'I 4leg L,S,G(struct),A',base)
|
||||
score(run4('any',4), 'J 4leg L,S,G(anyedge),A',base)
|
||||
score(run4('struct',3),'K 3leg L,G(struct),A',base)
|
||||
score(run4('any',3), 'L 3leg L,G(anyedge),A',base)
|
||||
+124
-38
@@ -7696,11 +7696,15 @@ static int64_t engram_assoc_leg(EngramStore* g,
|
||||
* survivable here only because the structural-relation filter leaves the
|
||||
* associative list EMPTY for most queries — a Memory node whose only edges are
|
||||
* `tagged` and `related` expands to nothing, so its ranking is untouched. */
|
||||
/* `no` is the count ALREADY in `out` — the promoted pass fills the head, the
|
||||
* suppressed pass appends behind it and must dedup against the whole prefix
|
||||
* (the same node can be promoted in one leg and suppressed in another, since
|
||||
* its background activation differs per leg). (2026-08-07, claim 44/45) */
|
||||
static int64_t engram_interleave3(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
const EngramSemEntry* A, int64_t nA,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0, ai = 0;
|
||||
int64_t lim, int64_t* out, int64_t no) {
|
||||
int64_t li = 0, si = 0, ai = 0;
|
||||
while (no < lim && (li < nL || si < nS || ai < nA)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
@@ -9591,6 +9595,47 @@ el_val_t engram_get_node_by_label(el_val_t label) {
|
||||
return el_wrap_str(el_strdup("{}"));
|
||||
}
|
||||
|
||||
/* ── Layer 2: the executive filter, on the recall read path (claims 44/45) ──
|
||||
*
|
||||
* 06-claims.md claim 44 (verbatim): "execute a first activation pass that
|
||||
* propagates spreading activation from query-matched seed node records ...
|
||||
* WITHOUT ANY THRESHOLD FILTERING, recording a background activation score for
|
||||
* every reachable node record; and execute a second executive filter pass that
|
||||
* computes a working memory weight for each background-activated node record by
|
||||
* multiplying the background activation score by a goal-state attentional bias
|
||||
* derived from the current query and by the node record's confidence value ...
|
||||
* wherein context compilation uses only node records whose working memory
|
||||
* weight exceeds a per-type threshold, and node records that do not exceed the
|
||||
* threshold retain their background activation scores and are not discarded."
|
||||
* Claim 45 keeps the un-promoted field available to callers.
|
||||
* 05-detailed-description l.221: "Results are sorted with promoted nodes first
|
||||
* ... followed by background-only nodes."
|
||||
*
|
||||
* This pass exists in engram_activate and NOWHERE on the route the app calls.
|
||||
* /api/neuron/recall reaches engram_search_json, whose three legs each get a
|
||||
* fixed share of the output slots by rotation — so on a query where a leg is
|
||||
* structurally incapable of being right, that leg still consumes its slots.
|
||||
* The promotion gate is Will's own answer to that: a candidate that does not
|
||||
* clear its per-type threshold is not discarded, it is moved behind the ones
|
||||
* that do, and the freed slots go to whichever leg still has promoted material.
|
||||
*
|
||||
* ENGRAM_WM_LEG_SCAN bounds the per-leg work: only the head of each already
|
||||
* sorted leg can reach a result slot at any sane limit.
|
||||
*/
|
||||
#define ENGRAM_WM_LEG_SCAN 64
|
||||
|
||||
static int eg_wm_promote(const EngramNode* n, const char* q, double bg,
|
||||
double* wm_out) {
|
||||
/* Same product engram_activate's pass 2 computes (l.8519), minus the
|
||||
* inhibitory / inhibition-of-return terms, which need activation state
|
||||
* this read path does not carry. */
|
||||
double bias = engram_goal_bias(n, q);
|
||||
double impf = (n->importance > 0.0) ? (0.5 + n->importance) : 1.0;
|
||||
double wm = bg * bias * n->confidence * impf;
|
||||
if (wm_out) *wm_out = wm;
|
||||
return wm > engram_type_threshold(n->node_type, n->tier);
|
||||
}
|
||||
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -9654,25 +9699,12 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
}
|
||||
if (sem && n->emb && n->emb_dim == qdim) {
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
/* Semantic leg, claim 24 verbatim: "returning the node
|
||||
* records whose embedding vectors have the HIGHEST
|
||||
* COSINE SIMILARITY to a query vector" — a ranking, with
|
||||
* no threshold anywhere in the claim. The leg used to be
|
||||
* gated at ENGRAM_EMBED_SEED_MIN and rescaled onto
|
||||
* [SEED_MIN,1]; that constant is defined (l.6083) as the
|
||||
* SEED-JOIN threshold for the HippoRAG pass, and reusing
|
||||
* it as a result filter is not authorised by claim 24.
|
||||
* Measured on this corpus, it is also not a quality
|
||||
* gate: true paraphrase targets score 0.459-0.657 while
|
||||
* the nonsense controls' own nearest neighbours score
|
||||
* 0.553-0.622 — the distributions overlap, so no value
|
||||
* of the constant separates them. What actually holds
|
||||
* the nonsense control is corpus vocabulary (see the
|
||||
* nhits==0 gate below), not cosine magnitude.
|
||||
* Claim 32: clamp the cosine to [0,1] rather than let a
|
||||
* negative value invert the signal. */
|
||||
if (c > 0.0) {
|
||||
double sv = c > 1.0 ? 1.0 : c;
|
||||
/* Semantic leg: identical to eg_sem_term(), which is
|
||||
* left in place and still used by engram_search().
|
||||
* Inlined here only so one cosine serves both uses. */
|
||||
if (c > ENGRAM_EMBED_SEED_MIN) {
|
||||
double sv = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN);
|
||||
if (sv > 1.0) sv = 1.0;
|
||||
sem[nsem].idx = i; sem[nsem].sem = sv; nsem++;
|
||||
}
|
||||
/* Graph seeds: top-K by RAW cosine, insertion-ordered. */
|
||||
@@ -9690,21 +9722,6 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
}
|
||||
}
|
||||
}
|
||||
/* CORPUS-VOCABULARY GATE — the thing that actually keeps an
|
||||
* unfloored semantic leg from answering gibberish.
|
||||
* nhits == 0 means NO stored record contains ANY query token
|
||||
* anywhere in its content, label or tags: the query is outside
|
||||
* the graph's vocabulary entirely. A vector index always has a
|
||||
* nearest neighbour, so without this gate the semantic leg
|
||||
* answers "zqxjvw plimforth grebulon" with its 0.55-cosine
|
||||
* garbage. It is also the honest reading of Will's retrieval
|
||||
* contract: 05-detailed-description l.64 has the caller supply
|
||||
* "one or more seed node UUIDs representing the current active
|
||||
* context", and every leg here is downstream of finding those
|
||||
* seeds. No seeds, no retrieval — the graph declines rather
|
||||
* than confabulates. Suppressing the graph seeds too keeps the
|
||||
* associative leg from running off the semantic top-K alone. */
|
||||
if (nhits == 0) { nsem = 0; nsemseed = 0; }
|
||||
/* BM25-shaped lexical score. Binary term frequency (the match
|
||||
* primitive is a substring test, not a count), Lucene-form IDF,
|
||||
* and length normalisation over the corpus mean. A token that
|
||||
@@ -9742,8 +9759,77 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
: 0;
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave3(hits, nhits, sem, nsem,
|
||||
assoc, nassoc, lim, order);
|
||||
/* ── Layer 1 → background activation, per leg ──
|
||||
* Each leg's raw score is put on a common [0,1] footing
|
||||
* WITHOUT blending the legs against each other (that
|
||||
* failed in the score-fusion cut of the semantic leg —
|
||||
* two rankings with different spreads cannot be summed).
|
||||
* Lexical: BM25 over the score a node would earn covering
|
||||
* every query token at mean field length, so the scale is
|
||||
* "how much of this query's rare vocabulary did you
|
||||
* actually account for". Semantic: the shift-and-floor
|
||||
* value, already in [0,1]. Associative: cosine to the
|
||||
* query, which is what orders that leg. */
|
||||
double idf_sum = 0.0;
|
||||
for (int t = 0; t < ntok; t++) idf_sum += idf[t];
|
||||
double w_ideal = idf_sum * (ENGRAM_BM25_K1 + 1.0)
|
||||
/ (1.0 + ENGRAM_BM25_K1);
|
||||
if (w_ideal <= 0.0) w_ideal = 1.0;
|
||||
|
||||
int64_t nLs = nhits < ENGRAM_WM_LEG_SCAN ? nhits : ENGRAM_WM_LEG_SCAN;
|
||||
int64_t nSs = nsem < ENGRAM_WM_LEG_SCAN ? nsem : ENGRAM_WM_LEG_SCAN;
|
||||
int64_t nAs = nassoc < ENGRAM_WM_LEG_SCAN ? nassoc : ENGRAM_WM_LEG_SCAN;
|
||||
|
||||
EngramRankEntry Lp[ENGRAM_WM_LEG_SCAN], Lq[ENGRAM_WM_LEG_SCAN];
|
||||
EngramSemEntry Sp[ENGRAM_WM_LEG_SCAN], Sq[ENGRAM_WM_LEG_SCAN];
|
||||
EngramSemEntry Ap[ENGRAM_WM_LEG_SCAN], Aq[ENGRAM_WM_LEG_SCAN];
|
||||
int64_t nLp = 0, nLq = 0, nSp = 0, nSq = 0, nAp = 0, nAq = 0;
|
||||
|
||||
/* ── Layer 2 → promote or suppress. Nothing is dropped. */
|
||||
for (int64_t i = 0; i < nLs; i++) {
|
||||
double bg = hits[i].w / w_ideal;
|
||||
if (bg > 1.0) bg = 1.0;
|
||||
if (eg_wm_promote(&g->nodes[hits[i].idx], q, bg, NULL))
|
||||
Lp[nLp++] = hits[i];
|
||||
else
|
||||
Lq[nLq++] = hits[i];
|
||||
}
|
||||
for (int64_t i = 0; i < nSs; i++) {
|
||||
/* RAW cosine, not the shift-and-floor value. The
|
||||
* first cut of this filter fed pass 2 the shifted
|
||||
* value, which for a genuine match (c .60-.70) is
|
||||
* 0.02-0.25 — under every per-type threshold — so the
|
||||
* whole semantic leg was suppressed while lexical
|
||||
* junk cleared its gate. Pass 2's thresholds are
|
||||
* calibrated against activation strengths in [0,1],
|
||||
* which is the scale raw cosine is on. Measured cost
|
||||
* of getting this wrong: paraphrase 61.5% -> 23.1%. */
|
||||
double bg = sem[i].sem * (1.0 - ENGRAM_EMBED_SEED_MIN)
|
||||
+ ENGRAM_EMBED_SEED_MIN;
|
||||
if (eg_wm_promote(&g->nodes[sem[i].idx], q, bg, NULL))
|
||||
Sp[nSp++] = sem[i];
|
||||
else
|
||||
Sq[nSq++] = sem[i];
|
||||
}
|
||||
for (int64_t i = 0; i < nAs; i++) {
|
||||
if (eg_wm_promote(&g->nodes[assoc[i].idx], q, assoc[i].sem, NULL))
|
||||
Ap[nAp++] = assoc[i];
|
||||
else
|
||||
Aq[nAq++] = assoc[i];
|
||||
}
|
||||
|
||||
/* Promoted material fills the head, in leg order; the
|
||||
* background-only field follows behind it (claim 45). */
|
||||
int64_t no = engram_interleave3(Lp, nLp, Sp, nSp,
|
||||
Ap, nAp, lim, order, 0);
|
||||
if (no < lim)
|
||||
no = engram_interleave3(Lq, nLq, Sq, nSq,
|
||||
Aq, nAq, lim, order, no);
|
||||
/* Anything past the scanned head of each leg, only if the
|
||||
* filter left the result short of the caller's limit. */
|
||||
if (no < lim)
|
||||
no = engram_interleave3(hits, nhits, sem, nsem,
|
||||
assoc, nassoc, lim, order, no);
|
||||
for (int64_t k = 0; k < no; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[order[k]], 0);
|
||||
|
||||
Reference in New Issue
Block a user