Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Tim Lingo
|
4eb4c9e287 | ||
|
Tim Lingo
|
55f9ee3cb0 | ||
|
Tim Lingo
|
9c39084e60 | ||
|
Tim Lingo
|
6f3a048f36 |
@@ -0,0 +1,21 @@
|
||||
import numpy as np, json, urllib.request
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
np.seterr(all='ignore')
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries']
|
||||
VALS=sorted({r for q in gold if q['category']=='paraphrase' for r in q['relevant']})
|
||||
VI=[eidx[v] for v in VALS]
|
||||
def emb(t):
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
return v/(np.linalg.norm(v)+1e-9)
|
||||
print("qid cat bestValueNodeGlobalRank goldGlobalRank goldSiblingRank")
|
||||
for q in gold:
|
||||
if q['category']!='paraphrase': continue
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
ranks=sorted(int((s>s[j]).sum())+1 for j in VI)
|
||||
g=eidx[q['relevant'][0]]; gr=int((s>s[g]).sum())+1
|
||||
sv=np.array([s[j] for j in VI]); sib=int((sv>s[g]).sum())+1
|
||||
print("%-4s %-11s best=%-5d (top3 val ranks %s) gold=%-5d sib=%d" % (q['id'],q['category'],ranks[0],ranks[:3],gr,sib))
|
||||
@@ -0,0 +1,43 @@
|
||||
import json,sys,pickle,numpy as np,itertools
|
||||
sys.path.insert(0,'.')
|
||||
from policy2 import legs3,outcome,G,NODES,merge
|
||||
# cache per-query leg id-lists, floored and unfloored
|
||||
cache={}
|
||||
for q in G['queries']:
|
||||
Lf,Sf,Af=legs3(q['query'])
|
||||
Lu,Su,Au=legs3(q['query'],unfloor=True)
|
||||
cache[q['id']]=dict(L=Lf,Sf=Sf,A=Af,Su=Su,Au=Au)
|
||||
pickle.dump(cache,open('ceil.pkl','wb'))
|
||||
def mrg(pattern,L,S,A,lim=10):
|
||||
out=[];p={'L':0,'S':0,'A':0};src={'L':L,'S':S,'A':A}
|
||||
i=0
|
||||
while len(out)<lim:
|
||||
prog=False
|
||||
for ch in pattern:
|
||||
lst=src[ch]
|
||||
if p[ch]<len(lst):
|
||||
x=lst[p[ch]];p[ch]+=1;prog=True
|
||||
if x not in out: out.append(x)
|
||||
if len(out)>=lim: return out
|
||||
if not prog: break
|
||||
return out
|
||||
def ev(pattern,unfl):
|
||||
res={}
|
||||
for q in G['queries']:
|
||||
c=cache[q['id']]
|
||||
S=c['Su'] if unfl else c['Sf']
|
||||
ids=[NODES[i]['id'] for i in mrg(pattern,c['L'],S,c['A'],10)]
|
||||
res[q['id']]=outcome(q,ids)
|
||||
return res
|
||||
base=ev('LSA',False)
|
||||
print("baseline",sum(base.values()))
|
||||
best=[]
|
||||
pats=['LSA','LAS','SLA','ALS','SAL','ASL','LSSA','LSASA','LSAA','LSSAA','LSAS','SSLA','LLSA','SALSA','LSAAS']
|
||||
for unfl in (False,True):
|
||||
for p in pats:
|
||||
r=ev(p,unfl)
|
||||
g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k])
|
||||
best.append((len(g)-len(l),p,unfl,g,l))
|
||||
best.sort(reverse=True)
|
||||
for n,p,u,g,l in best[:10]:
|
||||
print("net=%+d pat=%-6s unfloor=%s gains=%s losses=%s"%(n,p,u,g,l))
|
||||
@@ -0,0 +1,146 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "wsclaim24",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q14",
|
||||
"q25"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q28",
|
||||
"q33",
|
||||
"q34"
|
||||
],
|
||||
"discordant": 6,
|
||||
"net_queries": -2,
|
||||
"mcnemar_exact_p": 0.6875,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5768475572047,
|
||||
"recall@10": 0.6563414759843332,
|
||||
"precision@5": 0.19428571428571437,
|
||||
"mrr@10": 0.5026530612244898,
|
||||
"nonsense_clean": "0/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 524.8,
|
||||
"latency_ms_p95": 738.7,
|
||||
"latency_ms_max": 755.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 0,
|
||||
"avg_false_positives": 10.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"baseline": "assoc-leg",
|
||||
"candidate": "semseed",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q11"
|
||||
],
|
||||
"discordant": 4,
|
||||
"net_queries": 2,
|
||||
"mcnemar_exact_p": 0.625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6285714285714286,
|
||||
"hit@5_max": 0.6285714285714286,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "wordstart",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q35"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 1,
|
||||
"net_queries": 1,
|
||||
"mcnemar_exact_p": 1.0,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 542.6,
|
||||
"latency_ms_p95": 741.3,
|
||||
"latency_ms_max": 758.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
{
|
||||
"baseline": "hybrid-semantic",
|
||||
"candidate": "semseed",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22",
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q11"
|
||||
],
|
||||
"discordant": 8,
|
||||
"net_queries": 6,
|
||||
"mcnemar_exact_p": 0.0703125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
{
|
||||
"baseline": "semseed",
|
||||
"candidate": "bm25lex",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q11"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 2,
|
||||
"net_queries": 2,
|
||||
"mcnemar_exact_p": 0.5,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import numpy as np, json, urllib.request, collections, sys
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
N={n['id']:n for n in d['nodes']}
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list); hasstruct=set()
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
hasstruct.add(e['from_id']); hasstruct.add(e['to_id'])
|
||||
del d
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
LEX={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"))['rows']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
def inter3(L,S,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def run(mode, K=0):
|
||||
res={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if s[j]>SEED_MIN]
|
||||
A=[]
|
||||
if mode!='hybrid':
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
if mode=='semseed':
|
||||
seeds=seeds+[eids[j] for j in ordr[:K] if eids[j] in N and eids[j] not in seeds]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
res[qid]=inter3(L,S,A)
|
||||
return res
|
||||
def score(res,label):
|
||||
hits=0; det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok = (len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
rel=q['relevant']; must=q.get('must_outrank') or {}
|
||||
ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok = any(r in out for r in rel)
|
||||
else: ok = any(r in out for r in q['relevant'])
|
||||
det[qid]=ok; hits+=ok
|
||||
print("%-22s outcome-true=%d/38" % (label,hits))
|
||||
return det
|
||||
print("gold sample keys:", list(list(gold.values())[0].keys()))
|
||||
mk=[q for q in gold.values() if q['category']=='superseded'][0]
|
||||
print("superseded fields:", {k:v for k,v in mk.items() if k!='derivation'})
|
||||
a=score(run('hybrid'),'sim hybrid(L+S)')
|
||||
b=score(run('lexseed'),'sim assoc(lex seeds)')
|
||||
for K in (3,5,10):
|
||||
c=score(run('semseed',K),'sim assoc(+sem K=%d)'%K)
|
||||
d=[q for q in gold if c[q]!=b[q]]
|
||||
print(" vs lexseed: moved=%d gains=%s losses=%s"%(len(d),[q for q in d if c[q]],[q for q in d if not c[q]]))
|
||||
e=[q for q in gold if c[q]!=a[q]]
|
||||
print(" vs hybrid : moved=%d gains=%s losses=%s"%(len(e),[q for q in e if c[q]],[q for q in e if not c[q]]))
|
||||
|
||||
print("\n=== Will's own constant ENGRAM_EMBED_SEED_K = 8 ===")
|
||||
c=score(run('semseed',8),'sim assoc(+sem K=8)')
|
||||
for base,lab in ((b,'lexseed(iter2)'),(a,'hybrid(iter1 KEEP)')):
|
||||
dd=[q for q in gold if c[q]!=base[q]]
|
||||
print(" vs %-18s moved=%d gains=%s losses=%s"%(lab,len(dd),[q for q in dd if c[q]],[q for q in dd if not c[q]]))
|
||||
# diagnostic: what is assoc rank-1 for each paraphrase query at K=8
|
||||
print("\nassoc leg head at K=8 (paraphrase):")
|
||||
for qid,q in gold.items():
|
||||
if q['category'] not in ('paraphrase','nonsense'): continue
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]; ordr=np.argsort(-s)
|
||||
seeds=[x for x in L[:3] if x in N]+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in L[:3]]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
rel=set(q['relevant']); gr=next((i+1 for i,x in enumerate(A) if x in rel),None)
|
||||
print(" %-4s %-11s |A|=%-4d goldAssocRank=%-5s head=%s"%(qid,q['category'],len(A),gr,
|
||||
[ (N[x].get('label') or x)[:26] for x in A[:3] ]))
|
||||
@@ -0,0 +1,117 @@
|
||||
import json,pickle,os,math,urllib.request,numpy as np
|
||||
S='/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/sim/'
|
||||
C=pickle.load(open(S+'corpus.pkl','rb'))
|
||||
NODES=C['nodes']; EDGES=C['edges']; N=len(NODES)
|
||||
E=np.load(S+'emb.npy'); HAVE=np.load(S+'have.npy')
|
||||
En=E/np.maximum(np.linalg.norm(E,axis=1,keepdims=True),1e-12)
|
||||
LAYERS={int(l['layer_id']):l for l in (C['layers'] or [])} if C['layers'] else {}
|
||||
TRANS=set(i for i,l in LAYERS.items() if l.get('transparent'))
|
||||
def addressable(s):
|
||||
if not s: return False
|
||||
return all(0x20<=ord(ch)<=0x7e for ch in s)
|
||||
ADDR=np.array([addressable(n['id']) for n in NODES])
|
||||
OK=np.array([ (n['layer_id'] not in TRANS) and ADDR[i] for i,n in enumerate(NODES)])
|
||||
SAL=np.array([n['salience'] for n in NODES])
|
||||
LOW=[ (n['content']+'\x00'+n['label']+'\x00'+n['tags']).lower() for n in NODES]
|
||||
DL=np.array([float(len(n['content'])+len(n['label'])+len(n['tags'])) for n in NODES])
|
||||
IDX={}
|
||||
for i,n in enumerate(NODES):
|
||||
IDX.setdefault(n['id'],i)
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
ADJ_F=[[] for _ in range(N)]; ADJ_T=[[] for _ in range(N)]
|
||||
for e in EDGES:
|
||||
a=IDX.get(e['from']); b=IDX.get(e['to'])
|
||||
if a is None or b is None: continue
|
||||
ADJ_F[a].append((e,b)); ADJ_T[b].append((e,a))
|
||||
EXCL=np.array([n['node_type'] in ('Tag','InternalStateEvent') for n in NODES])
|
||||
avgdl_all=None
|
||||
def tokenize(q):
|
||||
out=[]
|
||||
for t in q.split():
|
||||
if not any(t.lower()==x.lower() for x in out): out.append(t)
|
||||
return out
|
||||
_qcache={}
|
||||
def qemb(q):
|
||||
if q in _qcache: return _qcache[q]
|
||||
body=json.dumps({"model":"nomic-embed-text","prompt":q}).encode()
|
||||
r=urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=body,headers={"Content-Type":"application/json"}),timeout=30)
|
||||
v=np.array(json.loads(r.read())["embedding"],dtype=np.float32)
|
||||
v=v/np.linalg.norm(v); _qcache[q]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
SEED_MIN=0.60; SEED_K=8; ASSOC_SEEDS=3; DEPTH=2; FIRE=0.02; AMAX=64; DECAY=0.7
|
||||
def legs(query):
|
||||
toks=tokenize(query)
|
||||
masks=[];
|
||||
hit_idx=[]; hit_mask=[]
|
||||
df=[0]*len(toks)
|
||||
lt=[t.lower() for t in toks]
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i]; m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s: m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i); hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum()); avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl); w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
qv=qemb(query)
|
||||
cos=En@qv
|
||||
cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)
|
||||
semfull=[(int(i),float(cos[i])) for i in order[:400]]
|
||||
Sleg=[(i,(c-SEED_MIN)/(1-SEED_MIN)) for i,c in semfull if c>SEED_MIN]
|
||||
semseed=[i for i,c in semfull[:SEED_K] if c>0.0]
|
||||
# assoc
|
||||
act={}; seen={}; qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0; seen[i]=2; qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0; seen[i]=2; qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh]; qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT: continue
|
||||
if EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=[]
|
||||
for i,st in seen.items():
|
||||
if st!=1: continue
|
||||
if not OK[i] or not HAVE[i]: continue
|
||||
c=float(cos[i])
|
||||
if c<=0.0: continue
|
||||
A.append((i,c))
|
||||
A.sort(key=lambda x:-x[1]); A=A[:AMAX]
|
||||
return L,Sleg,A,cos
|
||||
def interleave3(L,Sl,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(Sl) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li][0] not in out: out.append(L[li][0])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(Sl):
|
||||
if Sl[si][0] not in out: out.append(Sl[si][0])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai][0] not in out: out.append(A[ai][0])
|
||||
ai+=1
|
||||
return out
|
||||
@@ -0,0 +1,102 @@
|
||||
import json,sys,pickle,numpy as np
|
||||
sys.path.insert(0,'.')
|
||||
from legs import *
|
||||
GP='/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/'
|
||||
G=json.load(open(GP+'gold_set.json'))
|
||||
def legs3(query, sem_sal=False, assoc_sal=False, unfloor=False, sem_cap=None):
|
||||
toks=tokenize(query)
|
||||
hit_idx=[];hit_mask=[];df=[0]*len(toks);lt=[t.lower() for t in toks]
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i];m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s: m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i);hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl);w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
if not L: return [],[],[]
|
||||
qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)[:600]
|
||||
cand=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)]
|
||||
key=(lambda i:(SAL[i] if sem_sal else 1.0)*float(cos[i]))
|
||||
Sl=sorted(cand,key=lambda i:-key(i))
|
||||
if sem_cap: Sl=Sl[:sem_cap]
|
||||
semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0]
|
||||
act={};seen={};qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh];qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT: continue
|
||||
if EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=[]
|
||||
for i,st in seen.items():
|
||||
if st!=1 or not OK[i] or not HAVE[i]: continue
|
||||
c=float(cos[i])
|
||||
if c<=0.0: continue
|
||||
A.append((i,(SAL[i] if assoc_sal else 1.0)*c))
|
||||
A.sort(key=lambda x:-x[1]);A=[i for i,_ in A[:AMAX]]
|
||||
return [i for i,_,_ in L],Sl,A
|
||||
def merge(L,S,A,lim=10):
|
||||
out=[];li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def outcome(q,ids):
|
||||
c=q['category']
|
||||
if c=='nonsense': return len(ids)==0
|
||||
if c=='superseded':
|
||||
a,b=q['must_outrank']
|
||||
if a not in ids: return False
|
||||
if b not in ids: return True
|
||||
return ids.index(a)<ids.index(b)
|
||||
return any(x in ids[:5] for x in q['relevant'])
|
||||
def run(**kw):
|
||||
return {q['id']:outcome(q,[NODES[i]['id'] for i in merge(*legs3(q['query'],**kw),10)]) for q in G['queries']}
|
||||
base=run()
|
||||
print("baseline",sum(base.values()),"/38 misses:",[k for k,v in base.items() if not v])
|
||||
import itertools
|
||||
for name,kw in [
|
||||
('sem_sal(floored)',dict(sem_sal=True)),
|
||||
('unfloor',dict(unfloor=True)),
|
||||
('unfloor+sem_sal',dict(unfloor=True,sem_sal=True)),
|
||||
('assoc_sal',dict(assoc_sal=True)),
|
||||
('unfloor+sem_sal+assoc_sal',dict(unfloor=True,sem_sal=True,assoc_sal=True)),
|
||||
('sem_sal+assoc_sal(floored)',dict(sem_sal=True,assoc_sal=True)),
|
||||
]:
|
||||
r=run(**kw)
|
||||
g=sorted(k for k in base if r[k] and not base[k]); l=sorted(k for k in base if base[k] and not r[k])
|
||||
print("%-28s net=%+d gains=%s losses=%s"%(name,len(g)-len(l),g,l))
|
||||
@@ -0,0 +1,959 @@
|
||||
{
|
||||
"label": "bm25lex-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25lex",
|
||||
"soul_md5": "dfbd0f8e3646212db5c60026f8ad906f",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 50.1,
|
||||
"child_pid": 90371,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1183.8,
|
||||
"latency_ms_p95": 1611.3,
|
||||
"latency_ms_max": 1654.7,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 282.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 315.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 303.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 587.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 511.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 497.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 510.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"2c56a7a9-5323-4ce4-ba09-35836ba15d54",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 870.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"830ca37a-d334-4e41-ba89-64893dc8d628",
|
||||
"?",
|
||||
"ce9636dc-85a5-4dae-9e07-74ea2fcc6307",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 723.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"66c63082-b4da-4aa1-8fee-848db8a83210",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1584.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"f0eb6b13-909c-4674-91ef-23301d3abc8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"30a44d10-2487-420e-bf61-3892e4343c92",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"bfb5809e-d19a-4d3f-8c1a-796db622ad9d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1652.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"tag-fiction",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1341.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"bl-164b520b-c503-49db-89f9-bd2fdf4215f5",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"1219277c-1b95-45ec-95a2-07b4a47a4d92",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1064.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"bl-448bc514-c2f1-4520-a9b1-1f3a73678d26",
|
||||
"a1000001-0000-0000-0000-000000000012",
|
||||
"43098881-e044-482b-8e92-471728a8ba8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1398.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"d6b12ecf-702b-4101-b1bb-09ed9b220b29",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"c608a095-c98b-4bfa-bfe1-1611c1320290",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"451ae007-4219-4096-89fe-fa2e045fbeb1",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1252.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1581.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1363.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"5a2c118a-87bd-4239-97a7-9e02c5991983",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1611.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a0edad47-5f77-4fc3-a546-1e85f8c68e77",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"96497334-b18f-495c-9228-eeb8182bdc38",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1470.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"75e036d3-c170-4e3f-acc2-e456a6850ee2",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1513.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-759e78ca-5394-4244-aa39-1c1468bc5f3e",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1226.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"bl-0d8c5dfa-e163-4fef-a58b-56b0d076c5a8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"tag-childhood"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1342.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"3499d5da-0e9c-4de4-9bc4-8941b14e0b1f",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1433.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1104.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1183.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1199.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1202.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"?Z?<S???K ?",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"'?T?a\"B~-?8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1414.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 703.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 492.9,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 728.4,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1289.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7ac62daa-2eac-4c7a-a97e-e4203fc1b57b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1654.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 995.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,959 @@
|
||||
{
|
||||
"label": "bm25lex",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25lex",
|
||||
"soul_md5": "dfbd0f8e3646212db5c60026f8ad906f",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 50.4,
|
||||
"child_pid": 90325,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 287.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 316.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 284.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 304.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 282.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 591.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 516.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 520.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 507.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 509.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"2c56a7a9-5323-4ce4-ba09-35836ba15d54",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 872.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"830ca37a-d334-4e41-ba89-64893dc8d628",
|
||||
"?",
|
||||
"ce9636dc-85a5-4dae-9e07-74ea2fcc6307",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 735.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"66c63082-b4da-4aa1-8fee-848db8a83210",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1605.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"f0eb6b13-909c-4674-91ef-23301d3abc8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"30a44d10-2487-420e-bf61-3892e4343c92",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"bfb5809e-d19a-4d3f-8c1a-796db622ad9d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1655.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"tag-fiction",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1339.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"bl-164b520b-c503-49db-89f9-bd2fdf4215f5",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"1219277c-1b95-45ec-95a2-07b4a47a4d92",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1065.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"bl-448bc514-c2f1-4520-a9b1-1f3a73678d26",
|
||||
"a1000001-0000-0000-0000-000000000012",
|
||||
"43098881-e044-482b-8e92-471728a8ba8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1388.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"d6b12ecf-702b-4101-b1bb-09ed9b220b29",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"c608a095-c98b-4bfa-bfe1-1611c1320290",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"451ae007-4219-4096-89fe-fa2e045fbeb1",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1246.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1595.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1359.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"5a2c118a-87bd-4239-97a7-9e02c5991983",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1620.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a0edad47-5f77-4fc3-a546-1e85f8c68e77",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"96497334-b18f-495c-9228-eeb8182bdc38",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1478.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"75e036d3-c170-4e3f-acc2-e456a6850ee2",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1516.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-759e78ca-5394-4244-aa39-1c1468bc5f3e",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1237.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"bl-0d8c5dfa-e163-4fef-a58b-56b0d076c5a8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"tag-childhood"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1337.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"3499d5da-0e9c-4de4-9bc4-8941b14e0b1f",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1433.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1114.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1184.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1185.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1199.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"?Z?<S???K ?",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"'?T?a\"B~-?8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1422.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 703.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 488.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 720.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1301.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7ac62daa-2eac-4c7a-a97e-e4203fc1b57b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1651.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1021.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,957 @@
|
||||
{
|
||||
"label": "semseed-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-semseed",
|
||||
"soul_md5": "ab0c82781215f43d4907bd6f39d3615b",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 52.0,
|
||||
"child_pid": 88833,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1245.9,
|
||||
"latency_ms_p95": 1668.7,
|
||||
"latency_ms_max": 1713.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 349.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 294.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 319.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 339.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 318.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 604.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 541.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 515.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 532.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 887.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 762.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1659.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1713.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"?of?7???",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"????7???Ջ3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1402.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"7?e?7???\f3?",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"??o?'?B???k",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1129.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"?of?7???",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"ctx-cc7f",
|
||||
"ctx-4a41",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1452.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1293.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1622.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1412.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"?of?7???",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1668.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1535.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1565.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1279.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1408.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1538.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1168.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1290.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"ctx-bb74",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1277.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.09090909090909091,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1486.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 757.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 510.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 764.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1340.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1701.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"%???2??jH??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1029.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,957 @@
|
||||
{
|
||||
"label": "semseed",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-semseed",
|
||||
"soul_md5": "ab0c82781215f43d4907bd6f39d3615b",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 52.7,
|
||||
"child_pid": 88601,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 295.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 341.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 313.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 310.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 318.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 310.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 610.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 531.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 888.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 762.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1652.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1706.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"?of?7???",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"????7???Ջ3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1419.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"7?e?7???\f3?",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"??o?'?B???k",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"?of?7???",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"ctx-cc7f",
|
||||
"ctx-4a41",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1489.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1330.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1671.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1449.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"?of?7???",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1710.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1565.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1629.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1326.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1382.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1530.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1159.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1232.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"ctx-bb74",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1227.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1240.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.09090909090909091,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1464.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 756.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 517.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 755.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1329.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1692.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"%???2??jH??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1040.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,948 @@
|
||||
{
|
||||
"label": "wordstart-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-wordstart",
|
||||
"soul_md5": "32d4cf77672658a5f49dc7c9213e3ba2",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 28.7,
|
||||
"child_pid": 1490,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 537.6,
|
||||
"latency_ms_p95": 733.7,
|
||||
"latency_ms_max": 761.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 165.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 164.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 163.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 172.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 167.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 164.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 281.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 254.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 255.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 247.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 263.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"?Q??m?;?u?'",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 397.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"mem-fb44a2fc-7405-41ff-87b3-84643ac07313",
|
||||
"?",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 341.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-0228da71-d7f7-4f3b-b7b3-c5eede42b62a",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 761.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7452eb8f-be01-4b55-aec1-ff0c29e790f6",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"532277bf-2959-4beb-ae0d-b018c97678ee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"d4015bd7-c592-4ed8-8574-1f15ad37af75"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 733.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"tag-fiction",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-ca4d6a34-d354-413f-bc86-126cc17ca81c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 629.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"4da5dbaf-46e5-4f3e-b474-f60d9f8241d3",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 529.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bfad516b-c306-4c4c-874a-a347c46c05c2",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 747.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"345b6420-e004-4d2e-b55c-6a729393fa99",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-92a7fdc5-9dd0-48cf-a691-506058de3838",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 564.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 689.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"34356a36-0df5-4020-8dcc-5e7a423f8d4c",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 729.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-1b58b05c-9305-4f06-a586-a08c96008027",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"ctx-4a41",
|
||||
"mem-5708f4c9-3d61-4182-8543-2843698931e6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 638.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 679.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-a16deccb-16a7-419c-a013-ff824a4daa15",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"tag-childhood",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 613.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"efe53612-6914-4936-8e3b-1e694eb174e5",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 652.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 538.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 533.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"719aa819-00a9-4f4b-a857-4f9fe5ad44d7",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 673.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 350.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 254.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 345.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-c17aefb1-38b5-4ced-af50-fe524127e1a4",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 639.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"de3b6428-b76c-4c44-90e0-bf1dd6998027"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 733.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 507.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,948 @@
|
||||
{
|
||||
"label": "wordstart",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-wordstart",
|
||||
"soul_md5": "32d4cf77672658a5f49dc7c9213e3ba2",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 28.9,
|
||||
"child_pid": 1420,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 542.6,
|
||||
"latency_ms_p95": 741.3,
|
||||
"latency_ms_max": 758.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 170.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 210.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 167.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 166.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 162.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 162.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 306.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 261.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 272.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 265.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 271.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"?Q??m?;?u?'",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 412.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"mem-fb44a2fc-7405-41ff-87b3-84643ac07313",
|
||||
"?",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 377.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-0228da71-d7f7-4f3b-b7b3-c5eede42b62a",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 758.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7452eb8f-be01-4b55-aec1-ff0c29e790f6",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"532277bf-2959-4beb-ae0d-b018c97678ee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"d4015bd7-c592-4ed8-8574-1f15ad37af75"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 741.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"tag-fiction",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-ca4d6a34-d354-413f-bc86-126cc17ca81c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 642.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"4da5dbaf-46e5-4f3e-b474-f60d9f8241d3",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bfad516b-c306-4c4c-874a-a347c46c05c2",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 725.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"345b6420-e004-4d2e-b55c-6a729393fa99",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-92a7fdc5-9dd0-48cf-a691-506058de3838",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 562.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 686.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 584.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"34356a36-0df5-4020-8dcc-5e7a423f8d4c",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 719.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-1b58b05c-9305-4f06-a586-a08c96008027",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"ctx-4a41",
|
||||
"mem-5708f4c9-3d61-4182-8543-2843698931e6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 634.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 674.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-a16deccb-16a7-419c-a013-ff824a4daa15",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 571.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"tag-childhood",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 603.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"efe53612-6914-4936-8e3b-1e694eb174e5",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 656.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 536.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 562.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 556.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"719aa819-00a9-4f4b-a857-4f9fe5ad44d7",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 683.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 350.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 258.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 348.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-c17aefb1-38b5-4ced-af50-fe524127e1a4",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 664.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"de3b6428-b76c-4c44-90e0-bf1dd6998027"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 747.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 504.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
import numpy as np, json, urllib.request, collections, math, re, sys, time
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
t0=time.time()
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
print("loaded corpus %.1fs"%(time.time()-t0),file=sys.stderr)
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list)
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
nodes=d['nodes']
|
||||
N={n['id']:n for n in nodes}
|
||||
PRINT=re.compile(r'^[\x20-\x7e]+$')
|
||||
ids=[]; hay=[]; dl=[]; sal=[]; addressable=[]
|
||||
for n in nodes:
|
||||
i=n.get('id') or ''
|
||||
h=((n.get('content') or '')+'\x00'+(n.get('label') or '')+'\x00'+(n.get('tags') or '')).lower()
|
||||
ids.append(i); hay.append(h); dl.append(len(h)); sal.append(float(n.get('salience') or 0.0))
|
||||
addressable.append(bool(PRINT.match(i)))
|
||||
del d
|
||||
NN=len(ids); avgdl=sum(dl)/NN
|
||||
print("nodes=%d avgdl=%.0f addressable=%d %.1fs"%(NN,avgdl,sum(addressable),time.time()-t0),file=sys.stderr)
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
LEXMAIN={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"),) ['rows']} if False else {r['id']:r['returned'] for r in json.load(open(EV+"results-main.json",encoding='utf-8',errors='surrogateescape'))['rows']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
def lexleg(query, mode, guard, lim=10):
|
||||
toks=[]
|
||||
for w in query.split():
|
||||
wl=w.lower()
|
||||
if wl not in toks: toks.append(wl)
|
||||
nt=len(toks)
|
||||
masks=[]; df=[0]*nt
|
||||
for i in range(NN):
|
||||
if guard and not addressable[i]: continue
|
||||
h=hay[i]; m=0; sc=0
|
||||
for t in range(nt):
|
||||
if toks[t] in h: m|=(1<<t); sc+=1; df[t]+=1
|
||||
if sc: masks.append((i,m,sc))
|
||||
if mode=='tokcount':
|
||||
masks.sort(key=lambda x:(-x[2], -sal[x[0]]))
|
||||
return [ids[i] for i,m,sc in masks[:lim]]
|
||||
idf=[math.log(1.0+(NN-df[t]+0.5)/(df[t]+0.5)) for t in range(nt)]
|
||||
scored=[]
|
||||
for i,m,sc in masks:
|
||||
norm=1.0-B+B*dl[i]/avgdl
|
||||
s=0.0
|
||||
for t in range(nt):
|
||||
if m&(1<<t): s+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
scored.append((s,i))
|
||||
scored.sort(key=lambda x:(-x[0], -sal[x[1]]))
|
||||
return [ids[i] for s,i in scored[:lim]]
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
def inter3(L,S,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def run(mode, guard, use_main_lex=False):
|
||||
res={}; legs={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L = LEXMAIN[qid][:10] if use_main_lex else lexleg(q['query'], mode, guard)
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if s[j]>SEED_MIN]
|
||||
if guard: S=[x for x in S if PRINT.match(x or '')]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and (not guard or PRINT.match(eids[j] or ''))]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
if guard: A=[x for x in A if PRINT.match(x or '')]
|
||||
res[qid]=inter3(L,S,A); legs[qid]=(L,S,A)
|
||||
return res,legs
|
||||
def score(res,label,verbose=False):
|
||||
det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok=(len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
must=q.get('must_outrank') or {}; ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok=any(r in out for r in q['relevant'])
|
||||
else: ok=any(r in out for r in q['relevant'])
|
||||
det[qid]=ok
|
||||
print("%-28s outcome-true=%d/38"%(label,sum(det.values())))
|
||||
return det
|
||||
if __name__=="__main__":
|
||||
base,_=run('tokcount',False,use_main_lex=True); b=score(base,'BASELINE semseed(real lex)')
|
||||
variants=[('tokcount',False,'replica: tokcount,noguard'),
|
||||
('tokcount',True ,'A: tokcount + idguard'),
|
||||
('bm25', False,'B: bm25 only'),
|
||||
('bm25', True ,'C: bm25 + idguard')]
|
||||
dets={}
|
||||
for m,g,lab in variants:
|
||||
r,_=run(m,g); dets[lab]=score(r,lab)
|
||||
dd=[q for q in gold if dets[lab][q]!=b[q]]
|
||||
print(" vs BASELINE moved=%d gains=%s losses=%s"%(len(dd),[q for q in dd if dets[lab][q]],[q for q in dd if not dets[lab][q]]))
|
||||
@@ -0,0 +1,92 @@
|
||||
import json,sys,pickle,numpy as np
|
||||
sys.path.insert(0,'.')
|
||||
from legs import *
|
||||
GP='/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/'
|
||||
G=json.load(open(GP+'gold_set.json'))
|
||||
def wstart(s,tok):
|
||||
i=s.find(tok)
|
||||
while i!=-1:
|
||||
if i==0 or not s[i-1].isalnum(): return True
|
||||
i=s.find(tok,i+1)
|
||||
return False
|
||||
def legs4(query, wordstart=False, unfloor=False):
|
||||
toks=tokenize(query); lt=[t.lower() for t in toks]
|
||||
hit_idx=[];hit_mask=[];df=[0]*len(toks)
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i];m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s and (not wordstart or wstart(s,tok)): m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i);hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl);w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
if not L: return [],[],[]
|
||||
qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)[:600]
|
||||
Sl=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)]
|
||||
semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0]
|
||||
act={};seen={};qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh];qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT or EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=sorted([(i,float(cos[i])) for i,st in seen.items() if st==1 and OK[i] and HAVE[i] and cos[i]>0.0],key=lambda x:-x[1])[:AMAX]
|
||||
return [i for i,_,_ in L],Sl,[i for i,_ in A]
|
||||
def merge(L,S,A,lim=10):
|
||||
out=[];li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def outcome(q,ids):
|
||||
c=q['category']
|
||||
if c=='nonsense': return len(ids)==0
|
||||
if c=='superseded':
|
||||
a,b=q['must_outrank']
|
||||
if a not in ids: return False
|
||||
if b not in ids: return True
|
||||
return ids.index(a)<ids.index(b)
|
||||
return any(x in ids[:5] for x in q['relevant'])
|
||||
def run(**kw):
|
||||
return {q['id']:outcome(q,[NODES[i]['id'] for i in merge(*legs4(q['query'],**kw),10)]) for q in G['queries']}
|
||||
base=run()
|
||||
print("baseline",sum(base.values()),"misses",[k for k,v in base.items() if not v])
|
||||
for name,kw in [('wordstart',dict(wordstart=True)),
|
||||
('unfloor',dict(unfloor=True)),
|
||||
('wordstart+unfloor',dict(wordstart=True,unfloor=True))]:
|
||||
r=run(**kw)
|
||||
g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k])
|
||||
print("%-20s net=%+d gains=%s losses=%s"%(name,len(g)-len(l),g,l))
|
||||
@@ -0,0 +1,31 @@
|
||||
import numpy as np, json, urllib.request
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
M=np.load(SP+'/emb.npy'); ids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
np.seterr(all='ignore')
|
||||
bad=~np.isfinite(M).all(axis=1)
|
||||
M[bad]=0.0
|
||||
print("non-finite rows zeroed:",int(bad.sum()))
|
||||
idx={k:i for i,k in enumerate(ids)}
|
||||
gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries']
|
||||
def emb(t):
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
return v/(np.linalg.norm(v)+1e-9)
|
||||
out={}
|
||||
for q in gold:
|
||||
v=emb(q['query']); s=M@v
|
||||
s=s[np.isfinite(s)]
|
||||
mu=float(s.mean()); sd=float(s.std())
|
||||
top=np.sort(s)[::-1][:10]
|
||||
z=[(float(t)-mu)/sd for t in top]
|
||||
grank=[]
|
||||
for rel in q['relevant']:
|
||||
if rel in idx:
|
||||
j=idx[rel]; grank.append((int((M@v > (M@v)[j]).sum())+1, round(float((M@v)[j]),3)))
|
||||
grank.sort()
|
||||
out[q['id']]=dict(cat=q['category'],mu=round(mu,3),sd=round(sd,4),top1=round(float(top[0]),3),
|
||||
z1=round(z[0],2),z3=round(z[2],2),z5=round(z[4],2),gold=grank[:1])
|
||||
print("%s %-11s mu=%.3f sd=%.4f top1=%.3f z1=%5.2f z3=%5.2f z5=%5.2f gold=%s"%(
|
||||
q['id'],q['category'],mu,sd,top[0],z[0],z[2],z[4],grank[:1]))
|
||||
json.dump(out,open(SP+'/zprobe.json','w'),indent=1)
|
||||
+218
-12
@@ -7327,6 +7327,39 @@ static int istr_contains(const char* hay, const char* needle) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Word-START-anchored variant of istr_contains.
|
||||
*
|
||||
* WHY. The retrieval match primitive is a raw substring test, so a query token
|
||||
* matches ANYWHERE inside a corpus word: "throom" matches "bathroom", "cat"
|
||||
* matches "concatenate". Measured on this corpus over the 38-query gold set,
|
||||
* that is not a rare accident — it is the bulk of some queries' candidate
|
||||
* sets. q28's lexical leg is 36,954 records of which only 13 contain a query
|
||||
* token at a word start (99.96% mid-word noise); six other queries carry
|
||||
* ~20,500 mid-word-only records each; and the nonsense control q35
|
||||
* ("xxqzzt vurblenacht throom") returns 7 records ALL of which match only
|
||||
* mid-word, which is the entire reason that control has been dirty since main.
|
||||
*
|
||||
* WHAT CHANGES. A token must begin at a word boundary — the preceding
|
||||
* character is not alphanumeric. Suffixes are still matched ("value" still
|
||||
* hits "values", "unjailbreakable" still hits "unjailbreakables"), so this is
|
||||
* strictly a prefix anchor, not whole-word equality; whole-word equality would
|
||||
* break the morphological matching the phrase category depends on.
|
||||
*
|
||||
* PROVENANCE, stated honestly: this restores no engram claim. Will's design
|
||||
* has no lexical leg at all (05-detailed-description l.64 takes "one or more
|
||||
* seed node UUIDs representing the current active context" as its input), so
|
||||
* the lexical leg is the seed-finding step that feeds the designed mechanism.
|
||||
* Cleaner seeds serve that mechanism; they do not replace it. */
|
||||
static int istr_contains_wordstart(const char* hay, const char* needle) {
|
||||
if (!hay || !needle || !*needle) return 0;
|
||||
size_t nl = strlen(needle);
|
||||
for (const char* p = hay; *p; p++) {
|
||||
if (p != hay && isalnum((unsigned char)p[-1])) continue;
|
||||
if (strncasecmp(p, needle, nl) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Tokenized query matching ───────────────────────────────────────────
|
||||
* The engram query surface (search / activate / goal-bias) historically
|
||||
* matched the ENTIRE raw query string as a single case-insensitive
|
||||
@@ -7379,14 +7412,56 @@ static int engram_node_match_score(const EngramNode* n,
|
||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||
int score = 0;
|
||||
for (int t = 0; t < ntok; t++) {
|
||||
if (istr_contains(n->content, toks[t]) ||
|
||||
istr_contains(n->label, toks[t]) ||
|
||||
istr_contains(n->tags, toks[t]))
|
||||
if (istr_contains_wordstart(n->content, toks[t]) ||
|
||||
istr_contains_wordstart(n->label, toks[t]) ||
|
||||
istr_contains_wordstart(n->tags, toks[t]))
|
||||
score++;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
/* Same match test as engram_node_match_score, but returns the SET of matched
|
||||
* query tokens as a bitmask instead of only their count. ENGRAM_MAX_QTOKENS is
|
||||
* 32, so one uint32 covers every token the tokenizer can produce. The mask is
|
||||
* what lets the caller accumulate a per-token document frequency in the SAME
|
||||
* pass that finds the hits — no second scan of the corpus. */
|
||||
static uint32_t engram_node_match_mask(const EngramNode* n,
|
||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||
uint32_t m = 0;
|
||||
for (int t = 0; t < ntok && t < 32; t++) {
|
||||
if (istr_contains_wordstart(n->content, toks[t]) ||
|
||||
istr_contains_wordstart(n->label, toks[t]) ||
|
||||
istr_contains_wordstart(n->tags, toks[t]))
|
||||
m |= (uint32_t)1u << t;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/* Searchable byte length of a node: the same three fields the match test
|
||||
* reads. Used as the BM25 document length so a long node does not out-match a
|
||||
* short one merely by containing more text. */
|
||||
static double engram_node_len(const EngramNode* n) {
|
||||
double l = 0.0;
|
||||
if (n->content) l += (double)strlen(n->content);
|
||||
if (n->label) l += (double)strlen(n->label);
|
||||
if (n->tags) l += (double)strlen(n->tags);
|
||||
return l;
|
||||
}
|
||||
|
||||
/* Addressability guard. Claim 23 stores node records under a key encoding the
|
||||
* node identifier, claim 12 deduplicates merged results by node identifier,
|
||||
* and claim 27's competition map is indexed by node identifier — every one of
|
||||
* those requires the identifier to be a usable string. This corpus contains
|
||||
* records whose id field is binary garbage (a save-side corruption); they are
|
||||
* unfetchable by any caller, so returning one wastes a result slot. Printable
|
||||
* ASCII, non-empty, is the whole test. */
|
||||
static int eg_node_addressable(const EngramNode* n) {
|
||||
const unsigned char* p = (const unsigned char*)n->id;
|
||||
if (!p || !*p) return 0;
|
||||
for (; *p; p++) if (*p < 0x20 || *p > 0x7e) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Semantic leg of the read path (engram claim 24). Returns the query/target
|
||||
* cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0],
|
||||
* and exactly 0.0 when the pair is not comparable (no query embedding, target
|
||||
@@ -7406,7 +7481,12 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
|
||||
* (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
|
||||
* the semantic leg is a second ranking merged beside it, never a reweighting
|
||||
* of this one. */
|
||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||
typedef struct {
|
||||
int64_t idx; int score; double salience;
|
||||
uint32_t mask; /* which query tokens matched (BM25 leg) */
|
||||
double len; /* searchable byte length (BM25 leg) */
|
||||
double w; /* BM25-shaped weighted score */
|
||||
} EngramRankEntry;
|
||||
static int engram_rank_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
@@ -7416,6 +7496,21 @@ static int engram_rank_cmp(const void* a, const void* b) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* BM25-shaped ordering for the read path's lexical leg: rare-term weight and
|
||||
* length normalisation instead of a raw distinct-token count. Salience stays
|
||||
* the tiebreak, exactly as in engram_rank_cmp. */
|
||||
#define ENGRAM_BM25_K1 1.2
|
||||
#define ENGRAM_BM25_B 0.75
|
||||
static int engram_rank_w_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
if (ea->w < eb->w) return 1; /* desc */
|
||||
if (ea->w > eb->w) return -1;
|
||||
if (ea->salience < eb->salience) return 1;
|
||||
if (ea->salience > eb->salience) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Semantic rank entry: node index and its renormalized query similarity,
|
||||
* ordered by similarity desc. This is the claim-24 "embedding search"
|
||||
* ranking, computed independently of the lexical one. */
|
||||
@@ -7531,8 +7626,30 @@ static int eg_assoc_excluded(const EngramNode* n) {
|
||||
* function of hop count and ranks the relay hub above all of its own
|
||||
* children. Composing the two is mine, not Will's — the description ranks the
|
||||
* activation result set by strength (l.78). */
|
||||
/* Semantic seeding of the graph leg — the HippoRAG pass Will documents at
|
||||
* l.6082: "the query is embedded, the top-K nodes by cosine >= SEED_MIN join
|
||||
* the seed set", using his own ENGRAM_EMBED_SEED_K (8). It is "similarity used
|
||||
* twice, coherently": cosine picks where to STAND in the graph, the structural
|
||||
* walk decides what is REACHABLE from there, and cosine then ORDERS what was
|
||||
* reached (iteration-2's finding, kept intact).
|
||||
*
|
||||
* Why the seed list is NOT floored at ENGRAM_EMBED_SEED_MIN here. That
|
||||
* constant is calibrated for a cosine scale this corpus does not have: with
|
||||
* nomic-embed-text every true paraphrase target measures 0.46-0.66, and the
|
||||
* three nonsense controls' own nearest neighbours measure 0.55/0.60/0.62 —
|
||||
* they OVERLAP, so no absolute cosine floor separates signal from gibberish
|
||||
* (measured, all 38 queries). Iteration 2 established the same thing one step
|
||||
* later in the pipeline: applying the floor to graph CANDIDATES removed every
|
||||
* gain, because within a structurally-reached neighbourhood relative cosine
|
||||
* still discriminates below the absolute threshold. The gate that actually
|
||||
* works is reachability — eg_rel_is_structural() plus the firing threshold.
|
||||
* A semantically-near node with no structural attachment expands to nothing
|
||||
* and contributes nothing, which is exactly what happens to gibberish: the
|
||||
* nearest neighbours of q33/q34 are unattached, so their graph leg is empty.
|
||||
*/
|
||||
static int64_t engram_assoc_leg(EngramStore* g,
|
||||
const EngramRankEntry* L, int64_t nL,
|
||||
const int64_t* semseed, int64_t nsemseed,
|
||||
const float* qv, int32_t qdim,
|
||||
EngramSemEntry* out, int64_t out_cap) {
|
||||
if (!g || nL <= 0 || !qv || qdim <= 0 || out_cap <= 0) return 0;
|
||||
@@ -7553,6 +7670,14 @@ static int64_t engram_assoc_leg(EngramStore* g,
|
||||
act[idx] = 1.0; seen[idx] = 2; /* 2 = seed: never a result */
|
||||
if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; }
|
||||
}
|
||||
/* ...and the semantic seeds, on the same footing (strength 1.0, hop 0). */
|
||||
for (int64_t s = 0; s < nsemseed; s++) {
|
||||
int64_t idx = semseed[s];
|
||||
if (idx < 0 || idx >= g->node_count) continue;
|
||||
if (seen[idx]) continue;
|
||||
act[idx] = 1.0; seen[idx] = 2;
|
||||
if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; }
|
||||
}
|
||||
|
||||
const double SPREAD_DECAY = 0.7;
|
||||
while (qh < qt) {
|
||||
@@ -7585,6 +7710,7 @@ static int64_t engram_assoc_leg(EngramStore* g,
|
||||
if (seen[i] != 1) continue; /* skip unreached and seeds */
|
||||
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
||||
EngramNode* nd = &g->nodes[i];
|
||||
if (!eg_node_addressable(nd)) continue; /* unfetchable record */
|
||||
if (!nd->emb || nd->emb_dim != qdim) continue;
|
||||
double c = eg_cosine(nd->emb, qv, qdim);
|
||||
if (c <= 0.0) continue;
|
||||
@@ -9524,23 +9650,90 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
/* Raw (unfloored) cosine top-K, kept for the graph leg's
|
||||
* semantic seeds. Selected in THIS pass so the cosine is
|
||||
* computed exactly once per node — the seeding costs no extra
|
||||
* pass over the corpus and no extra embed round-trip. */
|
||||
int64_t semseed[ENGRAM_EMBED_SEED_K];
|
||||
double semseedc[ENGRAM_EMBED_SEED_K];
|
||||
int64_t nsemseed = 0;
|
||||
/* BM25 statistics gathered in this same pass: per-token
|
||||
* document frequency, and the corpus mean field length. */
|
||||
int64_t df[ENGRAM_MAX_QTOKENS];
|
||||
for (int t = 0; t < ntok; t++) df[t] = 0;
|
||||
double dl_total = 0.0;
|
||||
int64_t dl_n = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Filter transparent layers — same as engram_search. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
if (sc > 0) {
|
||||
/* Unaddressable records cannot be fetched by a caller and
|
||||
* must not consume a result slot (claims 12/23/27). */
|
||||
if (!eg_node_addressable(n)) continue;
|
||||
double dl = engram_node_len(n);
|
||||
dl_total += dl; dl_n++;
|
||||
uint32_t mask = engram_node_match_mask(n, toks, ntok);
|
||||
if (mask) {
|
||||
int sc = 0;
|
||||
for (int t = 0; t < ntok; t++)
|
||||
if (mask & ((uint32_t)1u << t)) { sc++; df[t]++; }
|
||||
hits[nhits].idx = i;
|
||||
hits[nhits].score = sc;
|
||||
hits[nhits].salience = n->salience;
|
||||
hits[nhits].mask = mask;
|
||||
hits[nhits].len = dl;
|
||||
hits[nhits].w = 0.0;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
if (sem && n->emb && n->emb_dim == qdim) {
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
/* Semantic leg: identical to eg_sem_term(), which is
|
||||
* left in place and still used by engram_search().
|
||||
* Inlined here only so one cosine serves both uses. */
|
||||
if (c > ENGRAM_EMBED_SEED_MIN) {
|
||||
double sv = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN);
|
||||
if (sv > 1.0) sv = 1.0;
|
||||
sem[nsem].idx = i; sem[nsem].sem = sv; nsem++;
|
||||
}
|
||||
/* Graph seeds: top-K by RAW cosine, insertion-ordered. */
|
||||
if (c > 0.0 && (nsemseed < ENGRAM_EMBED_SEED_K
|
||||
|| c > semseedc[nsemseed - 1])) {
|
||||
int64_t p = nsemseed < ENGRAM_EMBED_SEED_K
|
||||
? nsemseed : ENGRAM_EMBED_SEED_K - 1;
|
||||
while (p > 0 && semseedc[p - 1] < c) {
|
||||
semseedc[p] = semseedc[p - 1];
|
||||
semseed[p] = semseed[p - 1];
|
||||
p--;
|
||||
}
|
||||
semseedc[p] = c; semseed[p] = i;
|
||||
if (nsemseed < ENGRAM_EMBED_SEED_K) nsemseed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
/* BM25-shaped lexical score. Binary term frequency (the match
|
||||
* primitive is a substring test, not a count), Lucene-form IDF,
|
||||
* and length normalisation over the corpus mean. A token that
|
||||
* occurs in 30,000 nodes now weighs far less than one that
|
||||
* occurs in 1, and a 1.3 MB record no longer out-matches a
|
||||
* 300-byte one by sheer surface area. */
|
||||
double avgdl = dl_n ? (dl_total / (double)dl_n) : 1.0;
|
||||
if (avgdl <= 0.0) avgdl = 1.0;
|
||||
double idf[ENGRAM_MAX_QTOKENS];
|
||||
for (int t = 0; t < ntok; t++) {
|
||||
double dfx = (double)df[t];
|
||||
idf[t] = log(1.0 + ((double)dl_n - dfx + 0.5) / (dfx + 0.5));
|
||||
}
|
||||
for (int64_t h = 0; h < nhits; h++) {
|
||||
double norm = 1.0 - ENGRAM_BM25_B
|
||||
+ ENGRAM_BM25_B * (hits[h].len / avgdl);
|
||||
double s = 0.0;
|
||||
for (int t = 0; t < ntok; t++)
|
||||
if (hits[h].mask & ((uint32_t)1u << t))
|
||||
s += idf[t] * (ENGRAM_BM25_K1 + 1.0)
|
||||
/ (1.0 + ENGRAM_BM25_K1 * norm);
|
||||
hits[h].w = s;
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
/* Claim-10 associative leg: expand the top lexical hits along
|
||||
* structural relations only, order the reached set by query
|
||||
@@ -9549,9 +9742,22 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
* lexical ordering safe. */
|
||||
EngramSemEntry* assoc = qv ? malloc((size_t)ENGRAM_ASSOC_MAX * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nassoc = assoc
|
||||
? engram_assoc_leg(g, hits, nhits, qv, qdim, assoc, ENGRAM_ASSOC_MAX)
|
||||
? engram_assoc_leg(g, hits, nhits, semseed, nsemseed,
|
||||
qv, qdim, assoc, ENGRAM_ASSOC_MAX)
|
||||
: 0;
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
/* Corpus-vocabulary gate. If no stored record contains ANY
|
||||
* query token in its content, label or tags, the query is
|
||||
* outside this graph's vocabulary: there are no seeds, and
|
||||
* 05-detailed-description l.64 makes retrieval downstream of
|
||||
* seeds ("the caller provides one or more seed node UUIDs
|
||||
* representing the current active context"). No seeds, no
|
||||
* retrieval — the graph declines rather than confabulating a
|
||||
* nearest neighbour for gibberish. The mechanism is iteration
|
||||
* 6's (feat/claim24-unfloored-semantic); it is required here
|
||||
* because word-start matching empties the lexical leg for
|
||||
* q35-style queries whose only "hits" were mid-word, and the
|
||||
* semantic leg would otherwise answer them anyway. */
|
||||
int64_t* order = (nhits > 0) ? malloc((size_t)lim * sizeof(int64_t)) : NULL;
|
||||
if (order) {
|
||||
int64_t no = engram_interleave3(hits, nhits, sem, nsem,
|
||||
assoc, nassoc, lim, order);
|
||||
|
||||
Reference in New Issue
Block a user