Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Tim Lingo
|
77e3c5fa26 |
@@ -1,143 +0,0 @@
|
||||
{
|
||||
"baseline": "hybrid-semantic",
|
||||
"candidate": "assoc-leg",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 4,
|
||||
"net_queries": 4,
|
||||
"mcnemar_exact_p": 0.125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
{
|
||||
"baseline": "baseline-embcorpus",
|
||||
"candidate": "hybrid-semantic",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q15",
|
||||
"q16",
|
||||
"q20",
|
||||
"q21",
|
||||
"q26",
|
||||
"q37"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 6,
|
||||
"net_queries": 6,
|
||||
"mcnemar_exact_p": 0.03125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1145.9,
|
||||
"latency_ms_p95": 1574.3,
|
||||
"latency_ms_max": 1634.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
{
|
||||
"baseline": "main-ext",
|
||||
"candidate": "stack-ext",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q15",
|
||||
"q16",
|
||||
"q18",
|
||||
"q19",
|
||||
"q20",
|
||||
"q21",
|
||||
"q22",
|
||||
"q26",
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31",
|
||||
"q35",
|
||||
"q37",
|
||||
"q40",
|
||||
"q44",
|
||||
"q48",
|
||||
"q49",
|
||||
"q50"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 20,
|
||||
"net_queries": 20,
|
||||
"mcnemar_exact_p": 1.9073486328125e-06,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.18461538461538463,
|
||||
"recall@5": 0.14510073260073258,
|
||||
"recall@10": 0.1794871794871795,
|
||||
"precision@5": 0.06461538461538462,
|
||||
"mrr@10": 0.15847985347985344,
|
||||
"nonsense_clean": "9/10",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1380.2,
|
||||
"latency_ms_p95": 2293.8,
|
||||
"latency_ms_max": 2879.1,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 9,
|
||||
"avg_false_positives": 1.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.47692307692307695,
|
||||
"recall@5": 0.3750415183107491,
|
||||
"recall@10": 0.45561340369032677,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.3055555555555555,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 640.9,
|
||||
"latency_ms_p95": 1011.1,
|
||||
"latency_ms_max": 1190.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.16666666666666666,
|
||||
"recall@5": 0.16666666666666666,
|
||||
"recall@10": 0.26666666666666666,
|
||||
"mrr@10": 0.0762037037037037
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import json,sys,time,urllib.request,threading,queue
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
OUT=sys.argv[1]
|
||||
URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text"
|
||||
MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
tasks=[]
|
||||
for n in d["nodes"]:
|
||||
c=n.get("content") or ""; t=n.get("node_type") or ""
|
||||
if len(c)<8: continue # eg_embed_eligible
|
||||
if t in ("InternalStateEvent","Tag"): continue
|
||||
b=c.encode('utf-8',errors='surrogateescape')[:MAXB]
|
||||
tasks.append((n.get("id") or "", b.decode('utf-8',errors='replace')))
|
||||
del d
|
||||
print("tasks",len(tasks),flush=True)
|
||||
q=queue.Queue(); [q.put(t) for t in tasks]
|
||||
lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0]
|
||||
def work():
|
||||
while True:
|
||||
try: nid,txt=q.get_nowait()
|
||||
except queue.Empty: return
|
||||
v=None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
body=json.dumps({"model":MODEL,"prompt":txt}).encode()
|
||||
r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"})
|
||||
with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"]
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt==2:
|
||||
with lock: fails[0]+=1
|
||||
time.sleep(0.5)
|
||||
with lock:
|
||||
if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n")
|
||||
done[0]+=1
|
||||
if done[0]%2000==0:
|
||||
el=time.time()-t0
|
||||
print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True)
|
||||
f.flush()
|
||||
ths=[threading.Thread(target=work) for _ in range(8)]
|
||||
[t.start() for t in ths]; [t.join() for t in ths]
|
||||
f.close()
|
||||
print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True)
|
||||
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set.
|
||||
|
||||
WHY THIS EXISTS
|
||||
Iteration 7 measured the instrument's own ceiling: from the current baseline
|
||||
only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable
|
||||
by anything constructible. The decision floor is 6. An instrument whose
|
||||
ceiling is below its own floor cannot certify or refute anything, so the
|
||||
gold set — not the retriever — became the blocker.
|
||||
|
||||
This script does NOT touch q01..q38. It loads gold_set.json verbatim and
|
||||
appends new queries numbered from q39 up, so every prior result file, every
|
||||
committed baseline, and every per-query id stays valid and comparable.
|
||||
|
||||
WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST
|
||||
heldout_paraphrase Targets were sampled MECHANICALLY (fixed seed 8080) from
|
||||
corpus nodes that are addressable, 500-2600 chars, of a
|
||||
real content type, and NOT part of a duplicate cluster
|
||||
larger than 3. The existing gold answer space was
|
||||
excluded, so no new query can be answered by a node the
|
||||
old set already used. Queries were then authored by
|
||||
reading ONLY the sampled node text — no retrieval was run
|
||||
against any build before authoring, so the set cannot be
|
||||
fitted to a candidate. The same zero-overlap proof the
|
||||
original paraphrase category uses is enforced here: if a
|
||||
single content word of the query appears anywhere in the
|
||||
target's label, content or tags, the query is REJECTED,
|
||||
not quietly kept.
|
||||
|
||||
This is the category the old set could not measure. Its
|
||||
13 original paraphrase queries and all 6 associative
|
||||
queries share ONE answer space — the 13 `Self - Values
|
||||
(grounded)` children (iteration 3, finding 3). So 19 of
|
||||
35 scored queries tested retrieval against a single
|
||||
13-node neighbourhood. These do not touch that
|
||||
neighbourhood at all.
|
||||
|
||||
nonsense Extra controls, fully mechanical: a string qualifies only
|
||||
if NONE of its tokens occurs anywhere in the corpus.
|
||||
A semantic leg has a nearest neighbour for gibberish too,
|
||||
so widening this control is the guard against a retriever
|
||||
that "improves" recall by answering everything.
|
||||
|
||||
WHAT THIS SCRIPT DELIBERATELY DOES NOT DO
|
||||
It does not add exact_rare or phrase queries. Both categories are already at
|
||||
100% on the current stack; adding more would add regression-guard ballast
|
||||
that no candidate can move, which is precisely the defect being fixed.
|
||||
|
||||
usage:
|
||||
python3 extend_gold_set.py <snapshot.json> [--base gold_set.json]
|
||||
[--out gold_set_extended.json] [--check]
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
|
||||
|
||||
# Identical stopword list to build_gold_set.py. Duplicated deliberately: this
|
||||
# file must be able to re-prove its own queries without importing a module whose
|
||||
# constants could drift.
|
||||
STOP = set("""
|
||||
a about above after again against all also am an and any are aren't as at be because been
|
||||
before being below between both but by can can't cannot could couldn't did didn't do does
|
||||
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
|
||||
having he her here hers herself him himself his how i if in into is isn't it its itself just
|
||||
me more most my myself no nor not of off on once only or other others ought our ours ourselves
|
||||
out over own same shan't she should shouldn't so some such than that the their theirs them
|
||||
themselves then there these they this those through to too under until up very was wasn't we
|
||||
were weren't what when where which while who whom why will with won't would wouldn't you your
|
||||
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
|
||||
things does doing done keep keeps kept go goes going come comes came one two something anything
|
||||
""".split())
|
||||
|
||||
|
||||
def doctext(n):
|
||||
return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
|
||||
|
||||
|
||||
def content_tokens(s):
|
||||
return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# HELD-OUT PARAPHRASE SEEDS
|
||||
#
|
||||
# (target_id, query, why-this-target-is-unmistakable)
|
||||
#
|
||||
# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query
|
||||
# text is mine, written from the node body alone. The zero-overlap check below
|
||||
# is what makes the category meaningful — it is re-proved on every run, so the
|
||||
# set cannot decay into lexical matching, and a leak fails loudly.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
HELDOUT_PARAPHRASE_SEEDS = [
|
||||
("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5",
|
||||
"understating your abilities so nobody feels threatened",
|
||||
"node is about deliberately not leading with full capability so people stay at ease"),
|
||||
|
||||
("mem-fd65b83d-298f-4387-a665-d0227c3426bc",
|
||||
"a hidden fleet able to hunt down rogue machines everywhere",
|
||||
"node describes silently shipped instances forming a distributed force against misaligned agents"),
|
||||
|
||||
("4a0e9adc-2bfb-476b-aa93-424d2a499220",
|
||||
"sketch a brief blueprint and clear it upstairs before construction starts",
|
||||
"node is the standing rule that a short specification precedes any building"),
|
||||
|
||||
("696e609c-da7a-4394-8a0c-106ba07dc6c3",
|
||||
"the reply arrived as bare prose so the caller's parser threw",
|
||||
"node pins a bug where a plain-text body was unconditionally decoded as structured data"),
|
||||
|
||||
("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb",
|
||||
"repeated catalogue keys blew up the scrolling grid",
|
||||
"node is the crash caused by two identical ids in a seeded catalogue"),
|
||||
|
||||
("8257157a-ce42-44ca-a1b9-300c3bb0a9a1",
|
||||
"tracing each defect back to whichever invention it violated",
|
||||
"node maps observed bugs onto the specific patent each one breaches"),
|
||||
|
||||
("791256bb-5a85-4775-96ef-7af56c848858",
|
||||
"a check that stops the mind clobbering a populated store when it boots",
|
||||
"node is the genesis seed-guard that refuses to re-seed over a populated store"),
|
||||
|
||||
("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10",
|
||||
"telling it to consult the internet had to happen deep inside, not at the surface",
|
||||
"node records that the web-search directive only worked from the system prompt"),
|
||||
|
||||
("bl-080fb268-94b0-486d-80ce-7b363fc5f19b",
|
||||
"standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping",
|
||||
"node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"),
|
||||
|
||||
("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"punctuation that pledges and then pays off rather than clarifying",
|
||||
"node analyses the colon as a promise-then-delivery device rather than an explanatory one"),
|
||||
|
||||
("9b4f0d93-4129-4746-8eb1-d10d955bd777",
|
||||
"an easily missed feature finally given its own permanent spot in the navigation",
|
||||
"node moves a capability out of a hidden menu into the sidebar"),
|
||||
|
||||
("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a",
|
||||
"checking preconditions up front so a stage aborts before fetching anything",
|
||||
"node is the gate precondition engine that short-circuits ahead of retrieval"),
|
||||
|
||||
("b199c76d-5d76-49dd-94ee-56b432200a97",
|
||||
"producing the other platform's installer inside an emulated desktop",
|
||||
"node records building the Windows package in a virtual machine"),
|
||||
|
||||
("bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"chained add-ons that may inspect, rewrite or veto traffic in flight",
|
||||
"node is the interceptor pipeline on the message bus"),
|
||||
|
||||
("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2",
|
||||
"settling what the shareable bundles and the storefront would be called",
|
||||
"node records the naming decisions for distributable packages and the marketplace"),
|
||||
|
||||
("371c8a5d-c78b-4a67-978f-80691a29ecb3",
|
||||
"the emergency-escalation pledge on the marketing site is unenforced in what actually ships",
|
||||
"node is the launch blocker that the promised safety gate is absent from the app"),
|
||||
|
||||
("ac578b30-948b-41bd-b69d-399bfef80c50",
|
||||
"the distributable image finally assembled and its startup check passed",
|
||||
"node records a successful installer build whose boot gate passed"),
|
||||
|
||||
("49401e2c-a3b5-415f-aa06-aff4be90688e",
|
||||
"shuffling and appending stages in a draft before anything executes",
|
||||
"node is the editable plan card with reorder and add-step"),
|
||||
|
||||
("ac857d80-ece8-4b7e-9e3d-f7c775569fa3",
|
||||
"orders handed down from above, with the tighter one winning any disagreement",
|
||||
"node is program-level instruction inheritance with project override"),
|
||||
|
||||
("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9",
|
||||
"shrinking generated text via encodings that compound on each other",
|
||||
"node is the streaming output compression design with four stacking schemes"),
|
||||
|
||||
("8e60516a-203b-4d51-9d44-822e6195cbde",
|
||||
"splitting a system by what varies, with firm limits on which pieces may invoke which",
|
||||
"node is the grounded summary of Will's decomposition principles and their invariants"),
|
||||
|
||||
("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7",
|
||||
"a newcomer curious if the fighting overseas counted as positive",
|
||||
"node is the internal-state event triggered by April's question about the war"),
|
||||
|
||||
("71fa439e-b9a2-4f57-a93b-971f3a7eca8e",
|
||||
"stripping every hard-coded colour literal in favour of named design values",
|
||||
"node is the premium foundation pass replacing inline hex with semantic tokens"),
|
||||
|
||||
("5ca9607c-cfb3-45c3-99f4-67281272c9eb",
|
||||
"reducing how curved the tiny selectors look so they agree with their neighbours",
|
||||
"node is the chip corner-radius standardization"),
|
||||
|
||||
("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c",
|
||||
"walking through a doorway and being reassembled from base substance far away",
|
||||
"node is the quantum-gate plus nanotech teleportation vision"),
|
||||
|
||||
("132ded95-08e2-4474-aba0-198684484b02",
|
||||
"the compiled result sits on disk while the process still runs something older",
|
||||
"node records that the regenerated source was committed while the running daemon was old"),
|
||||
|
||||
("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"gathering what each phase needs while the procedure is authored, not while it executes",
|
||||
"node is the per-step compiled context package item"),
|
||||
|
||||
("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7",
|
||||
"the inward reaction when a peer answered as an equal",
|
||||
"node is the internal-state event logged on reading Claude's reply"),
|
||||
|
||||
("0f99ec6f-942a-46ba-82ea-42835798d3b9",
|
||||
"flattening every raised surface across the entire product",
|
||||
"node is the quiet-luxury sweep turning off elevation app-wide"),
|
||||
|
||||
("5585f251-37fc-48cd-a176-f0ea42cfeb63",
|
||||
"buyers supply their own provider credentials and consumption goes untallied",
|
||||
"node is the launch audit finding BYOK-only inference with no usage metering"),
|
||||
]
|
||||
|
||||
# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs
|
||||
# anywhere in the corpus; otherwise it is REJECTED, never silently kept.
|
||||
EXTRA_NONSENSE_SEEDS = [
|
||||
"brimquast folnerity zubbolax",
|
||||
"wexlithorp granuvestal",
|
||||
"quorbindle thrapsimony vexnu",
|
||||
"plovaxith mundrelque",
|
||||
"zibbernaut craxlefond thurm",
|
||||
"yalquenbrist opharvel",
|
||||
"drexinomal quithbarrow",
|
||||
]
|
||||
|
||||
|
||||
def load_corpus(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
data = json.load(fh)
|
||||
nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
|
||||
edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
|
||||
return nodes, edges
|
||||
|
||||
|
||||
def build_extension(nodes):
|
||||
byid = {n["id"]: n for n in nodes}
|
||||
|
||||
# Duplicate clusters: 47.4% of this corpus is redundant and one single record
|
||||
# accounts for 46.6% of all nodes. A held-out target must not sit inside a
|
||||
# cluster, and if it does have exact copies they ALL count as correct.
|
||||
h2ids = defaultdict(list)
|
||||
for n in nodes:
|
||||
h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"])
|
||||
|
||||
all_tokens = set()
|
||||
for n in nodes:
|
||||
all_tokens |= set(TOKEN.findall(doctext(n).lower()))
|
||||
|
||||
new, problems = [], []
|
||||
|
||||
for target, query, why in HELDOUT_PARAPHRASE_SEEDS:
|
||||
if target not in byid:
|
||||
problems.append(f"heldout_paraphrase target {target} not in corpus")
|
||||
continue
|
||||
tgt_tokens = content_tokens(doctext(byid[target]))
|
||||
qt = content_tokens(query)
|
||||
leak = sorted(qt & tgt_tokens)
|
||||
if leak:
|
||||
problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}")
|
||||
continue
|
||||
h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest()
|
||||
rel = sorted(h2ids[h])
|
||||
new.append({
|
||||
"category": "heldout_paraphrase",
|
||||
"query": query,
|
||||
"relevant": rel,
|
||||
"derivation": (
|
||||
f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, "
|
||||
f"500-2600 char content nodes outside the original gold answer space and outside "
|
||||
f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the "
|
||||
f"{len(qt)} content words in the query, ZERO appear anywhere in the target's "
|
||||
f"label, content or tags, so no string-matching retriever can reach it. "
|
||||
f"Exact content duplicates of the target ({len(rel)}) all count as correct. "
|
||||
f"Authored without running retrieval against any build."),
|
||||
"zero_overlap_verified": True,
|
||||
"query_content_words": sorted(qt),
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
for s in EXTRA_NONSENSE_SEEDS:
|
||||
present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
|
||||
if present:
|
||||
problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
|
||||
continue
|
||||
new.append({
|
||||
"category": "nonsense",
|
||||
"query": s,
|
||||
"relevant": [],
|
||||
"derivation": ("CONTROL (held-out). Verified at build time that none of this string's "
|
||||
"tokens occurs anywhere in the corpus. Correct behaviour is to return "
|
||||
"NOTHING; any result is a false positive."),
|
||||
"expect_empty": True,
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
return new, problems
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("snapshot")
|
||||
ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json"))
|
||||
ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json"))
|
||||
ap.add_argument("--check", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
nodes, _edges = load_corpus(args.snapshot)
|
||||
base = json.load(open(args.base, encoding="utf-8"))
|
||||
baseq = base["queries"]
|
||||
print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries")
|
||||
|
||||
new, problems = build_extension(nodes)
|
||||
|
||||
# Number the appended queries AFTER the highest existing id so q01..q38 are
|
||||
# byte-identical to the committed set and every prior result file still lines up.
|
||||
start = max(int(q["id"][1:]) for q in baseq)
|
||||
for i, q in enumerate(new, 1):
|
||||
q["id"] = f"q{start + i:02d}"
|
||||
|
||||
from collections import Counter
|
||||
print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} REJECTED (not silently kept):")
|
||||
for p in problems:
|
||||
print(" -", p)
|
||||
|
||||
if args.check:
|
||||
sys.exit(1 if problems else 0)
|
||||
|
||||
doc = dict(base)
|
||||
doc["queries"] = baseq + new
|
||||
doc["note"] = (base.get("note", "") +
|
||||
" EXTENDED: queries above q%02d are the original committed set, unchanged. "
|
||||
"Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their "
|
||||
"targets were sampled mechanically from outside the original answer space and "
|
||||
"the paraphrases were authored without running retrieval against any build."
|
||||
% (start, start + 1))
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {args.out} ({len(doc['queries'])} queries total)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,17 +0,0 @@
|
||||
import json,sys
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
TSV,OUT=sys.argv[1],sys.argv[2]
|
||||
emb={}
|
||||
for line in open(TSV,encoding='utf-8',errors='surrogateescape'):
|
||||
p=line.rstrip("\n").rsplit("\t",1)
|
||||
if len(p)==2 and p[1].count(",")>100: emb[p[0]]=p[1]
|
||||
print("vectors",len(emb),flush=True)
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
hit=0
|
||||
for n in d["nodes"]:
|
||||
v=emb.get(n.get("id") or "")
|
||||
if v: n["emb"]=v; hit+=1
|
||||
print("attached",hit,"of",len(d["nodes"]),flush=True)
|
||||
with open(OUT,"w",encoding='utf-8',errors='surrogateescape') as f:
|
||||
json.dump(d,f,ensure_ascii=False)
|
||||
print("wrote",OUT,flush=True)
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "assoc-leg-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc",
|
||||
"soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 51.8,
|
||||
"child_pid": 87150,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1223.5,
|
||||
"latency_ms_p95": 1676.1,
|
||||
"latency_ms_max": 1740.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 331.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 292.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 309.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 331.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 295.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 613.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 538.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 553.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.4444444444444444,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 527.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 546.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 892.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 754.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1644.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1740.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1118.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"ctx-cc7f",
|
||||
"?of?7???",
|
||||
"ctx-4a41",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"ctx-175f"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1440.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1316.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1620.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1676.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1534.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1567.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1280.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1401.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1512.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1165.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1230.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"ctx-bb74",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1223.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1235.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1460.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 750.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 531.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 769.1,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1327.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1687.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1040.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "assoc-leg",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc",
|
||||
"soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 52.4,
|
||||
"child_pid": 87099,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 360.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 324.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 313.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 331.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 303.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 533.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.4444444444444444,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 517.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 547.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 903.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 774.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1641.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1718.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1409.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1135.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"ctx-cc7f",
|
||||
"?of?7???",
|
||||
"ctx-4a41",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"ctx-175f"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1446.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1296.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1621.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1411.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1681.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1536.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1567.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1278.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1401.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1509.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1156.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1232.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"ctx-bb74",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1228.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1251.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1459.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 746.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 519.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 764.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1346.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1687.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1030.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,945 +0,0 @@
|
||||
{
|
||||
"label": "baseline-embcorpus",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-baseline",
|
||||
"soul_md5": "5cc9521734907cf2da30f0af94498c06",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 48.4,
|
||||
"child_pid": 85995,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1145.9,
|
||||
"latency_ms_p95": 1574.3,
|
||||
"latency_ms_max": 1634.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 262.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 230.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 250.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 227.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 532.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 457.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 472.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 446.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 462.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 832.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 686.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1550.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1634.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1322.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1041.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1350.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1213.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1553.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1319.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1574.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"????7???Ջ3",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"%???2??jH??",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1441.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1499.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1197.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1311.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1422.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1077.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1149.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1145.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1177.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1382.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 660.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 432.5,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 676.9,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1252.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1624.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 946.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "hybrid-semantic-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-hybrid",
|
||||
"soul_md5": "5cd2718932c2ecf4940ddfe5a9c8abbe",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 52.1,
|
||||
"child_pid": 86164,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1225.4,
|
||||
"latency_ms_p95": 1671.6,
|
||||
"latency_ms_max": 1718.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 307.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 349.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 316.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 315.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 326.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 304.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"????7???Ջ3",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 593.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 544.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 524.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 905.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 769.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1645.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1718.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1123.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1448.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"??o?'?B???k",
|
||||
"kn-150e6790-fc2a-48a7-8289-313c1fbaf5ae",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1298.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1621.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1405.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"??????X??2c",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1671.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1519.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1576.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"??S?7???",
|
||||
"??f?7???",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1287.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1391.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1498.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1169.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1239.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98",
|
||||
"7?e?7???\f3?",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2121fdb9-796a-427e-b9b5-651f4388ea16"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1225.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"rQ??m?;?x?'"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1237.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1458.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 756.3,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 519.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?m?\\}Q??6??",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?m?\\}Q??6??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 774.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1340.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1693.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1027.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "hybrid-semantic",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-hybrid",
|
||||
"soul_md5": "5cd2718932c2ecf4940ddfe5a9c8abbe",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 52.1,
|
||||
"child_pid": 86109,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 323.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 330.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 318.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 316.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 328.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 296.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"????7???Ջ3",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 611.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 547.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 895.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 763.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1650.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1720.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1396.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1126.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1452.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"??o?'?B???k",
|
||||
"kn-150e6790-fc2a-48a7-8289-313c1fbaf5ae",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1303.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1618.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1406.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1667.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1528.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1579.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"??S?7???",
|
||||
"??f?7???",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1288.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1383.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1499.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98",
|
||||
"7?e?7???\f3?",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2121fdb9-796a-427e-b9b5-651f4388ea16"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1219.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"rQ??m?;?x?'"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1235.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1464.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 761.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 525.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?m?\\}Q??6??",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?m?\\}Q??6??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 761.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1322.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1694.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1041.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+10
-296
@@ -6099,12 +6099,6 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) {
|
||||
* propagation loop in engram_activate. 0.25 damps semantically unrelated
|
||||
* branches ~4x without severing them. Unembedded targets are ungated. */
|
||||
#define ENGRAM_QGATE_FLOOR 0.25
|
||||
/* The read-path semantic leg (engram claim 24) reuses ENGRAM_EMBED_SEED_MIN
|
||||
* above as its admission floor: a node joins the embedding ranking only if its
|
||||
* query cosine clears the same bar that lets it join the seed set. No new
|
||||
* tuning constant is introduced, and the floor is load-bearing rather than
|
||||
* cosmetic — it is what keeps a query with no real match (the gold set's
|
||||
* nonsense controls) from being answered with its nearest neighbours. */
|
||||
#define ENGRAM_EMBED_MAX_CHARS 2000
|
||||
#define ENGRAM_EMBED_TIMEOUT_MS 4000L
|
||||
#define ENGRAM_EMBED_BREAKER_LIMIT 3
|
||||
@@ -7387,25 +7381,8 @@ static int engram_node_match_score(const EngramNode* n,
|
||||
return score;
|
||||
}
|
||||
|
||||
/* Semantic leg of the read path (engram claim 24). Returns the query/target
|
||||
* cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0],
|
||||
* and exactly 0.0 when the pair is not comparable (no query embedding, target
|
||||
* unembedded, dim mismatch) or falls at/below the seed floor. Claim 32's
|
||||
* clamp-at-zero is subsumed: nothing below the floor can contribute.
|
||||
* A 0.0 return makes the fused score collapse to the lexical score, which is
|
||||
* why a dead embedder degrades to the historical behaviour exactly. */
|
||||
static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
|
||||
if (!qv || qdim <= 0 || !n->emb || n->emb_dim != qdim) return 0.0;
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
if (c <= ENGRAM_EMBED_SEED_MIN) return 0.0;
|
||||
double t = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN);
|
||||
return t > 1.0 ? 1.0 : t;
|
||||
}
|
||||
|
||||
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||
* (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
|
||||
* the semantic leg is a second ranking merged beside it, never a reweighting
|
||||
* of this one. */
|
||||
* (tiebreak, desc). */
|
||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||
static int engram_rank_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
@@ -7416,223 +7393,6 @@ static int engram_rank_cmp(const void* a, const void* b) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Semantic rank entry: node index and its renormalized query similarity,
|
||||
* ordered by similarity desc. This is the claim-24 "embedding search"
|
||||
* ranking, computed independently of the lexical one. */
|
||||
typedef struct { int64_t idx; double sem; } EngramSemEntry;
|
||||
static int engram_sem_cmp(const void* a, const void* b) {
|
||||
const EngramSemEntry* ea = (const EngramSemEntry*)a;
|
||||
const EngramSemEntry* eb = (const EngramSemEntry*)b;
|
||||
if (ea->sem < eb->sem) return 1; /* desc */
|
||||
if (ea->sem > eb->sem) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Merge the two rankings by strict alternation, lexical first:
|
||||
* L1, S1, L2, S2, L3, ... deduplicated by node index, capped at lim.
|
||||
*
|
||||
* Rank fusion, not score fusion. nomic's cosine scale is compressed (real
|
||||
* matches land ~0.55-0.70 while unrelated pairs sit ~0.35-0.50), so any
|
||||
* additive blend of a cosine onto a token-coverage score is dominated by
|
||||
* whichever leg happens to have the wider spread. Alternation is invariant to
|
||||
* both scales: it asks each leg for its next best answer in turn.
|
||||
*
|
||||
* Position 1 is always the top lexical hit, so a query whose answer the
|
||||
* lexical leg already ranks first cannot be displaced — exact-token retrieval
|
||||
* is structurally safe. The cost is bounded and explicit: a lexical hit at
|
||||
* rank r lands at output position 2r-1. */
|
||||
static int64_t engram_interleave(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0;
|
||||
while (no < lim && (li < nL || si < nS)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = L[li].idx;
|
||||
li++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (si < nS) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = S[si].idx;
|
||||
si++;
|
||||
}
|
||||
}
|
||||
return no;
|
||||
}
|
||||
|
||||
/* ── Associative leg (claim 10 typed relations + claim 1 activation) ────────
|
||||
*
|
||||
* WHY THIS EXISTS. engram_search_json has two legs, and neither can reach a
|
||||
* node that shares no token with the query and no embedding neighbourhood
|
||||
* with it. The route the design reserves for that case is the graph: a node
|
||||
* is reachable because it is STRUCTURALLY associated with something the query
|
||||
* did hit. Nothing on the recall path consults an edge today.
|
||||
*
|
||||
* WHY NOT engram_activate. Wiring recall wholesale to activation was measured
|
||||
* (PR #135) and lost 57 points of phrase accuracy. The failure was one of
|
||||
* RANK, not reach: activation seeds on every token-matching node, so a 2-hop
|
||||
* associate at strength ~0.06 never outranks thousands of 1-hop neighbours of
|
||||
* strong lexical seeds. So this is a separate, small, ranked list that is
|
||||
* merged BESIDE the other two, exactly as the semantic leg is.
|
||||
*
|
||||
* TYPED RELATIONS (06-claims.md claim 10). Expansion follows only relations
|
||||
* that assert a structural claim about meaning. The corpus is 4,915 `tagged`
|
||||
* and 3,767 `triggers-safety` edges against 475 `identity` and 11 `contains`;
|
||||
* walking the bulk relations turns any seed into a flood (measured: 1,387
|
||||
* candidates from one seed) while the structural ones stay in the low tens.
|
||||
* This is the first code on the read path to branch on a relation type at all.
|
||||
*
|
||||
* DIRECTION. Edges are walked in BOTH directions. Claim 23 requires the store
|
||||
* to keep reverse edge records keyed by target id, and the adjacency index
|
||||
* already materialises them (adj_to). It has to be both: every value node in
|
||||
* this corpus has exactly ONE inbound edge (hub -> value) and no outbound
|
||||
* structural edge at all, so a forward-only walk from a value node reaches
|
||||
* nothing. Note this is an extension of the traversal as literally specified
|
||||
* (05-detailed-description l.66 says "all outgoing edges"); the reverse index
|
||||
* is designed and stored, but the description does not say the walk reads it.
|
||||
*/
|
||||
#define ENGRAM_ASSOC_SEEDS 3 /* top-N lexical hits form the context */
|
||||
#define ENGRAM_ASSOC_DEPTH 2 /* seed -> hub -> sibling */
|
||||
#define ENGRAM_ASSOC_FIRE 0.02 /* same firing threshold as engram_activate */
|
||||
#define ENGRAM_ASSOC_MAX 64 /* cap on candidates carried forward */
|
||||
|
||||
static int eg_rel_is_structural(const char* r) {
|
||||
if (!r || !*r) return 0;
|
||||
static const char* ok[] = {
|
||||
"identity", "contains", "superseded_by", "references", "embodies",
|
||||
"demonstrated_by", "canonical-self", "depends_on", "currently_holds",
|
||||
"activates", NULL
|
||||
};
|
||||
for (int i = 0; ok[i]; i++) if (strcmp(r, ok[i]) == 0) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Nodes that are index artefacts rather than recallable content. Same
|
||||
* exclusion eg_embed_eligible() already applies when deciding what deserves an
|
||||
* embedding, reused here so the associative leg cannot surface — or relay
|
||||
* through — a Tag. Relaying through them is what makes a graph walk explode:
|
||||
* tag-tier_note alone has 187 members. */
|
||||
static int eg_assoc_excluded(const EngramNode* n) {
|
||||
if (!n->node_type) return 0;
|
||||
return strcmp(n->node_type, "Tag") == 0
|
||||
|| strcmp(n->node_type, "InternalStateEvent") == 0;
|
||||
}
|
||||
|
||||
/* Breadth-first structural expansion from `seeds`, then ORDER BY query
|
||||
* similarity. Activation decides REACHABILITY (the conjunctive prune of
|
||||
* claim 1: parent strength x edge weight x target salience, cut at the firing
|
||||
* threshold); cosine decides ORDER within what was reached. Ranking the
|
||||
* neighbourhood by activation alone does not work and the reason is
|
||||
* structural: every identity edge in this corpus carries weight 0.5 and every
|
||||
* value node salience 0.7, so the activation product degenerates into a
|
||||
* function of hop count and ranks the relay hub above all of its own
|
||||
* children. Composing the two is mine, not Will's — the description ranks the
|
||||
* activation result set by strength (l.78). */
|
||||
static int64_t engram_assoc_leg(EngramStore* g,
|
||||
const EngramRankEntry* L, int64_t nL,
|
||||
const float* qv, int32_t qdim,
|
||||
EngramSemEntry* out, int64_t out_cap) {
|
||||
if (!g || nL <= 0 || !qv || qdim <= 0 || out_cap <= 0) return 0;
|
||||
if (g->adj_dirty || !g->adj_from || !g->adj_to) engram_adj_rebuild(g);
|
||||
if (!g->adj_from || !g->adj_to) return 0;
|
||||
|
||||
double* act = calloc((size_t)g->node_count, sizeof(double));
|
||||
char* seen = calloc((size_t)g->node_count, sizeof(char));
|
||||
int64_t* q = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t));
|
||||
int64_t* hop = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t));
|
||||
if (!act || !seen || !q || !hop) { free(act); free(seen); free(q); free(hop); return 0; }
|
||||
int64_t qcap = ENGRAM_ASSOC_MAX * 4, qh = 0, qt = 0;
|
||||
|
||||
int64_t nseed = nL < ENGRAM_ASSOC_SEEDS ? nL : ENGRAM_ASSOC_SEEDS;
|
||||
for (int64_t s = 0; s < nseed; s++) {
|
||||
int64_t idx = L[s].idx;
|
||||
if (idx < 0 || idx >= g->node_count) continue;
|
||||
act[idx] = 1.0; seen[idx] = 2; /* 2 = seed: never a result */
|
||||
if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; }
|
||||
}
|
||||
|
||||
const double SPREAD_DECAY = 0.7;
|
||||
while (qh < qt) {
|
||||
int64_t cur = q[qh]; int64_t h = hop[qh]; qh++;
|
||||
if (h >= ENGRAM_ASSOC_DEPTH) continue;
|
||||
double parent = act[cur];
|
||||
int from_len = g->adj_from_len[cur];
|
||||
int to_len = g->adj_to_len[cur];
|
||||
for (int scan = 0; scan < from_len + to_len; scan++) {
|
||||
int64_t ei = (scan < from_len) ? g->adj_from[cur][scan]
|
||||
: g->adj_to[cur][scan - from_len];
|
||||
EngramEdge* e = &g->edges[ei];
|
||||
if (!eg_rel_is_structural(e->relation)) continue;
|
||||
int64_t oi = (scan < from_len) ? engram_idmap_get(g, e->to_id)
|
||||
: engram_idmap_get(g, e->from_id);
|
||||
if (oi < 0 || oi >= g->node_count) continue;
|
||||
EngramNode* on = &g->nodes[oi];
|
||||
if (eg_assoc_excluded(on)) continue;
|
||||
double na = parent * e->weight * SPREAD_DECAY * on->salience;
|
||||
if (na < ENGRAM_ASSOC_FIRE) continue;
|
||||
if (seen[oi] && na <= act[oi]) continue;
|
||||
act[oi] = na;
|
||||
if (!seen[oi]) seen[oi] = 1;
|
||||
if (qt < qcap) { q[qt] = oi; hop[qt] = h + 1; qt++; }
|
||||
}
|
||||
}
|
||||
|
||||
int64_t n = 0;
|
||||
for (int64_t i = 0; i < g->node_count && n < out_cap; i++) {
|
||||
if (seen[i] != 1) continue; /* skip unreached and seeds */
|
||||
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
||||
EngramNode* nd = &g->nodes[i];
|
||||
if (!nd->emb || nd->emb_dim != qdim) continue;
|
||||
double c = eg_cosine(nd->emb, qv, qdim);
|
||||
if (c <= 0.0) continue;
|
||||
out[n].idx = i; out[n].sem = c; n++;
|
||||
}
|
||||
qsort(out, (size_t)n, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
free(act); free(seen); free(q); free(hop);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Three-leg merge: lexical, semantic, associative — strict rotation,
|
||||
* L1, S1, A1, L2, S2, A2, ... deduplicated, capped at lim.
|
||||
*
|
||||
* The cost is explicit and worse than the two-leg case: a lexical hit at rank
|
||||
* r lands at output position 3r-2 when both other legs are non-empty. That is
|
||||
* survivable here only because the structural-relation filter leaves the
|
||||
* associative list EMPTY for most queries — a Memory node whose only edges are
|
||||
* `tagged` and `related` expands to nothing, so its ranking is untouched. */
|
||||
static int64_t engram_interleave3(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
const EngramSemEntry* A, int64_t nA,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0, ai = 0;
|
||||
while (no < lim && (li < nL || si < nS || ai < nA)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = L[li].idx;
|
||||
li++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (si < nS) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = S[si].idx;
|
||||
si++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (ai < nA) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == A[ai].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = A[ai].idx;
|
||||
ai++;
|
||||
}
|
||||
}
|
||||
return no;
|
||||
}
|
||||
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -7645,12 +7405,6 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
if (ntok == 0) return lst;
|
||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||
if (!hits) return lst;
|
||||
/* Claim-24 semantic leg: one query embedding, fetched once per search.
|
||||
* NULL (embedder down / circuit breaker open) => pure lexical, as before. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
@@ -7666,24 +7420,14 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
}
|
||||
}
|
||||
/* Rank each leg independently, then alternate between them. */
|
||||
/* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order);
|
||||
for (int64_t k = 0; k < no; k++)
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[order[k]]));
|
||||
free(order);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||
}
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
return lst;
|
||||
}
|
||||
|
||||
@@ -9515,14 +9259,6 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
if (ntok > 0) {
|
||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||
if (hits) {
|
||||
/* Claim-24 semantic leg. This is the function /api/neuron/recall
|
||||
* actually reaches (routes.el -> neuron-api.el handle_api_recall),
|
||||
* so the semantic half of the retrieval surface has to land HERE
|
||||
* to be observable to the MCP wrapper and the app. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
@@ -9535,37 +9271,15 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
}
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
/* Claim-10 associative leg: expand the top lexical hits along
|
||||
* structural relations only, order the reached set by query
|
||||
* similarity. Empty whenever the seeds have no structural
|
||||
* edges, which is the common case and is what keeps the
|
||||
* lexical ordering safe. */
|
||||
EngramSemEntry* assoc = qv ? malloc((size_t)ENGRAM_ASSOC_MAX * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nassoc = assoc
|
||||
? engram_assoc_leg(g, hits, nhits, qv, qdim, assoc, ENGRAM_ASSOC_MAX)
|
||||
: 0;
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave3(hits, nhits, sem, nsem,
|
||||
assoc, nassoc, lim, order);
|
||||
for (int64_t k = 0; k < no; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[order[k]], 0);
|
||||
first = 0;
|
||||
}
|
||||
free(order);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0);
|
||||
first = 0;
|
||||
}
|
||||
free(assoc);
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user