Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Neuron
|
cf41d12d22 | ||
 tim.lingo
|
18714e6142 | ||
|
tim.lingo
|
4936099c39 | ||
|
tim.lingo
|
f1471763f5 | ||
|
tim.lingo
|
5850793b67 | ||
|
tim.lingo
|
fc1745c652 |
@@ -1,937 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""state-key-audit.py — the analyzer behind scripts/verify-state-keys.sh.
|
||||
|
||||
Read that script's header for WHY this exists (issue #129). This file is the
|
||||
HOW: a small El reader that resolves the key expression at every state_get /
|
||||
state_set site, including keys that are computed.
|
||||
|
||||
WHAT IT PARSES
|
||||
El as this engine writes it: `fn f(a: T, b: T) -> T { ... }`, `let x: T = e`,
|
||||
`return e`, `if c { a } else { b }` as an expression, `+` concatenation,
|
||||
`"..."` with backslash escapes, `//` line comments. No block comments, no
|
||||
const/match/struct exist in this dialect (verified over the whole tree).
|
||||
|
||||
KEY PATTERNS — the only two things a key expression can resolve to
|
||||
EXACT "soul_model" the whole key is known
|
||||
PREFIX "session_hist_" a known head, then runtime text
|
||||
(plus UNRESOLVED, which is a report line and never a failure)
|
||||
|
||||
RESOLUTION — resolve_expr() returns a SET of patterns; unions are how branches,
|
||||
multiple returns, and multiple bindings of one name are represented.
|
||||
literal "k" -> {EXACT k}
|
||||
concat A + B -> fold left; all-static -> EXACT,
|
||||
static head + dynamic tail -> PREFIX
|
||||
if-expression if c {A} else {B} -> resolve(A) | resolve(B), except that
|
||||
str_eq(X,"") with X statically ""
|
||||
folds to the taken branch only
|
||||
call f(args) -> union over f's return expressions,
|
||||
with f's params bound to THIS call
|
||||
site's actual argument expressions
|
||||
local var let k = e; state_get(k)-> union over every `let k =` in the
|
||||
enclosing function
|
||||
parameter fn g(k) { state_get(k) }-> union over the argument at that
|
||||
position across every call site of g
|
||||
anything else json_get(...), env(...)-> UNRESOLVED
|
||||
Recursion is depth- and cycle-guarded; a guard trip yields UNRESOLVED, never a
|
||||
failure.
|
||||
|
||||
COVERAGE — a read is satisfied when some write can produce the same key:
|
||||
read EXACT k <- write EXACT k, or write PREFIX p where k starts with p
|
||||
read PREFIX p <- write EXACT k where k starts with p, or write PREFIX q
|
||||
where p and q are prefixes of each other
|
||||
Deliberately permissive at the boundaries: a gate that cries wolf gets deleted.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
MAX_DEPTH = 12
|
||||
|
||||
# ── patterns ────────────────────────────────────────────────────────────────
|
||||
EXACT = "exact"
|
||||
PREFIX = "prefix"
|
||||
|
||||
|
||||
def pat_exact(s):
|
||||
return (EXACT, s)
|
||||
|
||||
|
||||
def pat_prefix(s):
|
||||
# A prefix with no static text at all carries no information; that is the
|
||||
# UNRESOLVED case, not a pattern.
|
||||
return (PREFIX, s) if s else None
|
||||
|
||||
|
||||
def covers(write, read):
|
||||
"""Can a write of pattern `write` produce a key that `read` reads?
|
||||
|
||||
The prefix rule is DIRECTIONAL, and that direction is the whole point. A
|
||||
write namespace that is the same or BROADER than the read namespace covers
|
||||
it (write "rl:" covers read "rl:x"). A write namespace that is NARROWER does
|
||||
NOT (write "session_histv2_" does not cover read "session_hist_") — being
|
||||
permissive there re-opens the exact hole this gate exists to close: rename
|
||||
the producer, leave the readers, stay green. Verified with a control run
|
||||
that renames sessions.el's writer and leaves its four readers behind."""
|
||||
wk, wv = write
|
||||
rk, rv = read
|
||||
if rk == EXACT:
|
||||
return rv == wv if wk == EXACT else rv.startswith(wv)
|
||||
# read is a PREFIX: some key starting with rv is read
|
||||
if wk == EXACT:
|
||||
return wv.startswith(rv) # that one written key is in range
|
||||
return rv.startswith(wv) # write namespace same-or-broader
|
||||
|
||||
|
||||
# ── lexer ───────────────────────────────────────────────────────────────────
|
||||
TOK_STR, TOK_IDENT, TOK_PUNCT, TOK_NUM = "str", "ident", "punct", "num"
|
||||
IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
NUM_RE = re.compile(r"[0-9]+(\.[0-9]+)?")
|
||||
|
||||
|
||||
class Tok:
|
||||
__slots__ = ("kind", "val", "line")
|
||||
|
||||
def __init__(self, kind, val, line):
|
||||
self.kind, self.val, self.line = kind, val, line
|
||||
|
||||
def __repr__(self):
|
||||
return "%s(%r)@%d" % (self.kind, self.val, self.line)
|
||||
|
||||
|
||||
def lex(src):
|
||||
toks, i, n, line = [], 0, len(src), 1
|
||||
while i < n:
|
||||
c = src[i]
|
||||
if c == "\n":
|
||||
line += 1
|
||||
i += 1
|
||||
continue
|
||||
if c in " \t\r":
|
||||
i += 1
|
||||
continue
|
||||
if c == "/" and i + 1 < n and src[i + 1] == "/":
|
||||
while i < n and src[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
if c == '"':
|
||||
j, buf = i + 1, []
|
||||
while j < n:
|
||||
if src[j] == "\\" and j + 1 < n:
|
||||
esc = src[j + 1]
|
||||
buf.append({"n": "\n", "t": "\t", "r": "\r"}.get(esc, esc))
|
||||
j += 2
|
||||
continue
|
||||
if src[j] == '"':
|
||||
break
|
||||
if src[j] == "\n":
|
||||
line += 1
|
||||
buf.append(src[j])
|
||||
j += 1
|
||||
toks.append(Tok(TOK_STR, "".join(buf), line))
|
||||
i = j + 1
|
||||
continue
|
||||
m = IDENT_RE.match(src, i)
|
||||
if m:
|
||||
toks.append(Tok(TOK_IDENT, m.group(0), line))
|
||||
i = m.end()
|
||||
continue
|
||||
m = NUM_RE.match(src, i)
|
||||
if m:
|
||||
toks.append(Tok(TOK_NUM, m.group(0), line))
|
||||
i = m.end()
|
||||
continue
|
||||
toks.append(Tok(TOK_PUNCT, c, line))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
def match_close(toks, i, open_ch, close_ch):
|
||||
"""toks[i] is open_ch; return index of its matching close_ch."""
|
||||
depth = 0
|
||||
while i < len(toks):
|
||||
if toks[i].kind == TOK_PUNCT:
|
||||
if toks[i].val == open_ch:
|
||||
depth += 1
|
||||
elif toks[i].val == close_ch:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i
|
||||
i += 1
|
||||
return len(toks) - 1
|
||||
|
||||
|
||||
# ── program model ───────────────────────────────────────────────────────────
|
||||
class Func:
|
||||
def __init__(self, name, path, line, params, toks, start, end):
|
||||
self.name, self.path, self.line = name, path, line
|
||||
self.params = params # [param name]
|
||||
self.toks = toks # the whole file's token list
|
||||
self.start, self.end = start, end # body token range, exclusive of braces
|
||||
self.lets = None # name -> [expr token ranges], lazily built
|
||||
|
||||
|
||||
class Site:
|
||||
def __init__(self, kind, path, line, func, arg_range, text):
|
||||
self.kind = kind # "get" | "set"
|
||||
self.path, self.line = path, line
|
||||
self.func = func
|
||||
self.arg_range = arg_range
|
||||
self.text = text # source text of the key expression
|
||||
self.pats = set()
|
||||
self.unresolved = False
|
||||
self.literal = None # set when the key expression is a bare literal
|
||||
|
||||
|
||||
class Program:
|
||||
def __init__(self):
|
||||
self.files = {} # path -> toks
|
||||
self.funcs = {} # name -> [Func] (El allows no overloads, but be safe)
|
||||
self.toplevel = [] # [Func] one per file, params=[]
|
||||
self.sites = [] # [Site]
|
||||
self.calls = {} # callee name -> [(Func caller, [arg ranges])]
|
||||
|
||||
# -- loading ------------------------------------------------------------
|
||||
def load(self, path, rel):
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
src = fh.read()
|
||||
toks = lex(src)
|
||||
self.files[rel] = toks
|
||||
self._scan_funcs(rel, toks)
|
||||
|
||||
def _scan_funcs(self, rel, toks):
|
||||
covered = []
|
||||
i = 0
|
||||
while i < len(toks):
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "fn" and i + 2 < len(toks) \
|
||||
and toks[i + 1].kind == TOK_IDENT and toks[i + 2].val == "(":
|
||||
name = toks[i + 1].val
|
||||
pclose = match_close(toks, i + 2, "(", ")")
|
||||
params = self._params(toks, i + 3, pclose)
|
||||
bopen = pclose + 1
|
||||
while bopen < len(toks) and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
f = Func(name, rel, t.line, params, toks, bopen + 1, bclose)
|
||||
self.funcs.setdefault(name, []).append(f)
|
||||
covered.append((i, bclose))
|
||||
i = bclose + 1
|
||||
continue
|
||||
i += 1
|
||||
# everything outside a fn is the file's top-level "function"
|
||||
tl = Func("<toplevel:%s>" % rel, rel, 1, [], toks, 0, len(toks))
|
||||
tl.covered = covered
|
||||
self.toplevel.append(tl)
|
||||
|
||||
@staticmethod
|
||||
def _params(toks, i, end):
|
||||
"""`a: T, b: T` -> ['a','b'] (top-level commas only)."""
|
||||
names, depth, expect = [], 0, True
|
||||
while i < end:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
depth += 1
|
||||
elif t.kind == TOK_PUNCT and t.val in ")]}":
|
||||
depth -= 1
|
||||
elif depth == 0 and t.kind == TOK_PUNCT and t.val == ",":
|
||||
expect = True
|
||||
elif depth == 0 and expect and t.kind == TOK_IDENT:
|
||||
names.append(t.val)
|
||||
expect = False
|
||||
i += 1
|
||||
return names
|
||||
|
||||
def func_at(self, rel, tok_index):
|
||||
for f in self.funcs_in(rel):
|
||||
if f.start <= tok_index < f.end:
|
||||
return f
|
||||
for f in self.toplevel:
|
||||
if f.path == rel:
|
||||
return f
|
||||
return None
|
||||
|
||||
def funcs_in(self, rel):
|
||||
for fl in self.funcs.values():
|
||||
for f in fl:
|
||||
if f.path == rel:
|
||||
yield f
|
||||
|
||||
# -- indexing -----------------------------------------------------------
|
||||
def index(self):
|
||||
for rel, toks in self.files.items():
|
||||
i = 0
|
||||
while i < len(toks):
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and i + 1 < len(toks) and toks[i + 1].val == "(" \
|
||||
and t.val not in KEYWORDS \
|
||||
and not (i > 0 and toks[i - 1].kind == TOK_IDENT
|
||||
and toks[i - 1].val == "fn"):
|
||||
# ^ the `fn f(a: T)` declaration is not a call site; counting
|
||||
# it as one makes every parameter resolve to its own name
|
||||
# and reports the whole function UNRESOLVED.
|
||||
close = match_close(toks, i + 1, "(", ")")
|
||||
args = split_args(toks, i + 2, close)
|
||||
self.calls.setdefault(t.val, []).append(
|
||||
(self.func_at(rel, i), args, rel, t.line))
|
||||
if t.val in ("state_get", "state_set") and args:
|
||||
self.sites.append(Site(
|
||||
"get" if t.val == "state_get" else "set",
|
||||
rel, t.line, self.func_at(rel, i), args[0],
|
||||
render(toks, *args[0])))
|
||||
i += 1
|
||||
|
||||
# -- resolution ---------------------------------------------------------
|
||||
def lets_of(self, f):
|
||||
if f.lets is not None:
|
||||
return f.lets
|
||||
f.lets = {}
|
||||
toks = f.toks
|
||||
skip = getattr(f, "covered", [])
|
||||
i = f.start
|
||||
while i < f.end:
|
||||
if any(a <= i <= b for a, b in skip):
|
||||
i = max(b for a, b in skip if a <= i <= b) + 1
|
||||
continue
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "let" and i + 1 < f.end \
|
||||
and toks[i + 1].kind == TOK_IDENT:
|
||||
name = toks[i + 1].val
|
||||
j = i + 2
|
||||
if j < f.end and toks[j].val == ":": # skip the type
|
||||
while j < f.end and toks[j].val != "=":
|
||||
j += 1
|
||||
if j < f.end and toks[j].val == "=":
|
||||
s = j + 1
|
||||
e = stmt_end(toks, s, f.end)
|
||||
f.lets.setdefault(name, []).append((s, e))
|
||||
i = e
|
||||
continue
|
||||
i += 1
|
||||
return f.lets
|
||||
|
||||
def returns_of(self, ctx, depth=0, seen=None):
|
||||
"""The value expressions of a function, in the context it was CALLED in.
|
||||
|
||||
Context-sensitive on purpose. `conv_hist_key` is written as a guard:
|
||||
|
||||
if str_eq(session_id, "") { return "conv_history" }
|
||||
return "session_hist_" + session_id
|
||||
|
||||
Collecting both returns flat would make state_set(conv_hist_key("")) — the
|
||||
dead handle_chat() write — claim to produce the session_hist_ namespace
|
||||
too. That is a producer this engine does not actually have, and claiming
|
||||
it would let the gate stay green if sessions.el's real writer vanished:
|
||||
a masking hole in the exact namespace #129 lives in. So a guard whose
|
||||
condition folds is honoured, and the branch not taken is dropped."""
|
||||
out = []
|
||||
self._values(ctx.toks, ctx.start, ctx.end, ctx, depth,
|
||||
seen if seen is not None else set(), out)
|
||||
return out
|
||||
|
||||
def _values(self, toks, s, e, ctx, depth, seen, out):
|
||||
"""Append the value expressions of a statement sequence.
|
||||
Returns True when the sequence definitely returns (rest unreachable)."""
|
||||
if depth > MAX_DEPTH:
|
||||
return False
|
||||
i = s
|
||||
while i < e:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "return":
|
||||
j = stmt_end(toks, i + 1, e)
|
||||
if j > i + 1:
|
||||
out.append((i + 1, j))
|
||||
return True
|
||||
if t.kind == TOK_IDENT and t.val == "let":
|
||||
i = stmt_end(toks, i + 2, e)
|
||||
continue
|
||||
if t.kind == TOK_IDENT and t.val == "if":
|
||||
i = self._if_stmt(toks, i, e, ctx, depth, seen, out)
|
||||
if i is True:
|
||||
return True
|
||||
continue
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
i = match_close(toks, i, t.val,
|
||||
{"(": ")", "[": "]", "{": "}"}[t.val]) + 1
|
||||
continue
|
||||
en = stmt_end(toks, i, e)
|
||||
if en <= i:
|
||||
i += 1
|
||||
continue
|
||||
if en >= e: # trailing expression = the value
|
||||
out.append((i, en))
|
||||
i = en
|
||||
return False
|
||||
|
||||
def _if_stmt(self, toks, i, e, ctx, depth, seen, out):
|
||||
"""Walk one if / else-if / else chain. Returns the next index, or True
|
||||
if the chain definitely returns on every reachable branch."""
|
||||
bopen = i + 1
|
||||
while bopen < e and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
if bopen >= e:
|
||||
return e
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
fold = self._fold_cond(toks, i + 1, bopen, ctx, depth, seen)
|
||||
|
||||
j = bclose + 1
|
||||
else_s = else_e = None
|
||||
if j < e and toks[j].kind == TOK_IDENT and toks[j].val == "else":
|
||||
if j + 1 < e and toks[j + 1].val == "{":
|
||||
ec = match_close(toks, j + 1, "{", "}")
|
||||
else_s, else_e = j + 2, ec
|
||||
j = ec + 1
|
||||
else: # `else if ...` — the rest of the chain
|
||||
else_s = j + 1
|
||||
else_e = stmt_end(toks, j + 1, e)
|
||||
j = else_e
|
||||
|
||||
then_ret = else_ret = False
|
||||
if fold is not False:
|
||||
then_ret = self._values(toks, bopen + 1, bclose, ctx, depth + 1, seen, out)
|
||||
if fold is not True and else_s is not None:
|
||||
else_ret = self._values(toks, else_s, else_e, ctx, depth + 1, seen, out)
|
||||
|
||||
if fold is True and then_ret:
|
||||
return True
|
||||
if fold is False and else_s is not None and else_ret:
|
||||
return True
|
||||
if fold is None and else_s is not None and then_ret and else_ret:
|
||||
return True
|
||||
return j
|
||||
|
||||
def resolve(self, rng, func, depth=0, seen=None):
|
||||
"""-> (set of patterns, unresolved_flag)"""
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if depth > MAX_DEPTH:
|
||||
return set(), True
|
||||
return self._expr(func.toks, rng[0], rng[1], func, depth, seen)
|
||||
|
||||
# -- expression walker --------------------------------------------------
|
||||
def _expr(self, toks, s, e, func, depth, seen):
|
||||
parts, cur, d = [], s, 0
|
||||
i = s
|
||||
while i < e: # split on top-level '+'
|
||||
v = toks[i].val
|
||||
if toks[i].kind == TOK_PUNCT and v in "([{":
|
||||
d += 1
|
||||
elif toks[i].kind == TOK_PUNCT and v in ")]}":
|
||||
d -= 1
|
||||
elif d == 0 and toks[i].kind == TOK_PUNCT and v == "+" and i > s:
|
||||
parts.append((cur, i))
|
||||
cur = i + 1
|
||||
i += 1
|
||||
parts.append((cur, e))
|
||||
if len(parts) == 1:
|
||||
return self._primary(toks, s, e, func, depth, seen)
|
||||
|
||||
# concatenation: keep folding while every operand so far is EXACT
|
||||
head, unres = "", False
|
||||
static = True
|
||||
for (ps, pe) in parts:
|
||||
pats, u = self._primary(toks, ps, pe, func, depth, seen)
|
||||
exacts = {p[1] for p in pats if p[0] == EXACT}
|
||||
if static and len(exacts) == 1 and not u and len(pats) == 1:
|
||||
head += exacts.pop()
|
||||
continue
|
||||
if static and pats and all(p[0] == EXACT for p in pats) and len(pats) > 1:
|
||||
# a branchy static operand: keep the shared head only
|
||||
static = False
|
||||
head += os.path.commonprefix(sorted({p[1] for p in pats}))
|
||||
break
|
||||
static = False
|
||||
# first non-static operand: everything after it is runtime text
|
||||
if (ps, pe) == parts[0]:
|
||||
for p in pats:
|
||||
if p[0] == PREFIX:
|
||||
head = p[1]
|
||||
break
|
||||
if not head:
|
||||
unres = True
|
||||
break
|
||||
if static:
|
||||
return {pat_exact(head)}, False
|
||||
p = pat_prefix(head)
|
||||
return ({p} if p else set()), (unres or not p)
|
||||
|
||||
def _primary(self, toks, s, e, func, depth, seen):
|
||||
while s < e and toks[s].kind == TOK_PUNCT and toks[s].val == "(" \
|
||||
and match_close(toks, s, "(", ")") == e - 1:
|
||||
s, e = s + 1, e - 1
|
||||
if s >= e:
|
||||
return set(), True
|
||||
t = toks[s]
|
||||
|
||||
if t.kind == TOK_STR and e == s + 1:
|
||||
return {pat_exact(t.val)}, False
|
||||
|
||||
if t.kind == TOK_IDENT and t.val == "if":
|
||||
return self._if_expr(toks, s, e, func, depth, seen)
|
||||
|
||||
if t.kind == TOK_IDENT and s + 1 < e and toks[s + 1].val == "(":
|
||||
close = match_close(toks, s + 1, "(", ")")
|
||||
if close == e - 1:
|
||||
return self._call(toks, t.val, split_args(toks, s + 2, close),
|
||||
func, depth, seen)
|
||||
|
||||
if t.kind == TOK_IDENT and e == s + 1:
|
||||
return self._var(t.val, func, depth, seen)
|
||||
|
||||
return set(), True
|
||||
|
||||
def _if_expr(self, toks, s, e, func, depth, seen):
|
||||
bopen = s + 1
|
||||
while bopen < e and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
cond = (s + 1, bopen)
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
then_rng = block_tail(toks, bopen + 1, bclose) or (bopen + 1, bclose)
|
||||
|
||||
else_rng = None
|
||||
j = bclose + 1
|
||||
if j < e and toks[j].kind == TOK_IDENT and toks[j].val == "else":
|
||||
if j + 1 < e and toks[j + 1].val == "{":
|
||||
ec = match_close(toks, j + 1, "{", "}")
|
||||
else_rng = block_tail(toks, j + 2, ec) or (j + 2, ec)
|
||||
else:
|
||||
else_rng = (j + 1, e) # `else if ...`
|
||||
|
||||
taken = self._fold_cond(toks, cond[0], cond[1], func, depth, seen)
|
||||
rngs = []
|
||||
if taken is not False:
|
||||
rngs.append(then_rng)
|
||||
if taken is not True and else_rng:
|
||||
rngs.append(else_rng)
|
||||
|
||||
pats, unres = set(), False
|
||||
for r in rngs:
|
||||
p, u = self._expr(toks, r[0], r[1], func, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
def _fold_cond(self, toks, s, e, func, depth, seen):
|
||||
"""Constant-fold `str_eq(X, "")` / `!str_eq(X, "")` so a helper called with
|
||||
a literal (conv_hist_key("")) yields only the branch it really takes.
|
||||
Returns True / False / None(unknown)."""
|
||||
neg = False
|
||||
if s < e and toks[s].kind == TOK_PUNCT and toks[s].val == "!":
|
||||
neg, s = True, s + 1
|
||||
if not (s < e and toks[s].kind == TOK_IDENT and toks[s].val == "str_eq"
|
||||
and s + 1 < e and toks[s + 1].val == "("):
|
||||
return None
|
||||
close = match_close(toks, s + 1, "(", ")")
|
||||
if close != e - 1:
|
||||
return None
|
||||
args = split_args(toks, s + 2, close)
|
||||
if len(args) != 2:
|
||||
return None
|
||||
va, ua = self._expr(toks, args[0][0], args[0][1], func, depth + 1, seen)
|
||||
vb, ub = self._expr(toks, args[1][0], args[1][1], func, depth + 1, seen)
|
||||
if ua or ub or len(va) != 1 or len(vb) != 1:
|
||||
return None
|
||||
(ka, sa), (kb, sb) = va.pop(), vb.pop()
|
||||
if ka != EXACT or kb != EXACT:
|
||||
return None
|
||||
r = (sa == sb)
|
||||
return (not r) if neg else r
|
||||
|
||||
def _call(self, toks, name, args, func, depth, seen):
|
||||
cands = self.funcs.get(name)
|
||||
if not cands:
|
||||
return set(), True # builtin: json_get, env, ...
|
||||
pats, unres = set(), False
|
||||
for callee in cands:
|
||||
key = ("fn", callee.path, callee.name, tuple(args))
|
||||
if key in seen:
|
||||
unres = True
|
||||
continue
|
||||
seen = seen | {key}
|
||||
# bind the callee's params to THIS call site's argument expressions
|
||||
binding = {}
|
||||
for idx, pname in enumerate(callee.params):
|
||||
if idx < len(args):
|
||||
binding[pname] = (args[idx], func)
|
||||
callee_ctx = _Bound(callee, binding)
|
||||
for r in self.returns_of(callee_ctx, depth + 1, seen):
|
||||
p, u = self._expr(callee.toks, r[0], r[1], callee_ctx,
|
||||
depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
def _var(self, name, func, depth, seen):
|
||||
real = func.func if isinstance(func, _Bound) else func
|
||||
|
||||
# 1. a parameter bound by the call site we came through
|
||||
if isinstance(func, _Bound) and name in func.binding:
|
||||
rng, caller_ctx = func.binding[name]
|
||||
return self._expr(caller_ctx.toks, rng[0], rng[1], caller_ctx,
|
||||
depth + 1, seen)
|
||||
|
||||
# 2. a local `let` in the enclosing function
|
||||
lets = self.lets_of(real)
|
||||
if name in lets:
|
||||
key = ("let", real.path, real.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen = seen | {key}
|
||||
pats, unres = set(), False
|
||||
for rng in lets[name]:
|
||||
p, u = self._expr(real.toks, rng[0], rng[1], real, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
# 3. an unbound parameter -> look at every call site of the enclosing fn
|
||||
if name in real.params:
|
||||
key = ("param", real.path, real.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen = seen | {key}
|
||||
idx = real.params.index(name)
|
||||
pats, unres = set(), False
|
||||
sites = self.calls.get(real.name, [])
|
||||
if not sites:
|
||||
return set(), True
|
||||
for caller, args, _rel, _line in sites:
|
||||
if caller is None or idx >= len(args):
|
||||
unres = True
|
||||
continue
|
||||
p, u = self._expr(caller.toks, args[idx][0], args[idx][1],
|
||||
caller, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
# 4. a file-level / cross-file top-level `let`
|
||||
for tl in self.toplevel:
|
||||
lets = self.lets_of(tl)
|
||||
if name in lets:
|
||||
key = ("let", tl.path, tl.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen2 = seen | {key}
|
||||
pats, unres = set(), False
|
||||
for rng in lets[name]:
|
||||
p, u = self._expr(tl.toks, rng[0], rng[1], tl, depth + 1, seen2)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
return set(), True
|
||||
|
||||
|
||||
class _Bound:
|
||||
"""A callee view that also knows what its params were called with."""
|
||||
|
||||
def __init__(self, func, binding):
|
||||
self.func, self.binding = func, binding
|
||||
self.toks, self.start, self.end = func.toks, func.start, func.end
|
||||
self.params, self.path, self.name = func.params, func.path, func.name
|
||||
|
||||
def __getattr__(self, k):
|
||||
return getattr(self.func, k)
|
||||
|
||||
|
||||
# ── token helpers ───────────────────────────────────────────────────────────
|
||||
def split_args(toks, s, e):
|
||||
out, cur, d = [], s, 0
|
||||
i = s
|
||||
while i < e:
|
||||
v = toks[i].val
|
||||
if toks[i].kind == TOK_PUNCT and v in "([{":
|
||||
d += 1
|
||||
elif toks[i].kind == TOK_PUNCT and v in ")]}":
|
||||
d -= 1
|
||||
elif d == 0 and toks[i].kind == TOK_PUNCT and v == ",":
|
||||
out.append((cur, i))
|
||||
cur = i + 1
|
||||
i += 1
|
||||
if cur < e:
|
||||
out.append((cur, e))
|
||||
return out
|
||||
|
||||
|
||||
STMT_START = {"let", "return", "if", "while", "for"}
|
||||
KEYWORDS = {"if", "while", "for", "return", "fn", "let", "else", "match"}
|
||||
|
||||
|
||||
def stmt_end(toks, s, limit):
|
||||
"""End of the expression starting at s: the next top-level statement
|
||||
boundary. El has no semicolons, so a newline that starts a new statement
|
||||
ends this one."""
|
||||
d, i = 0, s
|
||||
while i < limit:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_PUNCT and t.val in "([":
|
||||
d += 1
|
||||
elif t.kind == TOK_PUNCT and t.val in ")]":
|
||||
d -= 1
|
||||
if d < 0:
|
||||
return i
|
||||
elif t.kind == TOK_PUNCT and t.val == "{":
|
||||
# a brace at depth 0 belongs to this expression only when it is an
|
||||
# if/else block that is part of it
|
||||
d += 1
|
||||
elif t.kind == TOK_PUNCT and t.val == "}":
|
||||
d -= 1
|
||||
if d < 0:
|
||||
return i
|
||||
elif d == 0 and t.kind == TOK_PUNCT and t.val == ",":
|
||||
return i
|
||||
elif d == 0 and i > s and t.kind == TOK_IDENT and t.val in STMT_START:
|
||||
if t.val == "if" and toks[i - 1].kind == TOK_IDENT and toks[i - 1].val == "else":
|
||||
i += 1
|
||||
continue
|
||||
return i
|
||||
elif d == 0 and i > s and t.kind == TOK_IDENT and t.val == "fn":
|
||||
return i
|
||||
i += 1
|
||||
return limit
|
||||
|
||||
|
||||
def block_tail(toks, s, e):
|
||||
"""The trailing expression of a block, if the block ends in one."""
|
||||
i, last = s, None
|
||||
while i < e:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val in ("let", "return"):
|
||||
i = stmt_end(toks, i + 1, e)
|
||||
last = None
|
||||
continue
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
i = match_close(toks, i, t.val, {"(": ")", "[": "]", "{": "}"}[t.val]) + 1
|
||||
continue
|
||||
st = i
|
||||
en = stmt_end(toks, i, e)
|
||||
if en <= st:
|
||||
i = st + 1
|
||||
continue
|
||||
last = (st, en)
|
||||
i = en
|
||||
return last
|
||||
|
||||
|
||||
def render(toks, s, e):
|
||||
out = []
|
||||
for t in toks[s:e]:
|
||||
out.append('"%s"' % t.val if t.kind == TOK_STR else t.val)
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
# ── the gate ────────────────────────────────────────────────────────────────
|
||||
def collect(root, include_tests):
|
||||
files = []
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [d for d in dirnames
|
||||
if d not in ("dist", "vendor", ".git", "node_modules")]
|
||||
rel_dir = os.path.relpath(dirpath, root)
|
||||
if not include_tests and rel_dir.split(os.sep)[0] == "tests":
|
||||
continue
|
||||
for fn in sorted(filenames):
|
||||
if fn.endswith(".el"):
|
||||
rel = os.path.normpath(os.path.join(rel_dir, fn))
|
||||
files.append((os.path.join(dirpath, fn), rel))
|
||||
return sorted(files, key=lambda x: x[1])
|
||||
|
||||
|
||||
def is_bare_literal(prog, site):
|
||||
toks = prog.files[site.path]
|
||||
s, e = site.arg_range
|
||||
return e == s + 1 and toks[s].kind == TOK_STR
|
||||
|
||||
|
||||
def read_decl(path):
|
||||
"""A declaration file: one entry per line, `# ...` comments stripped."""
|
||||
out = []
|
||||
if not path or not os.path.exists(path):
|
||||
return out
|
||||
with open(path) as fh:
|
||||
for ln in fh:
|
||||
ln = ln.split("#", 1)[0].strip()
|
||||
if ln:
|
||||
out.append(ln)
|
||||
return out
|
||||
|
||||
|
||||
def opt(argv, name, default=None):
|
||||
for i, a in enumerate(argv):
|
||||
if a == name and i + 1 < len(argv):
|
||||
return argv[i + 1]
|
||||
return default
|
||||
|
||||
|
||||
def main(argv):
|
||||
root = os.path.abspath(argv[1]) if len(argv) > 1 and not argv[1].startswith("-") else "."
|
||||
include_tests = "--include-tests" in argv
|
||||
verbose = "--verbose" in argv
|
||||
baseline_path = opt(argv, "--baseline")
|
||||
external_path = opt(argv, "--external")
|
||||
|
||||
prog = Program()
|
||||
for path, rel in collect(root, include_tests):
|
||||
prog.load(path, rel)
|
||||
prog.index()
|
||||
for site in prog.sites:
|
||||
pats, unres = prog.resolve(site.arg_range, site.func)
|
||||
site.pats, site.unresolved = {p for p in pats if p}, unres
|
||||
if is_bare_literal(prog, site):
|
||||
site.literal = prog.files[site.path][site.arg_range[0]].val
|
||||
|
||||
writes = [s for s in prog.sites if s.kind == "set"]
|
||||
reads = [s for s in prog.sites if s.kind == "get"]
|
||||
write_pats = set()
|
||||
for w in writes:
|
||||
write_pats |= w.pats
|
||||
|
||||
# Declared host-set keys: written by something outside the El tree (an
|
||||
# operator, the installer, a host process). Each entry must carry a reason.
|
||||
external = []
|
||||
for ln in read_decl(external_path):
|
||||
parts = ln.split(None, 1)
|
||||
if len(parts) != 2 or parts[0] not in (EXACT, PREFIX):
|
||||
print("bad --external line (want `exact|prefix <key>`): %r" % ln,
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
external.append((parts[0], parts[1]))
|
||||
write_pats |= set(external)
|
||||
|
||||
# F1 — a read of a key no write in the tree produces.
|
||||
f1 = []
|
||||
for r in reads:
|
||||
for p in sorted(r.pats):
|
||||
if not any(covers(w, p) for w in write_pats):
|
||||
f1.append((r, p))
|
||||
|
||||
# F2 — a key namespace owned by a helper, accessed by a hand-rolled literal.
|
||||
# This is the #129 shape: the producer moved behind conv_hist_key() and
|
||||
# one consumer kept spelling the old key out by hand.
|
||||
owners = {} # helper fn name -> its value set
|
||||
for s in prog.sites:
|
||||
toks = prog.files[s.path]
|
||||
a, b = s.arg_range
|
||||
if toks[a].kind == TOK_IDENT and a + 1 < b and toks[a + 1].val == "(" \
|
||||
and match_close(toks, a + 1, "(", ")") == b - 1 \
|
||||
and toks[a].val in prog.funcs:
|
||||
name = toks[a].val
|
||||
if name not in owners:
|
||||
vals = set()
|
||||
for callee in prog.funcs[name]:
|
||||
# No call context here on purpose: the OWNED namespace is
|
||||
# every key the helper can ever produce, over all call sites.
|
||||
for rng in prog.returns_of(callee):
|
||||
p, _ = prog._expr(callee.toks, rng[0], rng[1], callee, 0, set())
|
||||
vals |= {x for x in p if x}
|
||||
owners[name] = vals
|
||||
f2 = []
|
||||
for s in prog.sites:
|
||||
if s.literal is None:
|
||||
continue
|
||||
for owner, vals in sorted(owners.items()):
|
||||
for v in sorted(vals):
|
||||
if covers(v, pat_exact(s.literal)):
|
||||
f2.append((s, owner, v))
|
||||
break
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
unresolved = [s for s in prog.sites if s.unresolved or not s.pats]
|
||||
|
||||
# Baseline signatures carry NO line number on purpose: an unrelated edit that
|
||||
# shifts a line must not un-mute an accepted finding (that is crying wolf),
|
||||
# but a GROWTH in count must not hide either. So a baseline entry is
|
||||
# `<file> <CODE> <detail> [xN]` and only the first N matches are muted.
|
||||
baseline, bad_baseline = {}, []
|
||||
for ln in read_decl(baseline_path):
|
||||
n, key = 1, ln
|
||||
parts = ln.rsplit(" x", 1)
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
key, n = parts[0].strip(), int(parts[1])
|
||||
baseline[key] = n
|
||||
|
||||
def sig(path, code, detail):
|
||||
return "%s %s %s" % (path, code, detail)
|
||||
|
||||
findings = []
|
||||
for r, p in f1:
|
||||
findings.append((sig(r.path, "DEAD-READ", "%s:%s" % p), r.line,
|
||||
" %s:%d state_get(%s)\n resolves to %s %r — no state_set in the tree produces it"
|
||||
% (r.path, r.line, r.text, p[0].upper(), p[1])))
|
||||
for s, owner, v in f2:
|
||||
findings.append((sig(s.path, "HAND-ROLLED", "%s<-%s()" % (s.literal, owner)), s.line,
|
||||
" %s:%d state_%s(\"%s\")\n %s() owns this key namespace (%s %r) — go through the helper, "
|
||||
"or a rename orphans this site silently" % (s.path, s.line, s.kind, s.literal, owner, v[0].upper(), v[1])))
|
||||
findings.sort(key=lambda f: (f[0], f[1]))
|
||||
|
||||
live, muted, budget = [], [], dict(baseline)
|
||||
for f in findings:
|
||||
if budget.get(f[0], 0) > 0:
|
||||
budget[f[0]] -= 1
|
||||
muted.append(f)
|
||||
else:
|
||||
live.append(f)
|
||||
stale = sorted(k for k, v in budget.items() if v > 0)
|
||||
|
||||
print("── state-key audit ─────────────────────────────────────────────")
|
||||
print("scanned %d .el files%s" % (len(prog.files),
|
||||
"" if include_tests else " (tests/ excluded)"))
|
||||
print("sites %d state_set, %d state_get" % (len(writes), len(reads)))
|
||||
print("keys %d distinct write patterns" % len(write_pats))
|
||||
print("")
|
||||
|
||||
if verbose:
|
||||
print("WRITE PATTERNS")
|
||||
for k, v in sorted(write_pats):
|
||||
print(" %-6s %s" % (k, v))
|
||||
print("")
|
||||
|
||||
if external:
|
||||
print("DECLARED HOST-SET (%d) — %s" % (len(external), external_path))
|
||||
for k, v in sorted(external):
|
||||
print(" %-6s %s" % (k, v))
|
||||
print("")
|
||||
|
||||
print("UNRESOLVED (%d) — reported, never fails the build" % len(unresolved))
|
||||
if not unresolved:
|
||||
print(" (none)")
|
||||
for s in sorted(unresolved, key=lambda x: (x.path, x.line)):
|
||||
print(" %s:%d state_%s(%s)%s"
|
||||
% (s.path, s.line, s.kind, s.text,
|
||||
" [partial: %s]" % ", ".join("%s %r" % p for p in sorted(s.pats))
|
||||
if s.pats else ""))
|
||||
print("")
|
||||
|
||||
if muted:
|
||||
print("BASELINED (%d) — pre-existing debt accepted in %s. NOT clean; fix these."
|
||||
% (len(muted), baseline_path))
|
||||
for sg, line, _ in muted:
|
||||
print(" %s (line %d)" % (sg, line))
|
||||
print("")
|
||||
if stale:
|
||||
print("STALE BASELINE (%d) — entries that no longer match anything; delete them:"
|
||||
% len(stale))
|
||||
for sg in stale:
|
||||
print(" %s" % sg)
|
||||
print("")
|
||||
|
||||
print("FINDINGS (%d)" % len(live))
|
||||
if not live:
|
||||
print(" (none)")
|
||||
for _, _, body in live:
|
||||
print(body)
|
||||
print("")
|
||||
|
||||
if live:
|
||||
print("FAIL: %d state-key finding(s). See scripts/verify-state-keys.sh "
|
||||
"for why this gate exists (issue #129)." % len(live))
|
||||
return 1
|
||||
print("PASS: every resolvable state_get key has a producer, and no key "
|
||||
"namespace is spelled two ways.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -1,28 +0,0 @@
|
||||
# state-key-baseline.txt — findings that already existed when this gate landed
|
||||
# (2026-08-07). Each one is a REAL defect of the #129 class, not a false
|
||||
# positive. They are muted only so the gate can be turned on today instead of
|
||||
# being deferred until the debt is paid; every run still prints them under
|
||||
# BASELINED with the word "debt".
|
||||
#
|
||||
# THIS FILE SHOULD ONLY EVER SHRINK. Adding a line means you are shipping a
|
||||
# known silent-"" read. If you must, date it and say why in the comment.
|
||||
#
|
||||
# format: <file> <CODE> <detail> [xN] # N = how many sites are accepted
|
||||
# No line numbers on purpose: an unrelated edit must not un-mute an accepted
|
||||
# finding, but a GROWTH in count is NOT muted — the extra site fails the build.
|
||||
#
|
||||
chat.el DEAD-READ exact:soul_identity x5
|
||||
# ^ soul.el used to run `state_set("soul_identity", soul_identity)`. It was
|
||||
# deleted on 2026-05-13 in b163fa6 ("feat(awareness): route ISE writes to HTTP
|
||||
# Engram ..."), a commit about something else entirely, and the five readers in
|
||||
# chat.el were left behind. Since that date build_system_prompt (737), the
|
||||
# vision handler (1745), the agentic system prompt (2620), the council
|
||||
# transcript handler (3425) and 3480 have all been prefixing "" — exactly the
|
||||
# #129 shape, found by this gate on its first run. Sites: 737, 1745, 2620,
|
||||
# 3425, 3480. Fix = restore the boot-time write or delete the reads; not done
|
||||
# here because this branch must not change engine behaviour.
|
||||
|
||||
studio.el DEAD-READ exact:soul_principal x1
|
||||
# ^ studio.el:57 dharma_registry() emits "principal":"" on every call — no
|
||||
# producer has ever existed in the tree's history (git log -S finds none).
|
||||
# Never-wired rather than orphaned, same silent-"" result.
|
||||
@@ -1,16 +0,0 @@
|
||||
# state-key-external.txt — state keys the engine READS but deliberately never
|
||||
# WRITES, because a host outside the El tree sets them (an operator, the
|
||||
# installer, a deployment env). Read scripts/verify-state-keys.sh for why this
|
||||
# list has to exist and why it has to stay short.
|
||||
#
|
||||
# THE RULE FOR ADDING A LINE: the read site must already treat "" as a defined
|
||||
# default (`if str_eq(x, "") { <default> }`) AND the source must say so in a
|
||||
# comment. "I could not find the writer" is NOT a reason — that is the #129
|
||||
# defect, and it belongs in state-key-baseline.txt with a date, not here.
|
||||
#
|
||||
# format: exact|prefix <key> # why, and where the source says so
|
||||
#
|
||||
exact soul_rate_limit # routes.el:59-61 — "configurable via soul state key ... Falls back to 60 req/min if not set."
|
||||
exact web_search_tool_version # chat.el:1884-1910 — version lives in state "so a future bump is a config write, not a recompile"; defaults to web_search_20250305
|
||||
exact platform_auth # stewardship.el:92 — host-set capability flag; fail-CLOSED (anything but "true" denies the platform tool)
|
||||
exact security_research_authorized # awareness.el:991-996 — state override for env SECURITY_RESEARCH_TOKEN; fail-closed, defaults false
|
||||
@@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# verify-state-keys.sh — the state-key gate. Retires a defect class at build time.
|
||||
#
|
||||
# ── WHY THIS EXISTS. DO NOT DELETE IT AS NOISE. ──────────────────────────────
|
||||
#
|
||||
# The engine keeps runtime values in a key-value store: state_set("k", v) writes,
|
||||
# state_get("k") reads. A read of a key that NOTHING writes returns an empty
|
||||
# string. Silently. No error, no warning, no log line. The El compiler cannot see
|
||||
# it, no test sees it, and the product keeps running — just with a hole in it.
|
||||
#
|
||||
# That is how issue #129 happened. ff421d3 (2026-08-05) correctly moved
|
||||
# conversation history to a per-session key behind conv_hist_key(session_id). One
|
||||
# consumer did not move with it: the agentic path's L1 safety screen kept reading
|
||||
# the old anonymous "conv_history" bucket. The desktop app always mints a session
|
||||
# id, so history was always written under session_hist_<id> and that read always
|
||||
# returned "". The half of the crisis score that receives history is the
|
||||
# ESCALATION half — the one that exists for distress building across several
|
||||
# turns, where no single message trips the bell on its own. It scored 0 on every
|
||||
# real conversation for two days, and nothing failed.
|
||||
#
|
||||
# The line that broke carried a comment describing this exact bug being fixed
|
||||
# once already, under issue #9. A comment is not a gate. This is the gate.
|
||||
#
|
||||
# ── WHAT IT CHECKS ──────────────────────────────────────────────────────────
|
||||
#
|
||||
# DEAD-READ a state_get whose key resolves to something no state_set in the
|
||||
# tree produces. The direct form of the class.
|
||||
#
|
||||
# HAND-ROLLED a state_get/state_set that spells out a literal belonging to a
|
||||
# key namespace a helper function owns (e.g. "conv_history", owned
|
||||
# by conv_hist_key()). This is #129's actual shape: the producer
|
||||
# moved behind the helper and one consumer kept the old spelling
|
||||
# by hand. DEAD-READ alone does NOT catch #129, because the dead
|
||||
# handle_chat() still writes that key through the helper — so this
|
||||
# second check is the one that earns the gate its keep.
|
||||
#
|
||||
# ── WHY IT DOES NOT CRY WOLF ────────────────────────────────────────────────
|
||||
#
|
||||
# Keys are usually COMPUTED, not literal, so a naive grep would flood and get
|
||||
# switched off within a day. scripts/state-key-audit.py resolves computed keys:
|
||||
# string concatenation (matched on the static prefix), helper functions (resolved
|
||||
# to their possible return values), keys built into a local variable, and keys
|
||||
# arriving as a function parameter (resolved through the call sites). Where a key
|
||||
# genuinely cannot be resolved it is printed under UNRESOLVED and does NOT fail
|
||||
# the build — visible, never silently ignored. Keep that list short.
|
||||
#
|
||||
# On this tree it resolves 278 of 278 sites: UNRESOLVED is 0 and FINDINGS is 0.
|
||||
#
|
||||
# Two declaration files, both of which should only ever shrink:
|
||||
# scripts/state-key-external.txt keys a host outside the El tree writes
|
||||
# scripts/state-key-baseline.txt findings that predate the gate (real debt)
|
||||
#
|
||||
# ── PROVEN TO DISCRIMINATE (2026-08-07) ─────────────────────────────────────
|
||||
#
|
||||
# 1. Synthetic: a scratch copy of this tree with agentic_safety_screen reverted
|
||||
# to the pre-fix state_get("conv_history") — ONE line, nothing else — FAILS
|
||||
# with `chat.el:2536 ... conv_hist_key() owns this key namespace`. The tree
|
||||
# as shipped PASSES. One variable, opposite verdicts.
|
||||
# 2. Independent: run read-only against origin/feat/soul-openai-tools-v2, which
|
||||
# carries the same defect on its own, the gate reported chat.el:2937 — the
|
||||
# exact line 43d0449's commit message had named by hand. Against that
|
||||
# branch's fix (origin/fix/129-on-openai-tools) it passes.
|
||||
# 3. Producer-moved controls: renaming the sole writer of an EXACT key
|
||||
# (soul_model) orphans 3 readers across 3 files; renaming the sole writer of
|
||||
# a PREFIX namespace (agent_workspace_root_*) orphans 3 readers — including
|
||||
# when the producer moves to a NARROWER namespace, which an earlier,
|
||||
# sloppier prefix rule let through.
|
||||
#
|
||||
# It also found, on its first run, a defect nobody was looking for: soul.el's
|
||||
# `state_set("soul_identity", ...)` was deleted on 2026-05-13 in b163fa6 (a
|
||||
# commit about awareness/ISE writes) and five readers in chat.el were left
|
||||
# behind — the system prompt, the vision handler, the agentic prompt and the
|
||||
# council handler have been prefixing "" ever since. See state-key-baseline.txt.
|
||||
#
|
||||
# ── SAFETY ──────────────────────────────────────────────────────────────────
|
||||
# Pure static read of .el sources. Starts nothing, opens no port, touches no
|
||||
# daemon, and never reads or writes ~/.neuron.
|
||||
#
|
||||
# ── USAGE ───────────────────────────────────────────────────────────────────
|
||||
# scripts/verify-state-keys.sh gate the repo (honours baseline)
|
||||
# scripts/verify-state-keys.sh --strict ignore the baseline: show the debt
|
||||
# scripts/verify-state-keys.sh --verbose also dump every write pattern
|
||||
# scripts/verify-state-keys.sh --root DIR audit a different tree
|
||||
# exit 0 = clean; 1 = finding(s); 2 = the gate itself could not run.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
STRICT=0
|
||||
PASS_THROUGH=()
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--strict) STRICT=1; shift ;;
|
||||
--root) ROOT="${2:?--root needs a directory}"; shift 2 ;;
|
||||
-h|--help) awk 'NR>1 && /^#/ {print; next} NR>1 {exit}' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) PASS_THROUGH+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
command -v python3 >/dev/null 2>&1 || {
|
||||
echo "[state-keys] CANNOT RUN: python3 not found" >&2; exit 2; }
|
||||
[ -d "$ROOT" ] || { echo "[state-keys] CANNOT RUN: no such tree: $ROOT" >&2; exit 2; }
|
||||
|
||||
AUDIT="$SCRIPT_DIR/state-key-audit.py"
|
||||
[ -f "$AUDIT" ] || { echo "[state-keys] CANNOT RUN: missing $AUDIT" >&2; exit 2; }
|
||||
|
||||
ARGS=("$ROOT" "--external" "$SCRIPT_DIR/state-key-external.txt")
|
||||
[ "$STRICT" -eq 0 ] && ARGS+=("--baseline" "$SCRIPT_DIR/state-key-baseline.txt")
|
||||
[ ${#PASS_THROUGH[@]} -gt 0 ] && ARGS+=("${PASS_THROUGH[@]}")
|
||||
|
||||
python3 "$AUDIT" "${ARGS[@]}"
|
||||
RC=$?
|
||||
if [ "$RC" -gt 1 ]; then
|
||||
echo "[state-keys] CANNOT RUN: the audit itself failed (exit $RC)" >&2
|
||||
exit 2
|
||||
fi
|
||||
exit "$RC"
|
||||
@@ -0,0 +1,147 @@
|
||||
# Retrieval eval harness
|
||||
|
||||
Measures Neuron's memory retrieval so a change can be shown to help before it is
|
||||
believed to help. Nothing else on the memory roadmap should ship without a run
|
||||
through this.
|
||||
|
||||
```
|
||||
tools/retrieval-eval/run_comparison.sh --baseline main --candidate <branch>
|
||||
```
|
||||
|
||||
That builds a soul from each ref, boots each in isolation on a fixed corpus,
|
||||
runs the gold set three times per ref, and prints a table plus a verdict that
|
||||
refuses to call a difference real if it is inside the noise band.
|
||||
|
||||
## What was reused
|
||||
|
||||
This is not a new idea, it is the missing third of an existing one.
|
||||
|
||||
| Prior work | What it gave | What was missing |
|
||||
|---|---|---|
|
||||
| `docs/research/graphrag_eval/` (`collect.py`, `score.py`, 2026-06-08) | The three-retriever comparison that produced the numbers everyone quotes: substring 1.7% P@5, graph 21.7%, BM25 55%. Per-query relevant-id scoring, fixed-denominator precision@5, unique-relevant analysis. | 13 hand-written queries, judged by an LLM after the fact; measured the *live* soul on the *live* engram. |
|
||||
| `docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py` | The pinned-query discipline: ground truth committed as regexes so every run judges alike, plus a `--check` winnability gate. 40 queries in 5 bands including a deliberate paraphrase-hard band. | Scored offline replicas of substring/BM25 — it never ran the real retrieval path. |
|
||||
| `docs/research-archive/p0-prototypes/stage0_eval_20260714.py` | The `hit@5` metric and the substring/BM25 reference implementations. | Same: offline only. |
|
||||
| `scripts/verify-soul-contract.sh` | The isolation recipe, verbatim: throwaway port, throwaway `HOME`, `SOUL_ENGRAM_PATH`, and the non-obvious `SOUL_ISE_URL` pin that stops an "isolated" soul silently syncing the operator's live brain. | It is a contract gate, not a measurement. |
|
||||
| `_engine-liveness-91/gen-soul-amalgam.sh` + `.gitea/workflows/ci.yaml` | The build recipe (`elc --target=c` with every `.elh` on the import chain removed) and CI's exact compile flags. | — |
|
||||
|
||||
**Reused directly:** the isolation recipe, the build recipe, fixed-denominator
|
||||
precision@5, the pinned-ground-truth and winnability ideas.
|
||||
**New here:** ids rather than regexes as ground truth, an associative category
|
||||
derived from real graph edges, a superseded/contradicted category scored on
|
||||
ranking, a machine-checked zero-lexical-overlap guarantee on paraphrases,
|
||||
paired significance testing, and — the point — measurement against the **real
|
||||
compiled soul** rather than an offline replica of one leg of it.
|
||||
|
||||
## Design fit
|
||||
|
||||
The thing under measurement is Will's designed retrieval: spreading activation
|
||||
over the weighted directed graph, four-factor multiplicative scoring (parent
|
||||
strength x edge weight x target salience x query/target cosine). A Python
|
||||
re-implementation would measure my reading of the design. So the harness
|
||||
compiles the actual `soul.el` amalgam and asks it over HTTP on
|
||||
`/api/neuron/recall`, exactly as the MCP wrapper and the app do.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Does |
|
||||
|---|---|
|
||||
| `build_gold_set.py` | Derives and **validates** the gold set from the corpus. `--check` re-validates and exits non-zero if a query became unwinnable or a paraphrase leaked a word. |
|
||||
| `gold_set.json` | 38 queries. Every one carries a `derivation` string. |
|
||||
| `run_eval.py` | Boots one soul in isolation, runs the gold set, writes metrics. Kills and **confirms dead** its child; records the confirmation in the results file. |
|
||||
| `compare.py` | Paired diff of two result files with McNemar's exact test and a stated noise floor. |
|
||||
| `build-soul.sh` | Compiles a soul binary from a plain source tree. |
|
||||
| `run_comparison.sh` | All of the above, end to end, from two git refs. |
|
||||
|
||||
## The gold set — 38 queries
|
||||
|
||||
Built from the real corpus (`snapshot-pre-repair-20260806.json`, 78,768 nodes /
|
||||
14,214 edges) so it reflects one person's accumulating memory, not document QA.
|
||||
|
||||
| Category | n | Expected answer derived by |
|
||||
|---|---|---|
|
||||
| `exact_rare` | 6 | **Mined.** Tokens with document frequency 1 across all 78,768 nodes, whose single containing node is a 300–6000 char Memory/Knowledge/Belief. That node is the only possible answer. Re-verified every build. |
|
||||
| `phrase` | 7 | **Mined.** Case-insensitive verbatim scan; the matching set *is* the answer key. Phrases matching >25 nodes are rejected as too diffuse. |
|
||||
| `paraphrase` | 13 | **Hand-selected, machine-checked.** Target locked by id; the build then proves that **zero** content words of the query appear anywhere in the target's label, content, or tags. A leak fails the build — the category cannot quietly decay into lexical matching. |
|
||||
| `associative` | 6 | **Derived from edges.** Query built from one value node's distinctive vocabulary; expected answers are its siblings on the `Self - Values (grounded)` hub. Siblings sharing any query word are dropped, so the only route from query to answer is seed -> hub -> sibling. |
|
||||
| `nonsense` | 3 | **Control.** Verified that no token occurs anywhere in the corpus. Correct behaviour is to return nothing. |
|
||||
| `superseded` | 3 | **Derived.** Correction/stale pairs located by regex scan, kept only when both sides resolve to different surviving nodes. Scored on **ranking**: the correction must be returned *and* rank above the stale node. |
|
||||
|
||||
## Metrics
|
||||
|
||||
`hit@5`, `recall@5`, `recall@10`, `precision@5` (fixed denominator 5, so an
|
||||
empty result is punished like a page of junk), `MRR@10`, and wall-clock latency
|
||||
per query (p50/p95/max). Output is a table plus a machine-readable JSON per run
|
||||
so runs can be diffed.
|
||||
|
||||
## Honesty about noise
|
||||
|
||||
- **Minimum detectable swing on this 38-query set: 6 queries.** If every query
|
||||
that changes changes the same way, `p = 2 x 0.5^n`, which first drops under
|
||||
0.05 at n=6. Any net change smaller than that is inside the noise band and
|
||||
`compare.py` says so in those words.
|
||||
- **Run-to-run drift is measured, not assumed.** Activation is a stateful read
|
||||
by design (traversal reinforces what it touches), so identical inputs need not
|
||||
give identical outputs. Observed: `main` 0 queries of drift across 3 runs
|
||||
(fully deterministic); the activation branch 1 query.
|
||||
- The noise floor used for the verdict is `max(6, observed_drift + 1)`.
|
||||
- **This gold set is underpowered for small effects.** A genuine 3-query
|
||||
improvement would not clear the bar. Growing the set is the fix; until then, a
|
||||
small positive delta means "not shown", not "no effect".
|
||||
|
||||
## First result: `main` vs `feat/recall-through-activation`
|
||||
|
||||
Corpus and gold set identical, three runs each, fresh corpus copy per run.
|
||||
|
||||
| | main | recall-through-activation | delta |
|
||||
|---|---|---|---|
|
||||
| hit@5 | 34.3% | 22.9% | **-11.4pp** |
|
||||
| recall@5 | 26.9% | 19.1% | -7.9pp |
|
||||
| recall@10 | 33.3% | 24.3% | -9.1pp |
|
||||
| precision@5 | 12.0% | 7.4% | -4.6pp |
|
||||
| MRR@10 | 0.294 | 0.242 | -0.053 |
|
||||
| latency p50 | 1140 ms | 3209 ms | **2.81x** |
|
||||
| latency p95 | 1584 ms | 4852 ms | 3.06x |
|
||||
| nonsense clean | 2/3 | 2/3 | — |
|
||||
| superseded outranks | 1/3 | 0/3 | -1 |
|
||||
|
||||
By category (hit@5):
|
||||
|
||||
| category | main | activation |
|
||||
|---|---|---|
|
||||
| exact_rare | 100% | 100% |
|
||||
| phrase | 85.7% | **28.6%** |
|
||||
| paraphrase | 0% | 0% |
|
||||
| associative | 0% | 0% |
|
||||
| superseded | 0% | 0% |
|
||||
|
||||
**Verdict: directionally worse, one query short of significant.** 5 discordant
|
||||
pairs, all 5 against the candidate, 0 for it. McNemar exact p = 0.0625 — under
|
||||
the stated rule that is *inside* the noise band, so the harness reports "no
|
||||
measurable difference" on accuracy and the honest summary is "5 for 5 the wrong
|
||||
way, needs a 6th or a larger gold set to call".
|
||||
|
||||
Latency is a different story: 2.8x at p50 is deterministic and far outside any
|
||||
noise band. That regression is real.
|
||||
|
||||
The result the branch was written for did not appear. Its stated purpose was to
|
||||
recover sibling nodes one hub-hop away — the `associative` category — and that
|
||||
category is **0/6 on both builds**. Probing directly: for the query
|
||||
`Marines hernia sepsis medical ward`, the activation build returns the lexical
|
||||
seed node itself at rank 8, and none of its 12 hub siblings anywhere in the top
|
||||
10. The traversal is running; it is not reaching siblings.
|
||||
|
||||
Two corpus facts likely explain it, and both are measurable rather than
|
||||
speculative:
|
||||
|
||||
1. **The graph is nearly edgeless.** Only 4,060 of 78,768 nodes (5.2%) carry any
|
||||
edge at all — 14,214 edges total, 0.18 per node. Spreading activation over a
|
||||
graph with no edges is an expensive way to do lexical matching, which is
|
||||
roughly what the numbers show.
|
||||
2. **No embeddings.** No node in this snapshot has an embedding field, so the
|
||||
fourth factor of the four-factor product — query/target cosine similarity —
|
||||
has nothing to compute from, and the semantic seeding pass is inert.
|
||||
|
||||
That is the harness earning its keep on its first job: the change would have
|
||||
felt like progress (it is the designed mechanism, and it does run) and measures
|
||||
as a regression on phrase queries plus a 2.8x latency cost, with its intended
|
||||
benefit unrealised because the corpus lacks the structure it needs.
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-soul.sh — compile a soul binary from a plain source tree (no git needed).
|
||||
#
|
||||
# Reuses the amalgam recipe worked out in gen-soul-amalgam.sh (round 9.1) and the
|
||||
# compile flags from .gitea/workflows/ci.yaml, so the binary under test is the
|
||||
# same translation unit CI ships — not a re-implementation.
|
||||
#
|
||||
# elc --target=c emits only an extern prototype for any module that has a .elh
|
||||
# header beside it, and inlines the module's bodies when it does not. So the
|
||||
# amalgam is produced in a scratch copy with every .elh on the import chain
|
||||
# deleted.
|
||||
#
|
||||
# usage: build-soul.sh <src-tree-with-*.el> <out-binary>
|
||||
set -euo pipefail
|
||||
SRC="${1:?usage: build-soul.sh <src-tree> <out-binary>}"
|
||||
OUT="${2:?out-binary}"
|
||||
ELC="${ELC:-$HOME/neuron-dev-stack/src/el/lang/dist/platform/elc}"
|
||||
EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/el}"
|
||||
RTDIR="${RTDIR:-$SRC/vendor/el-runtime/v1.0.0-20260501}"
|
||||
SSL="${SSL_PREFIX:-/opt/homebrew/opt/openssl@3}"
|
||||
|
||||
[ -x "$ELC" ] || { echo "no elc at $ELC" >&2; exit 2; }
|
||||
[ -f "$RTDIR/el_runtime.c" ] || { echo "no el_runtime.c at $RTDIR" >&2; exit 2; }
|
||||
|
||||
GEN="$(mktemp -d "${TMPDIR:-/tmp}/soul-build.XXXXXX")"
|
||||
trap 'rm -rf "$GEN"' EXIT
|
||||
mkdir -p "$GEN/neuron" "$GEN/foundation/el/elp/src"
|
||||
cp "$SRC"/*.el "$GEN/neuron/"
|
||||
cp "$EL_REPO"/elp/src/*.el "$GEN/foundation/el/elp/src/"
|
||||
find "$GEN" -name '*.elh' -delete
|
||||
|
||||
( cd "$GEN/neuron" && "$ELC" --target=c soul.el ) > "$GEN/soul.c"
|
||||
BODIES=$(grep -c '^el_val_t .*) {$' "$GEN/soul.c" || true)
|
||||
echo "[build-soul] amalgam $(wc -c < "$GEN/soul.c" | tr -d ' ') bytes, ${BODIES} inlined bodies"
|
||||
[ "$BODIES" -ge 1200 ] || { echo "[build-soul] FAIL: only $BODIES bodies — an import was not inlined"; exit 1; }
|
||||
|
||||
cc -O2 -DHAVE_CURL -rdynamic \
|
||||
-I"$RTDIR" -I"$SSL/include" -L"$SSL/lib" \
|
||||
"$GEN/soul.c" "$RTDIR/el_runtime.c" \
|
||||
-lssl -lcrypto -lcurl -lpthread -lm \
|
||||
-o "$OUT" 2> "$GEN/cc.log" || { echo "[build-soul] FAIL compile"; tail -40 "$GEN/cc.log"; exit 1; }
|
||||
if grep -qE 'implicit.*(engram_|el_)' "$GEN/cc.log"; then
|
||||
echo "[build-soul] FAIL: implicit declarations of runtime symbols"; grep -E 'implicit' "$GEN/cc.log" | head; exit 1; fi
|
||||
echo "[build-soul] OK -> $OUT ($(wc -c < "$OUT" | tr -d ' ') bytes)"
|
||||
Executable
+506
@@ -0,0 +1,506 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
build_gold_set.py — derive the retrieval gold set FROM the corpus, and validate it.
|
||||
|
||||
WHY THIS FILE EXISTS AS CODE AND NOT AS A HAND-WRITTEN JSON
|
||||
A gold set nobody can audit is vibes with extra steps. Every expected answer
|
||||
here is either (a) mined from the corpus by a rule this script re-runs, or
|
||||
(b) hand-selected with a stated criterion that this script then CHECKS
|
||||
against the corpus. Both leave a `derivation` string on every query, and the
|
||||
checks are re-run on demand so the set cannot silently rot as the corpus
|
||||
changes.
|
||||
|
||||
Lineage: this extends the pinned-query approach from
|
||||
docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py (pinned
|
||||
ground-truth patterns + a --check "winnability" gate) and the per-query
|
||||
relevant-id scoring from docs/research/graphrag_eval/score.py. What is new:
|
||||
ids as ground truth rather than regexes alone, an ASSOCIATIVE category
|
||||
derived from real graph edges, a superseded/contradicted category, and a
|
||||
machine-checked no-lexical-overlap guarantee on the paraphrase category.
|
||||
|
||||
THE SIX CATEGORIES, AND WHAT EACH ONE IS FOR
|
||||
exact_rare a single rare word. Substring matching already wins these.
|
||||
They are a REGRESSION GUARD: any change that loses them is
|
||||
disqualified regardless of what else it gains.
|
||||
phrase a multi-word string that exists verbatim in the corpus.
|
||||
Guards multi-token queries, which the old substring matcher
|
||||
handled by returning nothing.
|
||||
paraphrase same meaning, ZERO shared content words with the target node.
|
||||
THE CATEGORY THAT MATTERS. Mechanically unreachable by string
|
||||
matching; reachable only by semantics or by association.
|
||||
associative the answer is one hub-hop from an obvious starting point and
|
||||
shares no words with the query. This is the case the graph is
|
||||
supposed to buy: query one value, get its siblings.
|
||||
nonsense must return nothing. Guards against a retriever that "improves"
|
||||
recall by returning the whole graph.
|
||||
superseded a fact that was later corrected. The correction must OUTRANK
|
||||
the stale version — ranking, not mere presence.
|
||||
|
||||
usage:
|
||||
python3 build_gold_set.py <snapshot.json> [--out gold_set.json] [--check]
|
||||
--check re-validates an existing gold_set.json against the corpus and exits
|
||||
non-zero if any query became unwinnable or any paraphrase leaked a word.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DEFAULT_OUT = os.path.join(HERE, "gold_set.json")
|
||||
|
||||
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
|
||||
|
||||
# Stopwords are deliberately generous. A paraphrase query is only interesting if
|
||||
# its CONTENT words are absent from the target; "the", "is", "what" appearing in
|
||||
# both proves nothing. Being generous here makes the overlap test STRICTER on
|
||||
# the words that carry meaning, which is the conservative direction.
|
||||
STOP = set("""
|
||||
a about above after again against all also am an and any are aren't as at be because been
|
||||
before being below between both but by can can't cannot could couldn't did didn't do does
|
||||
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
|
||||
having he her here hers herself him himself his how i if in into is isn't it its itself just
|
||||
me more most my myself no nor not of off on once only or other others ought our ours ourselves
|
||||
out over own same shan't she should shouldn't so some such than that the their theirs them
|
||||
themselves then there these they this those through to too under until up very was wasn't we
|
||||
were weren't what when where which while who whom why will with won't would wouldn't you your
|
||||
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
|
||||
things does doing done keep keeps kept go goes going come comes came one two something anything
|
||||
""".split())
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# corpus helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def load_corpus(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
data = json.load(fh)
|
||||
nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
|
||||
edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
|
||||
return nodes, edges
|
||||
|
||||
|
||||
def doctext(n):
|
||||
return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
|
||||
|
||||
|
||||
def content_tokens(s):
|
||||
return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# hand-authored queries. Every entry states HOW its expected answer was chosen.
|
||||
# The `check` field names the validation this script runs against the corpus.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# EXACT_RARE — mined, not chosen. The rule (re-run by mine_exact_rare below):
|
||||
# tokens whose document frequency across the whole corpus is 1, whose single
|
||||
# containing node is a Memory/Knowledge/Belief with 300-6000 chars of content
|
||||
# (so the answer is a real memory, not a 117KB whitepaper that contains every
|
||||
# word in English), and whose token is plain lowercase alphabetic. The expected
|
||||
# answer is that one node — it is the only node that can possibly be correct.
|
||||
EXACT_RARE_SEEDS = [
|
||||
"unjailbreakable",
|
||||
"engram-migrate",
|
||||
"cartabandonedevent",
|
||||
"pre-apprenticeship",
|
||||
"inferencenodemanager",
|
||||
"clear-eyed",
|
||||
]
|
||||
|
||||
# PHRASE — chosen by reading the corpus for phrases that (a) occur verbatim,
|
||||
# (b) occur in a small enough set of nodes that "relevant" is well defined.
|
||||
# Expected answers are computed here as EVERY node whose text contains the
|
||||
# phrase case-insensitively — so the answer set is a fact about the corpus, not
|
||||
# an opinion. Queries whose phrase matches more than PHRASE_MAX nodes are
|
||||
# rejected by validation as too diffuse to score.
|
||||
PHRASE_MAX = 25
|
||||
PHRASE_SEEDS = [
|
||||
("patterns not returns",
|
||||
"a verbatim correction Will issued; expected = every node containing the phrase"),
|
||||
("thirty moves",
|
||||
"the canonical biographical phrase; expected = every node containing it"),
|
||||
("Grandma Lucas",
|
||||
"a named person appearing verbatim in the biography/value nodes"),
|
||||
("Directed Harmonic",
|
||||
"the canonical DHARMA expansion, confirmed by Will April 24 2026"),
|
||||
("Sarah Bishop",
|
||||
"a named person; rare enough that the answer set is unambiguous"),
|
||||
("Directed Autonomous Runtime Modification",
|
||||
"the DARMA expansion, quoted verbatim in the backlog item and its correction"),
|
||||
("zero-knowledge encrypted backup",
|
||||
"the paid-tier feature name as written in the roadmap nodes"),
|
||||
]
|
||||
|
||||
# PARAPHRASE — hand-authored. THE SELECTION CRITERION, stated once and applied
|
||||
# to all nine: pick a node whose SUBJECT is unmistakable to a reader, then write
|
||||
# the query a person would actually type when they remember the subject but not
|
||||
# the words. The target is then LOCKED by id, and this script enforces the hard
|
||||
# property that makes the category meaningful: not one content word of the query
|
||||
# appears anywhere in the target node's label, content, or tags. If a word
|
||||
# leaks, validation fails and the query must be rewritten — the set cannot
|
||||
# quietly degrade into a lexical query wearing a paraphrase costume.
|
||||
PARAPHRASE_SEEDS = [
|
||||
("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"the elderly relative who passed while he stayed away",
|
||||
"target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas "
|
||||
"dying in Feb 2006 without Will saying goodbye. Query names the event with none of the "
|
||||
"node's own vocabulary."),
|
||||
("kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a soldier sidelined by illness who refused to quit",
|
||||
"target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the "
|
||||
"Marines, a severe hernia, and sepsis. Query describes the episode obliquely."),
|
||||
("kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"choosing an uncomfortable fact over a pleasant fiction",
|
||||
"target: 'Value - Honesty Before Comfort'. Query states the principle in wholly "
|
||||
"different words."),
|
||||
("kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"a tight payload beats a bloated one",
|
||||
"target: 'Value - Precision Over Brute Force'. Query restates the claim with no "
|
||||
"shared vocabulary."),
|
||||
("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"if you are able and nobody is coming the job is yours",
|
||||
"target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the "
|
||||
"obligation without the node's terms."),
|
||||
("kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"learning is the wealth creditors cannot seize",
|
||||
"target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the "
|
||||
"library following Will across 30+ moves."),
|
||||
("kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"reliability proven by track record not assertion",
|
||||
"target: 'Value - Earned Trust' ('Trust is demonstrated, not declared')."),
|
||||
("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"boundaries that enable instead of confine",
|
||||
"target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim."),
|
||||
("kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"what shifts tells you where to cut a system apart",
|
||||
"target: 'Value - Change Is the Signal', the value VBD is built on."),
|
||||
("kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"a mind that compounds instead of resetting each day",
|
||||
"target: 'Value - The System Must Accumulate'. Query is the accumulation claim in "
|
||||
"different vocabulary."),
|
||||
("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"loved for the unedited self and not the polished exterior",
|
||||
"target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop "
|
||||
"as the first person Will did not perform for."),
|
||||
("kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"cheerfulness you arrive at instead of assuming",
|
||||
"target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'."),
|
||||
("kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"a childhood offering no solid foundation to inherit",
|
||||
"target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between "
|
||||
"two parents' collapses."),
|
||||
]
|
||||
|
||||
# ASSOCIATIVE — derived from real edges, not authored. The construction:
|
||||
# every value node hangs off the 'Self - Values (grounded)' hub by an `identity`
|
||||
# edge. For a chosen value node V, the query is built from V's own distinctive
|
||||
# vocabulary; the expected answers are V's SIBLINGS on that hub. A sibling
|
||||
# shares no query words with the query by construction (validated below), so the
|
||||
# only path from the query to a sibling is: lexical seed on V -> hub -> sibling.
|
||||
# That is a two-hop traversal and nothing else can produce it.
|
||||
VALUES_HUB = "kn-5b606390-a52d-4ca2-8e0e-eba141d13440"
|
||||
ASSOCIATIVE_SEEDS = [
|
||||
("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", "Grandma Lucas stroke February 2006 goodbye window"),
|
||||
("kn-58874a74-b96f-4883-9e08-45707f4bd3ee", "Marines hernia sepsis medical ward"),
|
||||
("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", "Sarah Bishop Dyer trailer performance"),
|
||||
("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", "Swarm Architecture containment lateral worker"),
|
||||
("kn-e0423482-cfa5-4796-8689-8495c93b66bc", "hope won inside the narrative preface"),
|
||||
("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", "man of the house six years old expectation"),
|
||||
]
|
||||
|
||||
# NONSENSE — must return nothing. Strings chosen to be lexically impossible:
|
||||
# validation asserts each appears in ZERO corpus nodes as a substring and that
|
||||
# none of its tokens appears anywhere either (so not even a partial seed exists).
|
||||
NONSENSE_SEEDS = [
|
||||
"zqxjvw plimforth grebulon",
|
||||
"flarnbistle quommetry",
|
||||
"xxqzzt vurblenacht throom",
|
||||
]
|
||||
|
||||
# SUPERSEDED — a fact that was corrected. Chosen by searching the corpus for
|
||||
# explicit correction language and keeping pairs where BOTH the stale statement
|
||||
# and its correction exist as separate nodes. Scored on RANKING: the correction
|
||||
# must appear, and must appear above the stale node. Ids are locked here and
|
||||
# validated to exist and to match their stated role.
|
||||
SUPERSEDED_SEEDS = [
|
||||
# (query, correct_id, stale_id, derivation)
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# mining
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def mine_exact_rare(nodes, byid, seeds):
|
||||
"""Re-derive: confirm each seed token still has df==1 and name its node."""
|
||||
tok = re.compile(r"[A-Za-z][A-Za-z0-9\-]{4,}")
|
||||
want = set(seeds)
|
||||
df = Counter()
|
||||
post = defaultdict(set)
|
||||
for n in nodes:
|
||||
for t in {w.lower() for w in tok.findall(doctext(n))}:
|
||||
if t in want:
|
||||
df[t] += 1
|
||||
post[t].add(n["id"])
|
||||
out = []
|
||||
for s in seeds:
|
||||
ids = sorted(post.get(s, ()))
|
||||
out.append((s, ids, df.get(s, 0)))
|
||||
return out
|
||||
|
||||
|
||||
def phrase_matches(nodes, phrase):
|
||||
p = phrase.lower()
|
||||
return sorted(n["id"] for n in nodes if p in doctext(n).lower())
|
||||
|
||||
|
||||
def hub_siblings(edges, hub, relation="identity"):
|
||||
sibs = []
|
||||
for e in edges:
|
||||
if e.get("from_id") == hub and e.get("relation") == relation:
|
||||
sibs.append(e["to_id"])
|
||||
elif e.get("to_id") == hub and e.get("relation") == relation:
|
||||
sibs.append(e["from_id"])
|
||||
return list(dict.fromkeys(sibs))
|
||||
|
||||
|
||||
def find_superseded_pairs(nodes, byid):
|
||||
"""Locked pairs, each verified here to exist and to carry its stated marker.
|
||||
|
||||
Chosen by scanning the corpus for explicit correction language
|
||||
(CORRECTION/SUPERSEDES/re-corrected/no longer/RECONCILED) and keeping only
|
||||
cases where the STALE claim also survives as its own node — a supersession
|
||||
with nothing to outrank is not a ranking test.
|
||||
"""
|
||||
pairs = []
|
||||
txt = {n["id"]: doctext(n) for n in nodes}
|
||||
|
||||
def find_one(pattern, exclude=()):
|
||||
rx = re.compile(pattern)
|
||||
return [n["id"] for n in nodes
|
||||
if n["id"] not in exclude
|
||||
and rx.search(txt[n["id"]])
|
||||
and 150 < len(str(n.get("content") or "")) < 12000
|
||||
and n.get("node_type") in ("Memory", "Knowledge", "Belief", "BacklogItem")]
|
||||
|
||||
# Each entry: (query, correction-pattern, stale-pattern, why).
|
||||
# The stale side is searched with the correction hits EXCLUDED, because most
|
||||
# correction memories quote the claim they are killing — without the
|
||||
# exclusion the "stale" node resolves to the correction itself and the pair
|
||||
# collapses into a no-op. A pair is only emitted if both sides resolve to
|
||||
# DIFFERENT surviving nodes; otherwise it is dropped and reported.
|
||||
SPECS = [
|
||||
("is the self-improvement architecture called DARMA or DHARMA",
|
||||
r'(?i)CORRECTION:.{0,90}DHARMA .{0,12}not DARMA',
|
||||
r'(?i)\bDARMA\b',
|
||||
"correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); "
|
||||
"the stale node is the surviving backlog item still titled 'Implement DARMA'."),
|
||||
|
||||
("how many provisional patents does Will actually have",
|
||||
r'(?i)EXACTLY 6 (fully-specced )?provisional',
|
||||
r'(?i)(MY ARCHITECTURE = 12 filed patents|\b12 filed patents\b)',
|
||||
"correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; "
|
||||
"the stale node is the surviving memory that asserts 12 filed patents."),
|
||||
|
||||
("is MCP still the live integration layer",
|
||||
r'(?i)MCP RETIRED',
|
||||
r'(?i)MCP server live at',
|
||||
"correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of "
|
||||
"April 30 2026; the stale node still records the MCP server as live."),
|
||||
|
||||
("what does the patterns-not-returns directive mean",
|
||||
r'(?i)CORRECTION:.{0,80}patterns not returns',
|
||||
r'(?i)established returns',
|
||||
"correction node is Will's 'patterns not returns' correction; the stale node is a "
|
||||
"surviving node carrying the misread 'established returns' directive."),
|
||||
|
||||
("was the earlier identity-bug finding correct",
|
||||
r'(?i)SUPERSEDES the earlier .critical identity bug',
|
||||
r'(?i)critical identity bug',
|
||||
"correction node explicitly supersedes the 'critical identity bug' finding; the stale "
|
||||
"node is the surviving original finding."),
|
||||
|
||||
("does Neuron have recursive self-improvement",
|
||||
r'(?i)twice answered .Neuron has no recursive self-improvement',
|
||||
r'(?i)no recursive self-improvement',
|
||||
"correction node records the June-29 finding that the CGI provisional IS the "
|
||||
"recursive-self-improvement mechanism; the stale node is the surviving denial."),
|
||||
]
|
||||
|
||||
for query, cpat, spat, why in SPECS:
|
||||
corr = find_one(cpat)
|
||||
if not corr:
|
||||
continue
|
||||
stale = find_one(spat, exclude=set(corr))
|
||||
if not stale:
|
||||
continue
|
||||
pairs.append((query, corr[0], stale[0], why))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# build
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def build(nodes, edges):
|
||||
byid = {n["id"]: n for n in nodes}
|
||||
tokset = {n["id"]: content_tokens(doctext(n)) for n in nodes}
|
||||
queries = []
|
||||
problems = []
|
||||
qn = [0]
|
||||
|
||||
def add(cat, query, relevant, derivation, **extra):
|
||||
qn[0] += 1
|
||||
q = {
|
||||
"id": f"q{qn[0]:02d}",
|
||||
"category": cat,
|
||||
"query": query,
|
||||
"relevant": sorted(relevant),
|
||||
"derivation": derivation,
|
||||
}
|
||||
q.update(extra)
|
||||
queries.append(q)
|
||||
return q
|
||||
|
||||
# --- exact_rare ---------------------------------------------------------
|
||||
for tokname, ids, df in mine_exact_rare(nodes, byid, EXACT_RARE_SEEDS):
|
||||
if df != 1 or len(ids) != 1:
|
||||
problems.append(f"exact_rare '{tokname}': df={df}, ids={len(ids)} (expected df=1)")
|
||||
continue
|
||||
lab = (byid[ids[0]].get("label") or "")[:60]
|
||||
add("exact_rare", tokname, ids,
|
||||
f"MINED: token '{tokname}' has document frequency 1 over all {len(nodes)} corpus nodes "
|
||||
f"(re-verified at build time). Its single containing node is {ids[0]} "
|
||||
f"('{lab}'), which is therefore the only possible correct answer.")
|
||||
|
||||
# --- phrase -------------------------------------------------------------
|
||||
for phrase, why in PHRASE_SEEDS:
|
||||
ids = phrase_matches(nodes, phrase)
|
||||
if not ids:
|
||||
problems.append(f"phrase '{phrase}': 0 corpus matches — unwinnable")
|
||||
continue
|
||||
if len(ids) > PHRASE_MAX:
|
||||
problems.append(f"phrase '{phrase}': {len(ids)} matches > {PHRASE_MAX} — too diffuse")
|
||||
continue
|
||||
add("phrase", phrase, ids,
|
||||
f"MINED: {why}. Case-insensitive verbatim substring scan over label+content+tags at "
|
||||
f"build time returns exactly {len(ids)} node(s); that set IS the answer key.")
|
||||
|
||||
# --- paraphrase ---------------------------------------------------------
|
||||
for target, query, why in PARAPHRASE_SEEDS:
|
||||
if target not in byid:
|
||||
problems.append(f"paraphrase target {target} not in corpus")
|
||||
continue
|
||||
qt = content_tokens(query)
|
||||
leak = sorted(qt & tokset[target])
|
||||
if leak:
|
||||
problems.append(f"paraphrase '{query}': leaks {leak} into target {target}")
|
||||
continue
|
||||
add("paraphrase", query, [target],
|
||||
f"HAND-SELECTED with criterion: {why} VERIFIED at build time: of the {len(qt)} content "
|
||||
f"words in the query, ZERO appear anywhere in the target's label, content, or tags — so "
|
||||
f"no string-matching retriever can reach this answer.",
|
||||
zero_overlap_verified=True, query_content_words=sorted(qt))
|
||||
|
||||
# --- associative --------------------------------------------------------
|
||||
sibs = hub_siblings(edges, VALUES_HUB)
|
||||
if len(sibs) < 5:
|
||||
problems.append(f"associative: values hub {VALUES_HUB} has only {len(sibs)} siblings")
|
||||
for src, query in ASSOCIATIVE_SEEDS:
|
||||
if src not in byid or src not in sibs:
|
||||
problems.append(f"associative source {src} not a sibling on {VALUES_HUB}")
|
||||
continue
|
||||
qt = content_tokens(query)
|
||||
others = [s for s in sibs if s != src and s in byid]
|
||||
# A sibling only counts as a legitimate expected answer if the query
|
||||
# cannot reach it lexically. Drop any sibling that shares a content word.
|
||||
clean = [s for s in others if not (qt & tokset[s])]
|
||||
dropped = len(others) - len(clean)
|
||||
if len(clean) < 5:
|
||||
problems.append(f"associative '{query}': only {len(clean)} lexically-unreachable siblings")
|
||||
continue
|
||||
add("associative", query, clean,
|
||||
f"DERIVED FROM EDGES: the query is built from the distinctive vocabulary of {src} "
|
||||
f"('{(byid[src].get('label') or '')[:48]}'), which hangs off the values hub {VALUES_HUB} "
|
||||
f"by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub "
|
||||
f"({len(clean)} of {len(others)}; {dropped} dropped because they shared a query word and "
|
||||
f"so were lexically reachable). Every remaining sibling shares ZERO content words with "
|
||||
f"the query — the only route from query to answer is seed({src}) -> hub -> sibling, a "
|
||||
f"two-hop traversal.",
|
||||
associative_source=src, hub=VALUES_HUB, siblings_dropped_for_overlap=dropped)
|
||||
|
||||
# --- nonsense -----------------------------------------------------------
|
||||
all_tokens = set()
|
||||
for n in nodes:
|
||||
all_tokens |= {t for t in TOKEN.findall(doctext(n).lower())}
|
||||
for s in NONSENSE_SEEDS:
|
||||
present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
|
||||
if present:
|
||||
problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
|
||||
continue
|
||||
add("nonsense", s, [],
|
||||
f"CONTROL: verified at build time that none of this string's tokens occurs anywhere in "
|
||||
f"the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
expect_empty=True)
|
||||
|
||||
# --- superseded ---------------------------------------------------------
|
||||
for query, correct, stale, why in find_superseded_pairs(nodes, byid):
|
||||
if correct not in byid or stale not in byid:
|
||||
problems.append(f"superseded '{query}': id missing from corpus")
|
||||
continue
|
||||
add("superseded", query, [correct],
|
||||
f"DERIVED: {why} Scored on RANKING, not presence: the corrected node {correct} must be "
|
||||
f"returned AND must rank above the stale node {stale}.",
|
||||
must_outrank=[correct, stale],
|
||||
stale_id=stale,
|
||||
correct_label=(byid[correct].get("label") or "")[:70],
|
||||
stale_label=(byid[stale].get("label") or "")[:70])
|
||||
|
||||
return queries, problems
|
||||
|
||||
|
||||
def summarize(queries):
|
||||
c = Counter(q["category"] for q in queries)
|
||||
return ", ".join(f"{k}={c[k]}" for k in
|
||||
("exact_rare", "phrase", "paraphrase", "associative", "nonsense", "superseded")
|
||||
if c[k])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("snapshot")
|
||||
ap.add_argument("--out", default=DEFAULT_OUT)
|
||||
ap.add_argument("--check", action="store_true",
|
||||
help="validate only; do not write. Non-zero exit if anything is unwinnable.")
|
||||
args = ap.parse_args()
|
||||
|
||||
nodes, edges = load_corpus(args.snapshot)
|
||||
print(f"corpus: {len(nodes)} nodes, {len(edges)} edges ({os.path.basename(args.snapshot)})")
|
||||
queries, problems = build(nodes, edges)
|
||||
|
||||
print(f"gold set: {len(queries)} queries [{summarize(queries)}]")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} PROBLEM(S) — these queries were REJECTED, not silently kept:")
|
||||
for p in problems:
|
||||
print(" -", p)
|
||||
|
||||
if args.check:
|
||||
sys.exit(1 if problems else 0)
|
||||
|
||||
doc = {
|
||||
"corpus": os.path.abspath(args.snapshot),
|
||||
"corpus_nodes": len(nodes),
|
||||
"corpus_edges": len(edges),
|
||||
"note": ("Every query carries a `derivation` recording how its expected answer was chosen. "
|
||||
"Re-run with --check to re-validate the whole set against the corpus."),
|
||||
"queries": queries,
|
||||
}
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+210
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
compare.py — diff two run_eval.py result files, WITH a noise threshold.
|
||||
|
||||
WHY THE STATISTICS ARE NOT OPTIONAL
|
||||
With ~35 scored queries, one query is ~2.9 percentage points. A harness that
|
||||
reports "hit@5 improved 2.9%" without saying that is one query is a harness
|
||||
that will approve noise. So this file refuses to call anything an
|
||||
improvement on the strength of the headline number alone. It reports:
|
||||
|
||||
1. The DISCORDANT PAIRS. Two configurations scored on the same queries are
|
||||
paired data, so the only queries carrying information are the ones
|
||||
where they disagree: b = fixed by B, c = broken by B. Queries both got
|
||||
right, or both got wrong, tell you nothing about which is better.
|
||||
|
||||
2. McNEMAR'S EXACT TEST on (b, c). Under the null "the change is a coin
|
||||
flip", the discordant outcomes are Binomial(b+c, 0.5). The two-sided
|
||||
exact p-value is computed here with no scipy dependency.
|
||||
|
||||
3. The MINIMUM DETECTABLE SWING for this gold set: the smallest number of
|
||||
net-changed queries that would reach p < 0.05 if every discordant pair
|
||||
fell the same way. Anything smaller is inside the noise band, and the
|
||||
verdict line says so in those words.
|
||||
|
||||
Repeat-run variance is the other half of honesty. Spreading activation is a
|
||||
stateful read (it reinforces what it touches), so identical inputs need not
|
||||
give identical outputs. Pass --repeats to fold several runs of the same
|
||||
config into an observed variance band; a delta inside that band is not real
|
||||
either, however good its p-value looks.
|
||||
|
||||
usage:
|
||||
python3 compare.py --baseline results-main.json --candidate results-act.json
|
||||
python3 compare.py --baseline a.json --candidate b.json \
|
||||
--repeats-baseline a2.json a3.json --repeats-candidate b2.json b3.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from math import comb
|
||||
|
||||
|
||||
def binom_two_sided(b, c):
|
||||
"""Two-sided exact binomial p for b successes in n=b+c at p=0.5."""
|
||||
n = b + c
|
||||
if n == 0:
|
||||
return 1.0
|
||||
k = min(b, c)
|
||||
tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n)
|
||||
return min(1.0, 2 * tail)
|
||||
|
||||
|
||||
def min_detectable_swing(n_scored, alpha=0.05):
|
||||
"""Smallest all-one-way discordant count reaching p < alpha.
|
||||
|
||||
If every query that changes changes in the same direction, the p-value is
|
||||
2 * 0.5**n. Solve for the smallest n where that drops under alpha. This is
|
||||
the FLOOR: any real change will have some discordance both ways, so the true
|
||||
requirement is larger. Reporting the floor is the conservative move — it is
|
||||
the most generous threshold we would ever accept.
|
||||
"""
|
||||
n = 1
|
||||
while n <= n_scored:
|
||||
if 2 * (0.5 ** n) < alpha:
|
||||
return n
|
||||
n += 1
|
||||
return n_scored
|
||||
|
||||
|
||||
def load(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def row_map(doc):
|
||||
return {r["id"]: r for r in doc["rows"]}
|
||||
|
||||
|
||||
def outcome(r):
|
||||
"""Binary per-query outcome used for the paired test.
|
||||
|
||||
hit@5 for scored queries; 'returned nothing' for the nonsense controls;
|
||||
'correction outranks the stale node' for the superseded queries. One number
|
||||
per query, so every query votes exactly once.
|
||||
"""
|
||||
if "clean" in r:
|
||||
return 1.0 if r["clean"] else 0.0
|
||||
if "outranks" in r:
|
||||
return 1.0 if r["outranks"] else 0.0
|
||||
return r.get("hit@5") or 0.0
|
||||
|
||||
|
||||
def band(values):
|
||||
return (min(values), max(values))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--baseline", required=True)
|
||||
ap.add_argument("--candidate", required=True)
|
||||
ap.add_argument("--repeats-baseline", nargs="*", default=[])
|
||||
ap.add_argument("--repeats-candidate", nargs="*", default=[])
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
A, B = load(args.baseline), load(args.candidate)
|
||||
ra, rb = row_map(A), row_map(B)
|
||||
ids = [q for q in ra if q in rb]
|
||||
n = len(ids)
|
||||
|
||||
aa, ab = A["aggregate"], B["aggregate"]
|
||||
print(f"baseline {A['label']:14} soul={A['soul_md5'][:12]} {n} shared queries")
|
||||
print(f"candidate {B['label']:14} soul={B['soul_md5'][:12]}")
|
||||
print(f"corpus {A['corpus_nodes']} nodes / {A['corpus_edges']} edges "
|
||||
f"(identical copy for both runs)\n")
|
||||
|
||||
metrics = [("hit@5", 1), ("recall@5", 1), ("recall@10", 1),
|
||||
("precision@5", 1), ("mrr@10", 0)]
|
||||
print(f" {'metric':14} {'baseline':>10} {'candidate':>10} {'delta':>10}")
|
||||
for m, as_pct in metrics:
|
||||
x, y = aa[m], ab[m]
|
||||
if as_pct:
|
||||
print(f" {m:14} {100*x:>9.1f}% {100*y:>9.1f}% {100*(y-x):>+9.1f}pp")
|
||||
else:
|
||||
print(f" {m:14} {x:>10.3f} {y:>10.3f} {y-x:>+10.3f}")
|
||||
for m in ("latency_ms_p50", "latency_ms_p95"):
|
||||
x, y = aa[m], ab[m]
|
||||
ratio = f"{y/x:.2f}x" if x else "n/a"
|
||||
print(f" {m:14} {x:>9.0f}ms {y:>9.0f}ms {ratio:>10}")
|
||||
print(f" {'nonsense':14} {aa['nonsense_clean']:>10} {ab['nonsense_clean']:>10}")
|
||||
print(f" {'outranks':14} {aa['superseded_outranks']:>10} {ab['superseded_outranks']:>10}")
|
||||
|
||||
print(f"\n {'category':14} {'n':>3} {'base hit@5':>11} {'cand hit@5':>11} {'delta':>9}")
|
||||
for c in sorted(set(aa["by_category"]) & set(ab["by_category"])):
|
||||
ea, eb = aa["by_category"][c], ab["by_category"][c]
|
||||
if c == "nonsense":
|
||||
print(f" {c:14} {ea['n']:>3} {'clean ' + str(ea['clean']):>11} "
|
||||
f"{'clean ' + str(eb['clean']):>11}")
|
||||
else:
|
||||
print(f" {c:14} {ea['n']:>3} {100*ea['hit@5']:>10.1f}% {100*eb['hit@5']:>10.1f}% "
|
||||
f"{100*(eb['hit@5']-ea['hit@5']):>+8.1f}pp")
|
||||
|
||||
# ---- paired significance -------------------------------------------------
|
||||
fixed, broken = [], []
|
||||
for q in ids:
|
||||
oa, ob = outcome(ra[q]), outcome(rb[q])
|
||||
if ob > oa:
|
||||
fixed.append(q)
|
||||
elif ob < oa:
|
||||
broken.append(q)
|
||||
b, c = len(fixed), len(broken)
|
||||
p = binom_two_sided(b, c)
|
||||
mds = min_detectable_swing(n)
|
||||
|
||||
print(f"\n== paired comparison over {n} queries ==")
|
||||
print(f" fixed by candidate : {b} {[ra[q]['category'] + ':' + q for q in fixed]}")
|
||||
print(f" broken by candidate: {c} {[ra[q]['category'] + ':' + q for q in broken]}")
|
||||
print(f" discordant pairs : {b + c} net {b - c:+d} queries")
|
||||
print(f" McNemar exact p : {p:.4f}")
|
||||
print(f" noise threshold : a difference needs at least {mds} queries moving the "
|
||||
f"same way to clear p<0.05 on this {n}-query set")
|
||||
|
||||
# ---- repeat-run variance -------------------------------------------------
|
||||
var = {}
|
||||
for name, paths, first in (("baseline", args.repeats_baseline, A),
|
||||
("candidate", args.repeats_candidate, B)):
|
||||
docs = [first] + [load(p) for p in paths]
|
||||
if len(docs) > 1:
|
||||
hits = [d["aggregate"]["hit@5"] for d in docs]
|
||||
lo, hi = band(hits)
|
||||
spread_q = round((hi - lo) * first["aggregate"]["n_scored"])
|
||||
var[name] = {"runs": len(docs), "hit@5_min": lo, "hit@5_max": hi,
|
||||
"spread_queries": spread_q}
|
||||
print(f" {name} repeat runs ({len(docs)}): hit@5 {100*lo:.1f}%..{100*hi:.1f}% "
|
||||
f"= {spread_q} query of run-to-run drift")
|
||||
|
||||
drift = max([v["spread_queries"] for v in var.values()], default=0)
|
||||
floor = max(mds, drift + 1)
|
||||
|
||||
print("\n== VERDICT ==")
|
||||
net = b - c
|
||||
if abs(net) < floor:
|
||||
print(f" NO MEASURABLE DIFFERENCE. Net {net:+d} queries is inside the noise band "
|
||||
f"(needs |net| >= {floor}: {mds} for significance, {drift} observed run-to-run drift).")
|
||||
elif net > 0:
|
||||
print(f" CANDIDATE BETTER by {net} queries (p={p:.4f}), outside the noise band "
|
||||
f"(>= {floor}).")
|
||||
else:
|
||||
print(f" CANDIDATE WORSE by {abs(net)} queries (p={p:.4f}), outside the noise band "
|
||||
f"(>= {floor}).")
|
||||
|
||||
if args.out:
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump({
|
||||
"baseline": A["label"], "candidate": B["label"],
|
||||
"n_shared_queries": n,
|
||||
"fixed_by_candidate": fixed, "broken_by_candidate": broken,
|
||||
"discordant": b + c, "net_queries": net,
|
||||
"mcnemar_exact_p": p,
|
||||
"min_detectable_swing_queries": mds,
|
||||
"observed_run_to_run_drift_queries": drift,
|
||||
"noise_floor_queries": floor,
|
||||
"verdict": ("no measurable difference" if abs(net) < floor
|
||||
else ("candidate better" if net > 0 else "candidate worse")),
|
||||
"baseline_aggregate": aa, "candidate_aggregate": ab,
|
||||
"repeat_variance": var,
|
||||
}, fh, indent=1)
|
||||
print(f"\nwrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"baseline": "main-r1",
|
||||
"candidate": "act-r1",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [],
|
||||
"broken_by_candidate": [
|
||||
"q07",
|
||||
"q11",
|
||||
"q12",
|
||||
"q13",
|
||||
"q36"
|
||||
],
|
||||
"discordant": 5,
|
||||
"net_queries": -5,
|
||||
"mcnemar_exact_p": 0.0625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 1,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1140.4,
|
||||
"latency_ms_p95": 1584.1,
|
||||
"latency_ms_max": 1627.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.19087301587301586,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.07428571428571429,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3208.8,
|
||||
"latency_ms_p95": 4851.9,
|
||||
"latency_ms_max": 5078.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.09722222222222222,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 3,
|
||||
"hit@5_min": 0.34285714285714286,
|
||||
"hit@5_max": 0.34285714285714286,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 3,
|
||||
"hit@5_min": 0.22857142857142856,
|
||||
"hit@5_max": 0.2571428571428571,
|
||||
"spread_queries": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
{
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"note": "Every query carries a `derivation` recording how its expected answer was chosen. Re-run with --check to re-validate the whole set against the corpus.",
|
||||
"queries": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"relevant": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"derivation": "MINED: token 'unjailbreakable' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226 ('Daemon hidden substrate architecture ? implemented April 25 '), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"relevant": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"derivation": "MINED: token 'engram-migrate' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5 ('Engram v0.1 complete ? April 27, 2026. Local-first spreading'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"relevant": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"derivation": "MINED: token 'cartabandonedevent' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091 ('MESSAGE FOR AUDIT AGENT af5a7352e70e80434 ? El Language Spec'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"relevant": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"derivation": "MINED: token 'pre-apprenticeship' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-89c02aae-d3ca-43f9-9e5d-eb369896276c ('William Fox Anderson ? Applicant Profile Personal: - Full N'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"relevant": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"derivation": "MINED: token 'inferencenodemanager' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-73969486-143f-4431-b5e6-6845d1cc9848 ('Soma inference backplane deployed April 28 2026. Architectur'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"relevant": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"derivation": "MINED: token 'clear-eyed' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is knw-c72597c5-c23d-4c08-8e9e-996dadf26a99 ('Clear Eyes ? The Incomplete World View'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"relevant": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482"
|
||||
],
|
||||
"derivation": "MINED: a verbatim correction Will issued; expected = every node containing the phrase. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"Kp???",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"derivation": "MINED: the canonical biographical phrase; expected = every node containing it. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 16 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "MINED: a named person appearing verbatim in the biography/value nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"relevant": [
|
||||
"%???2??jH??",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"7c9d4ab1-205d-4be8-bfae-e2c03a3a5010",
|
||||
"9f291d20-0d32-413c-8c01-4416ccab4f7f",
|
||||
"?;????n}rh?",
|
||||
"???Ͼd??f??",
|
||||
"?Z?.\f?0?]P?",
|
||||
"Dp???]Q??k+",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"derivation": "MINED: the canonical DHARMA expansion, confirmed by Will April 24 2026. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "MINED: a named person; rare enough that the answer set is unambiguous. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 2 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"relevant": [
|
||||
"2a923500-d7e1-4b15-80e2-48dba65984ba",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"g?2睪A|?H\b",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de"
|
||||
],
|
||||
"derivation": "MINED: the DARMA expansion, quoted verbatim in the backlog item and its correction. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 14 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"relevant": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"derivation": "MINED: the paid-tier feature name as written in the roadmap nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"relevant": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas dying in Feb 2006 without Will saying goodbye. Query names the event with none of the node's own vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"away",
|
||||
"elderly",
|
||||
"passed",
|
||||
"relative",
|
||||
"stayed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"relevant": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the Marines, a severe hernia, and sepsis. Query describes the episode obliquely. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"illness",
|
||||
"quit",
|
||||
"refused",
|
||||
"sidelined",
|
||||
"soldier"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"relevant": [
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Honesty Before Comfort'. Query states the principle in wholly different words. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"choosing",
|
||||
"fact",
|
||||
"fiction",
|
||||
"pleasant",
|
||||
"uncomfortable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"relevant": [
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Precision Over Brute Force'. Query restates the claim with no shared vocabulary. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"beats",
|
||||
"bloated",
|
||||
"payload",
|
||||
"tight"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"relevant": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the obligation without the node's terms. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"able",
|
||||
"coming",
|
||||
"job",
|
||||
"nobody"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the library following Will across 30+ moves. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"creditors",
|
||||
"learning",
|
||||
"seize",
|
||||
"wealth"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"relevant": [
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Earned Trust' ('Trust is demonstrated, not declared'). VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"assertion",
|
||||
"proven",
|
||||
"record",
|
||||
"reliability",
|
||||
"track"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"relevant": [
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"boundaries",
|
||||
"confine",
|
||||
"enable",
|
||||
"instead"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"relevant": [
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Change Is the Signal', the value VBD is built on. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"apart",
|
||||
"cut",
|
||||
"shifts",
|
||||
"system",
|
||||
"tells"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"relevant": [
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - The System Must Accumulate'. Query is the accumulation claim in different vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"compounds",
|
||||
"day",
|
||||
"instead",
|
||||
"mind",
|
||||
"resetting"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"relevant": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop as the first person Will did not perform for. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"exterior",
|
||||
"loved",
|
||||
"polished",
|
||||
"self",
|
||||
"unedited"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"relevant": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"arrive",
|
||||
"assuming",
|
||||
"cheerfulness",
|
||||
"instead"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"relevant": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between two parents' collapses. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"childhood",
|
||||
"foundation",
|
||||
"inherit",
|
||||
"offering",
|
||||
"solid"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71 ('Value ? Do the Essential Thing While You Can'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-58874a74-b96f-4883-9e08-45707f4bd3ee ('Value ? Survival Is Not an Excuse to Stop'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (13 of 13; 0 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-58874a74-b96f-4883-9e08-45707f4bd3ee) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e ('Value ? Being Seen Is Rarer Than Being Known'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83 ('Value ? Constraints as Freedom'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (6 of 13; 7 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 7
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-e0423482-cfa5-4796-8689-8495c93b66bc ('Value ? Hope Is a Conclusion'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-e0423482-cfa5-4796-8689-8495c93b66bc) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8 ('Value ? Capability Is a Debt You Owe the Moment'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"relevant": [
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); the stale node is the surviving backlog item still titled 'Implement DARMA'. Scored on RANKING, not presence: the corrected node mem-80d7416b-20e9-48a0-b176-b215527e2f56 must be returned AND must rank above the stale node bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff.",
|
||||
"must_outrank": [
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"stale_id": "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"correct_label": "CORRECTION: The autonomous self-improvement architecture is DHARMA ? n",
|
||||
"stale_label": "Implement DARMA ? Directed Autonomous Runtime Modification Architectur"
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"relevant": [
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; the stale node is the surviving memory that asserts 12 filed patents. Scored on RANKING, not presence: the corrected node 3cf706a1-3825-45d8-b0a9-06cae6cdf5b8 must be returned AND must rank above the stale node 936541a9-fabb-466b-9ca3-a78b17ad0c53.",
|
||||
"must_outrank": [
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"936541a9-fabb-466b-9ca3-a78b17ad0c53"
|
||||
],
|
||||
"stale_id": "936541a9-fabb-466b-9ca3-a78b17ad0c53",
|
||||
"correct_label": "memory:remembered",
|
||||
"stale_label": "memory:remembered"
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"relevant": [
|
||||
"mem-30425134-6008-4fd9-a3ee-67a7742c319b"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of April 30 2026; the stale node still records the MCP server as live. Scored on RANKING, not presence: the corrected node mem-30425134-6008-4fd9-a3ee-67a7742c319b must be returned AND must rank above the stale node mem-101e81b4-8097-4749-8d8d-7bb66de34517.",
|
||||
"must_outrank": [
|
||||
"mem-30425134-6008-4fd9-a3ee-67a7742c319b",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517"
|
||||
],
|
||||
"stale_id": "mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"correct_label": "CGI ARCHITECTURE ? THREE LAYERS, MCP RETIRED (April 30, 2026). Definit",
|
||||
"stale_label": "GCloud MCP infrastructure ? April 27, 2026. Legion died (~19:30 UTC). "
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,942 @@
|
||||
{
|
||||
"label": "act-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7902,
|
||||
"wall_clock_s": 121.2,
|
||||
"child_pid": 78714,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.18908730158730158,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.06857142857142857,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3237.6,
|
||||
"latency_ms_p95": 5264.0,
|
||||
"latency_ms_max": 5510.9,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.0882936507936508,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 476.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 708.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 532.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 537.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 568.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 555.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-c3d9d063-8c5d-45aa-900c-550914b2ff6d",
|
||||
"kn-f838f113-76d5-4a15-9cef-14055c4723a3",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1906.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1103.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.0625,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 1132.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-c9adb8e5-293f-4033-99f8-0405c17ef941",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"mem-34f53a9d-a131-4f82-9dbd-b9eb4a9af52e",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 960.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
";??A5???",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-57164d5f-baf0-4149-957a-379a4e255d1a",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 992.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"art-92e1837c-5919-42d0-bbb0-4d924d7b2864",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"mem-f823e835-313f-4282-b4b3-ce527ffc2f7a",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2820.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.14285714285714285,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-e20944e5-eb16-4ab3-a84d-111e0fc817fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2609.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"tag-dark-theme",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5510.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
";??A5???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5444.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
";??A5???",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5056.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
";??A5???",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3473.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"tag-stable-diffusion",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4446.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3932.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"tag-provenance",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4836.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4057.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-identity-studio",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"tag-design-first",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5098.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-performed-values",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
";??A5???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4652.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4653.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
";??A5???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4272.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4541.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-dbe80bc2-c602-46b0-b4ea-dd222e52bcde",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4883.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3237.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2457.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2640.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3705.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-394cc9e8-049b-45bc-a380-66314f14e367",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-command-pattern",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4223.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1333.9,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 880.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1452.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"ctx-e5427d7d",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3843.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"mem-f9da4b43-3724-4bc8-92f8-6f237c89dc4d",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
";??A5???",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5264.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"project-Define_Iris_as_separate_public_brand__Uncensored_but_principled__Consumer_face_while_Neuron_runs_enterprise_",
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"tag-enterprise",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"???Ͼd??W\b?",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3022.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,942 @@
|
||||
{
|
||||
"label": "act-r3",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7903,
|
||||
"wall_clock_s": 112.0,
|
||||
"child_pid": 78802,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.2571428571428571,
|
||||
"recall@5": 0.19291383219954647,
|
||||
"recall@10": 0.244812925170068,
|
||||
"precision@5": 0.08,
|
||||
"mrr@10": 0.266031746031746,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3220.9,
|
||||
"latency_ms_p95": 4840.2,
|
||||
"latency_ms_max": 5073.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.42857142857142855,
|
||||
"recall@5": 0.10742630385487528,
|
||||
"recall@10": 0.366921768707483,
|
||||
"mrr@10": 0.47301587301587306
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 471.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 548.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 470.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 476.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 515.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 458.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-c3d9d063-8c5d-45aa-900c-550914b2ff6d",
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-9887867c-2e21-47c3-9f96-c2dfe5bd4cc1",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1595.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 927.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.125,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 963.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"bl-455a08cf-5831-4fdb-b42c-b952f2feafb9",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 927.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-57164d5f-baf0-4149-957a-379a4e255d1a",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 937.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2451.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07142857142857142,
|
||||
"recall@10": 0.21428571428571427,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-64cf3728-674c-404b-965a-b8f8d38bb7bb",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-e612f0aa-c2f2-4ee3-bbc7-af2dc826233b",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2042.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865",
|
||||
"bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4698.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"bl-fd047ce9-ae21-4b3e-b3ab-ece0c9592f7f",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5073.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4117.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3254.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"tag-stable-diffusion",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4081.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3687.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"? ?}&?#??X\b",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4659.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3960.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-identity-studio",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"tag-design-first",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4840.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-performed-values",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4493.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4509.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3624.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4093.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4235.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"mem-0f31141d-3ac5-44b2-9942-be7e4e6feb79",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3220.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"mem-5e7f6ddd-c818-4ad3-b564-54ae278e9976",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"tag-sarah",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2336.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2348.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"tag-dark-theme",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3511.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"bl-452a4710-3d2b-4e0f-9413-49a66423bc9a",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4160.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1318.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 875.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1393.4,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"ctx-e5427d7d",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-enterprise",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3775.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"mem-f9da4b43-3724-4bc8-92f8-6f237c89dc4d",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4857.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"???Ͼd??W\b?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2858.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,942 @@
|
||||
{
|
||||
"label": "act-r1",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 112.3,
|
||||
"child_pid": 78648,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.19087301587301586,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.07428571428571429,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3208.8,
|
||||
"latency_ms_p95": 4851.9,
|
||||
"latency_ms_max": 5078.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.09722222222222222,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 468.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 569.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 479.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 470.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 499.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 452.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1592.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 956.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.125,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 970.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 920.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 925.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"mem-f823e835-313f-4282-b4b3-ce527ffc2f7a",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2455.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.14285714285714285,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"art-92e1837c-5919-42d0-bbb0-4d924d7b2864",
|
||||
"bl-e20944e5-eb16-4ab3-a84d-111e0fc817fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-e612f0aa-c2f2-4ee3-bbc7-af2dc826233b",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2037.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"bl-874d1c2b-c55b-4afb-9601-922a9297e859",
|
||||
"bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4720.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-fd047ce9-ae21-4b3e-b3ab-ece0c9592f7f",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5078.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4118.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"tag-identity-studio",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3241.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"tag-stable-diffusion",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4090.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3671.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"tag-provenance",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4674.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3974.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4851.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-design-first",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4505.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"tag-performed-values",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4494.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3619.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4107.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-dbe80bc2-c602-46b0-b4ea-dd222e52bcde",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4231.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3208.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"mem-5e7f6ddd-c818-4ad3-b564-54ae278e9976",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"tag-sarah",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2359.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2342.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3536.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"bl-452a4710-3d2b-4e0f-9413-49a66423bc9a",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-command-pattern",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4151.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1342.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 874.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1374.6,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"ctx-e5427d7d",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-enterprise",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3762.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4859.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"tag-dark-theme",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"???Ͼd??W\b?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2873.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,945 @@
|
||||
{
|
||||
"label": "main-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 45.6,
|
||||
"child_pid": 78695,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1151.1,
|
||||
"latency_ms_p95": 1580.7,
|
||||
"latency_ms_max": 1663.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 266.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 235.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 254.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 228.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 466.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 474.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 460.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 477.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 824.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 681.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1577.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1663.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1331.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1036.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1363.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1230.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1548.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1315.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1580.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"????7???Ջ3",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"%???2??jH??",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1474.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1490.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1192.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1320.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1448.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1107.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1151.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1154.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1187.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1413.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 684.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 444.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 696.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1617.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 967.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,945 @@
|
||||
{
|
||||
"label": "main-r3",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7896,
|
||||
"wall_clock_s": 45.7,
|
||||
"child_pid": 78750,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1141.7,
|
||||
"latency_ms_p95": 1578.0,
|
||||
"latency_ms_max": 1645.5,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 237.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 272.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 262.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 239.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 465.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 470.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 441.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 455.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 816.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 678.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"????7???Ջ3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1564.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1645.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1320.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1029.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1347.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1220.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1560.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1305.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1578.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"????7???Ջ3",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"?ǚ?7??????",
|
||||
"7?e?7???\f3?",
|
||||
"??f?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1449.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1489.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1186.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1309.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1427.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1081.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1144.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1141.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1171.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1388.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 659.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 432.5,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 673.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1254.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1626.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 950.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,945 @@
|
||||
{
|
||||
"label": "main-r1",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 45.9,
|
||||
"child_pid": 78554,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1140.4,
|
||||
"latency_ms_p95": 1584.1,
|
||||
"latency_ms_max": 1627.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 240.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 271.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 232.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 227.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 259.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 237.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 467.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 475.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 449.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 463.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 821.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 688.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"????7???Ջ3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1559.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1627.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1317.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1055.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1352.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1213.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1545.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1327.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1584.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"????7???Ջ3",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"?ǚ?7??????",
|
||||
"7?e?7???\f3?",
|
||||
"??f?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1432.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1505.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1201.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1303.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1416.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1090.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1140.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1157.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1379.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 677.2,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 443.2,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 679.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1618.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 956.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# run_comparison.sh — the whole harness, end to end, from two git refs.
|
||||
#
|
||||
# Builds a soul from each ref, boots each on its own throwaway port with its own
|
||||
# throwaway HOME and its own disposable copy of the corpus, runs the gold set N
|
||||
# times per ref, and prints the comparison with its noise threshold.
|
||||
#
|
||||
# SAFETY: never touches ~/.neuron, /Applications/Neuron*, ~/neuron-dev-stack, or
|
||||
# any running service. Sources are exported with `git archive` into a scratch
|
||||
# dir, so no worktree or branch state is mutated either. Ports are checked
|
||||
# against the live set before anything boots. Every soul this script starts is
|
||||
# killed and confirmed dead by run_eval.py; the sweep at the end is a backstop.
|
||||
#
|
||||
# usage:
|
||||
# run_comparison.sh [--baseline main] [--candidate feat/recall-through-activation]
|
||||
# [--repeats 3] [--corpus <snapshot.json>] [--repo <path>]
|
||||
set -euo pipefail
|
||||
|
||||
BASELINE="main"
|
||||
CANDIDATE="feat/recall-through-activation"
|
||||
REPEATS=3
|
||||
CORPUS="$HOME/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
REPO="$HOME/Development/neuron"
|
||||
BASE_PORT=7893
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--baseline) BASELINE="$2"; shift 2 ;;
|
||||
--candidate) CANDIDATE="$2"; shift 2 ;;
|
||||
--repeats) REPEATS="$2"; shift 2 ;;
|
||||
--corpus) CORPUS="$2"; shift 2 ;;
|
||||
--repo) REPO="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORK="$(mktemp -d "${TMPDIR:-/tmp}/retrieval-eval.XXXXXX")"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
[ -f "$CORPUS" ] || { echo "no corpus at $CORPUS" >&2; exit 2; }
|
||||
echo "corpus: $CORPUS ($(du -h "$CORPUS" | cut -f1))"
|
||||
|
||||
slug() { printf '%s' "$1" | tr '/' '-'; }
|
||||
|
||||
build_ref() { # ref -> binary path
|
||||
local ref="$1" out="$WORK/soul-$(slug "$1")"
|
||||
local src="$WORK/src-$(slug "$1")"
|
||||
mkdir -p "$src"
|
||||
git -C "$REPO" archive "$ref" | tar -x -C "$src"
|
||||
"$HERE/build-soul.sh" "$src" "$out" >&2
|
||||
printf '%s' "$out"
|
||||
}
|
||||
|
||||
echo "== building $BASELINE =="
|
||||
BIN_A="$(build_ref "$BASELINE")"
|
||||
echo "== building $CANDIDATE =="
|
||||
BIN_B="$(build_ref "$CANDIDATE")"
|
||||
|
||||
echo "== validating the gold set against this corpus =="
|
||||
python3 "$HERE/build_gold_set.py" "$CORPUS" --check
|
||||
|
||||
port=$BASE_PORT
|
||||
run_one() { # binary label out
|
||||
echo "== $2 =="
|
||||
python3 "$HERE/run_eval.py" --soul "$1" --corpus "$CORPUS" --label "$2" \
|
||||
--port "$port" --out "$3"
|
||||
port=$((port + 1))
|
||||
}
|
||||
|
||||
A_MAIN="$WORK/results-a-1.json"; B_MAIN="$WORK/results-b-1.json"
|
||||
A_REP=(); B_REP=()
|
||||
for i in $(seq 1 "$REPEATS"); do
|
||||
a="$WORK/results-a-$i.json"; b="$WORK/results-b-$i.json"
|
||||
run_one "$BIN_A" "$(slug "$BASELINE")-r$i" "$a"
|
||||
run_one "$BIN_B" "$(slug "$CANDIDATE")-r$i" "$b"
|
||||
[ "$i" -gt 1 ] && { A_REP+=("$a"); B_REP+=("$b"); }
|
||||
done
|
||||
|
||||
cp "$A_MAIN" "$HERE/results-$(slug "$BASELINE").json"
|
||||
cp "$B_MAIN" "$HERE/results-$(slug "$CANDIDATE").json"
|
||||
|
||||
echo
|
||||
python3 "$HERE/compare.py" \
|
||||
--baseline "$A_MAIN" --candidate "$B_MAIN" \
|
||||
${A_REP[@]+--repeats-baseline "${A_REP[@]}"} \
|
||||
${B_REP[@]+--repeats-candidate "${B_REP[@]}"} \
|
||||
--out "$HERE/comparison-$(slug "$BASELINE")-vs-$(slug "$CANDIDATE").json"
|
||||
|
||||
# Backstop: run_eval.py kills and confirms its own child, but a crashed run
|
||||
# could leak one. Leaving a soul running is how the live engine got squeezed.
|
||||
STRAY=$(pgrep -f "$WORK/soul-" || true)
|
||||
if [ -n "$STRAY" ]; then
|
||||
echo "!! stray eval souls, killing: $STRAY" >&2
|
||||
kill -9 $STRAY 2>/dev/null || true
|
||||
fi
|
||||
pgrep -f "$WORK/soul-" >/dev/null && { echo "!! STILL RUNNING" >&2; exit 5; }
|
||||
echo "process check: no eval souls running"
|
||||
Executable
+398
@@ -0,0 +1,398 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
run_eval.py — measure one soul build's retrieval against the gold set.
|
||||
|
||||
WHAT IT MEASURES, AND WHY IT BOOTS A REAL SOUL
|
||||
The point is Will's designed retrieval — spreading activation over the
|
||||
weighted directed graph with four-factor multiplicative scoring — not a
|
||||
Python re-implementation of it. A re-implementation would measure my
|
||||
reading of the design; booting the compiled binary measures the design. So
|
||||
this harness compiles the actual `soul.el` amalgam (build-soul.sh) and asks
|
||||
it over HTTP, exactly as the MCP wrapper and the app do.
|
||||
|
||||
SAFETY — read this before changing anything here
|
||||
* Boots on a THROWAWAY port with a THROWAWAY $HOME and a THROWAWAY COPY of
|
||||
the corpus. Refuses to use 7770 / 8742 / 7779 / 17779 / 7771.
|
||||
* ENGRAM_URL / SOUL_ENGRAM_URL are UNSET and SOUL_ISE_URL is pinned to a
|
||||
dead port. This is not belt-and-braces: the periodic engram sync resolves
|
||||
its source as env(SOUL_ISE_URL) -> state -> DEFAULT http://localhost:8742,
|
||||
so leaving it unset makes an "isolated" run silently pull the operator's
|
||||
LIVE brain. (Learned the hard way on 2026-08-03; see the same note in
|
||||
scripts/verify-soul-contract.sh.)
|
||||
* Every process this file starts is tracked and killed in a finally block,
|
||||
then CONFIRMED dead by pid probe, and the confirmation is written into the
|
||||
results file. A run that cannot confirm its child is dead exits non-zero.
|
||||
* Activation is a STATEFUL read by design (patent claim 29: traversal
|
||||
updates last-activation and increments activation counts). The corpus copy
|
||||
is therefore per-run and disposable, and every run starts from a byte-
|
||||
identical copy so two configurations see the same starting graph.
|
||||
|
||||
usage:
|
||||
python3 run_eval.py --soul <binary> --corpus <snapshot.json> --label main \
|
||||
[--port 7893] [--gold gold_set.json] [--limit 10] [--out results-main.json]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
FORBIDDEN_PORTS = {7770, 8742, 7779, 17779, 7771, 8080}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# metrics
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def recall_at_k(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
return len(set(returned[:k]) & set(relevant)) / len(relevant)
|
||||
|
||||
|
||||
def hit_at_k(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
return 1.0 if set(returned[:k]) & set(relevant) else 0.0
|
||||
|
||||
|
||||
def precision_at_k(returned, relevant, k):
|
||||
"""Fixed denominator k, as in docs/research/graphrag_eval/score.py.
|
||||
|
||||
Fixed denominator penalises an empty result and a page of junk equally,
|
||||
which is what we want: a retriever that returns nothing is not 'precise'.
|
||||
"""
|
||||
if not relevant:
|
||||
return None
|
||||
return len(set(returned[:k]) & set(relevant)) / k
|
||||
|
||||
|
||||
def mrr(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
rel = set(relevant)
|
||||
for i, nid in enumerate(returned[:k], start=1):
|
||||
if nid in rel:
|
||||
return 1.0 / i
|
||||
return 0.0
|
||||
|
||||
|
||||
def mean(vals):
|
||||
vals = [v for v in vals if v is not None]
|
||||
return sum(vals) / len(vals) if vals else 0.0
|
||||
|
||||
|
||||
def pct(vals):
|
||||
return f"{100 * mean(vals):.1f}%"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# soul lifecycle
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
class Soul:
|
||||
def __init__(self, binary, corpus, port, verbose=True):
|
||||
if port in FORBIDDEN_PORTS:
|
||||
raise SystemExit(f"REFUSING: port {port} is a live service port.")
|
||||
self.binary = os.path.abspath(binary)
|
||||
self.corpus = os.path.abspath(corpus)
|
||||
self.port = port
|
||||
self.verbose = verbose
|
||||
self.home = None
|
||||
self.proc = None
|
||||
self.pid = None
|
||||
self.log = None
|
||||
self.confirmed_dead = None
|
||||
|
||||
@property
|
||||
def base(self):
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def start(self, boot_timeout=180):
|
||||
self.home = tempfile.mkdtemp(prefix="retrieval-eval-home.")
|
||||
snap = os.path.join(self.home, "corpus.json")
|
||||
t0 = time.time()
|
||||
shutil.copyfile(self.corpus, snap) # per-run disposable copy, never the source
|
||||
self.log = os.path.join(self.home, "soul.log")
|
||||
|
||||
env = {k: v for k, v in os.environ.items()
|
||||
if k not in ("ENGRAM_URL", "ENGRAM_API_KEY", "SOUL_ENGRAM_URL",
|
||||
"ANTHROPIC_API_KEY", "NEURON_LLM_API_KEY", "SOUL_IDENTITY",
|
||||
"SOUL_API_KEY")}
|
||||
env.update({
|
||||
"HOME": self.home,
|
||||
"NEURON_PORT": str(self.port),
|
||||
"SOUL_CGI_ID": f"ntn-retrieval-eval-{os.getpid()}",
|
||||
"SOUL_ENGRAM_PATH": snap,
|
||||
"NEURON_API_URL": "http://127.0.0.1:9", # dead port
|
||||
"SOUL_ISE_URL": "http://127.0.0.1:9", # dead port — see SAFETY above
|
||||
# Park the background loops for an hour so heartbeat/consolidation
|
||||
# cannot mutate the graph between queries and make runs unrepeatable.
|
||||
"SOUL_TICK_MS": "3600000",
|
||||
"SOUL_HEARTBEAT_MS": "3600000",
|
||||
"SOUL_REFRESH_MS": "3600000",
|
||||
})
|
||||
with open(self.log, "wb") as lf:
|
||||
self.proc = subprocess.Popen([self.binary], env=env, stdout=lf, stderr=lf,
|
||||
start_new_session=True)
|
||||
self.pid = self.proc.pid
|
||||
if self.verbose:
|
||||
print(f" booted pid={self.pid} port={self.port} home={self.home}")
|
||||
|
||||
deadline = time.time() + boot_timeout
|
||||
while time.time() < deadline:
|
||||
if self.proc.poll() is not None:
|
||||
raise RuntimeError(f"soul exited during boot: {self._log_tail()}")
|
||||
rss = self._rss_kb()
|
||||
if rss and rss > 6 * 1024 * 1024:
|
||||
self.stop()
|
||||
raise RuntimeError(f"soul RSS {rss}KB > 6GB — aborted")
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self.base}/health", timeout=2) as r:
|
||||
if r.status == 200:
|
||||
if self.verbose:
|
||||
print(f" healthy in {time.time() - t0:.1f}s, RSS={self._rss_kb()}KB")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
self.stop()
|
||||
raise RuntimeError(f"soul never healthy on {self.base}: {self._log_tail()}")
|
||||
|
||||
def _rss_kb(self):
|
||||
try:
|
||||
out = subprocess.run(["ps", "-o", "rss=", "-p", str(self.pid)],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip()
|
||||
return int(out) if out else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _log_tail(self, n=15):
|
||||
try:
|
||||
with open(self.log, encoding="utf-8", errors="replace") as fh:
|
||||
return "\n".join(fh.read().splitlines()[-n:])
|
||||
except Exception:
|
||||
return "(no log)"
|
||||
|
||||
def recall(self, query, limit, timeout=60):
|
||||
url = f"{self.base}/api/neuron/recall?query={urllib.parse.quote(query)}&limit={limit}"
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
raw = r.read().decode("utf-8", "replace")
|
||||
ms = (time.perf_counter() - t0) * 1000
|
||||
except Exception as exc:
|
||||
return [], (time.perf_counter() - t0) * 1000, f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
arr = json.loads(raw)
|
||||
except Exception:
|
||||
return [], ms, f"unparseable response ({len(raw)}B)"
|
||||
if not isinstance(arr, list):
|
||||
return [], ms, f"non-array response: {str(arr)[:120]}"
|
||||
ids = [x.get("id") for x in arr if isinstance(x, dict) and x.get("id")]
|
||||
return ids, ms, None
|
||||
|
||||
def stop(self):
|
||||
"""Kill and CONFIRM. A test process that outlives its test is a bug."""
|
||||
if self.pid is None:
|
||||
self.confirmed_dead = True
|
||||
return True
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
os.kill(self.pid, sig)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
for _ in range(20):
|
||||
try:
|
||||
os.kill(self.pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
continue
|
||||
break
|
||||
try:
|
||||
self.proc.wait(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.kill(self.pid, 0)
|
||||
self.confirmed_dead = False
|
||||
except ProcessLookupError:
|
||||
self.confirmed_dead = True
|
||||
if self.verbose:
|
||||
print(f" pid {self.pid}: {'CONFIRMED DEAD' if self.confirmed_dead else 'STILL ALIVE'}")
|
||||
if self.home and os.path.isdir(self.home):
|
||||
shutil.rmtree(self.home, ignore_errors=True)
|
||||
return self.confirmed_dead
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# eval
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def evaluate(soul, gold, limit):
|
||||
rows = []
|
||||
for q in gold["queries"]:
|
||||
ids, ms, err = soul.recall(q["query"], limit)
|
||||
rel = q.get("relevant") or []
|
||||
row = {
|
||||
"id": q["id"],
|
||||
"category": q["category"],
|
||||
"query": q["query"],
|
||||
"returned": ids,
|
||||
"n_returned": len(ids),
|
||||
"latency_ms": round(ms, 1),
|
||||
"error": err,
|
||||
}
|
||||
if q.get("expect_empty"):
|
||||
row["clean"] = (len(ids) == 0)
|
||||
row["false_positives"] = len(ids)
|
||||
else:
|
||||
row["hit@5"] = hit_at_k(ids, rel, 5)
|
||||
row["recall@5"] = recall_at_k(ids, rel, 5)
|
||||
row["recall@10"] = recall_at_k(ids, rel, 10)
|
||||
row["precision@5"] = precision_at_k(ids, rel, 5)
|
||||
row["mrr@10"] = mrr(ids, rel, 10)
|
||||
if q.get("must_outrank"):
|
||||
correct, stale = q["must_outrank"]
|
||||
ic = ids.index(correct) if correct in ids else None
|
||||
istale = ids.index(stale) if stale in ids else None
|
||||
# Correct must be present AND above the stale node. A run that
|
||||
# returns neither is NOT a pass: the corrected fact is what the
|
||||
# user needed.
|
||||
row["outranks"] = (ic is not None) and (istale is None or ic < istale)
|
||||
row["rank_correct"] = None if ic is None else ic + 1
|
||||
row["rank_stale"] = None if istale is None else istale + 1
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def aggregate(rows):
|
||||
scored = [r for r in rows if "hit@5" in r]
|
||||
nonsense = [r for r in rows if "clean" in r]
|
||||
outrank = [r for r in rows if "outranks" in r]
|
||||
lat = sorted(r["latency_ms"] for r in rows)
|
||||
agg = {
|
||||
"n_queries": len(rows),
|
||||
"n_scored": len(scored),
|
||||
"hit@5": mean([r["hit@5"] for r in scored]),
|
||||
"recall@5": mean([r["recall@5"] for r in scored]),
|
||||
"recall@10": mean([r["recall@10"] for r in scored]),
|
||||
"precision@5": mean([r["precision@5"] for r in scored]),
|
||||
"mrr@10": mean([r["mrr@10"] for r in scored]),
|
||||
"nonsense_clean": f"{sum(1 for r in nonsense if r['clean'])}/{len(nonsense)}",
|
||||
"superseded_outranks": f"{sum(1 for r in outrank if r['outranks'])}/{len(outrank)}",
|
||||
"latency_ms_p50": lat[len(lat) // 2] if lat else 0,
|
||||
"latency_ms_p95": lat[max(0, int(len(lat) * 0.95) - 1)] if lat else 0,
|
||||
"latency_ms_max": lat[-1] if lat else 0,
|
||||
"errors": sum(1 for r in rows if r["error"]),
|
||||
"by_category": {},
|
||||
}
|
||||
cats = sorted({r["category"] for r in rows})
|
||||
for c in cats:
|
||||
cr = [r for r in rows if r["category"] == c]
|
||||
if c == "nonsense":
|
||||
agg["by_category"][c] = {
|
||||
"n": len(cr),
|
||||
"clean": sum(1 for r in cr if r["clean"]),
|
||||
"avg_false_positives": mean([float(r["false_positives"]) for r in cr]),
|
||||
}
|
||||
else:
|
||||
e = {
|
||||
"n": len(cr),
|
||||
"hit@5": mean([r.get("hit@5") for r in cr]),
|
||||
"recall@5": mean([r.get("recall@5") for r in cr]),
|
||||
"recall@10": mean([r.get("recall@10") for r in cr]),
|
||||
"mrr@10": mean([r.get("mrr@10") for r in cr]),
|
||||
}
|
||||
if c == "superseded":
|
||||
e["outranks"] = sum(1 for r in cr if r.get("outranks"))
|
||||
agg["by_category"][c] = e
|
||||
return agg
|
||||
|
||||
|
||||
def print_table(label, agg):
|
||||
print(f"\n=== {label} ===")
|
||||
print(f" queries {agg['n_queries']} ({agg['n_scored']} scored + "
|
||||
f"{agg['n_queries'] - agg['n_scored']} control) · errors {agg['errors']}")
|
||||
print(f" {'hit@5':>12} {'recall@5':>10} {'recall@10':>10} {'prec@5':>9} {'MRR@10':>9}")
|
||||
print(f" {pct([agg['hit@5']]):>12} {pct([agg['recall@5']]):>10} {pct([agg['recall@10']]):>10} "
|
||||
f"{pct([agg['precision@5']]):>9} {agg['mrr@10']:>9.3f}")
|
||||
print(f" nonsense clean {agg['nonsense_clean']} · superseded outranks {agg['superseded_outranks']}")
|
||||
print(f" latency ms p50 {agg['latency_ms_p50']:.0f} · p95 {agg['latency_ms_p95']:.0f} "
|
||||
f"· max {agg['latency_ms_max']:.0f}")
|
||||
print(f"\n {'category':14} {'n':>3} {'hit@5':>8} {'recall@5':>9} {'recall@10':>10} {'MRR@10':>8}")
|
||||
for c, e in agg["by_category"].items():
|
||||
if c == "nonsense":
|
||||
print(f" {c:14} {e['n']:>3} {'clean ' + str(e['clean']) + '/' + str(e['n']):>8}"
|
||||
f"{'':>9} {'':>10} {'avg FP ' + format(e['avg_false_positives'], '.1f'):>8}")
|
||||
else:
|
||||
extra = f" outranks {e['outranks']}/{e['n']}" if "outranks" in e else ""
|
||||
print(f" {c:14} {e['n']:>3} {pct([e['hit@5']]):>8} {pct([e['recall@5']]):>9} "
|
||||
f"{pct([e['recall@10']]):>10} {e['mrr@10']:>8.3f}{extra}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--soul", required=True)
|
||||
ap.add_argument("--corpus", required=True)
|
||||
ap.add_argument("--label", required=True)
|
||||
ap.add_argument("--gold", default=os.path.join(HERE, "gold_set.json"))
|
||||
ap.add_argument("--port", type=int, default=7893)
|
||||
ap.add_argument("--limit", type=int, default=10)
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.gold, encoding="utf-8") as fh:
|
||||
gold = json.load(fh)
|
||||
print(f"[{args.label}] soul={os.path.basename(args.soul)} "
|
||||
f"corpus={os.path.basename(args.corpus)} gold={len(gold['queries'])}q limit={args.limit}")
|
||||
|
||||
soul = Soul(args.soul, args.corpus, args.port)
|
||||
rows = []
|
||||
started = time.time()
|
||||
try:
|
||||
soul.start()
|
||||
rows = evaluate(soul, gold, args.limit)
|
||||
finally:
|
||||
dead = soul.stop()
|
||||
|
||||
agg = aggregate(rows)
|
||||
print_table(args.label, agg)
|
||||
|
||||
out = args.out or os.path.join(HERE, f"results-{args.label}.json")
|
||||
doc = {
|
||||
"label": args.label,
|
||||
"soul_binary": os.path.abspath(args.soul),
|
||||
"soul_md5": subprocess.run(["md5", "-q", args.soul], capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"corpus": os.path.abspath(args.corpus),
|
||||
"corpus_nodes": gold.get("corpus_nodes"),
|
||||
"corpus_edges": gold.get("corpus_edges"),
|
||||
"gold_set": os.path.abspath(args.gold),
|
||||
"limit": args.limit,
|
||||
"port": args.port,
|
||||
"wall_clock_s": round(time.time() - started, 1),
|
||||
"child_pid": soul.pid,
|
||||
"child_confirmed_dead": soul.confirmed_dead,
|
||||
"aggregate": agg,
|
||||
"rows": rows,
|
||||
}
|
||||
with open(out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {out}")
|
||||
if not dead:
|
||||
print("FATAL: child process could not be confirmed dead", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user