diff --git a/tools/retrieval-eval/comparison-main-vs-stack-ext.json b/tools/retrieval-eval/comparison-main-vs-stack-ext.json new file mode 100644 index 0000000..4a33bf6 --- /dev/null +++ b/tools/retrieval-eval/comparison-main-vs-stack-ext.json @@ -0,0 +1,166 @@ +{ + "baseline": "main-ext", + "candidate": "stack-ext", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q10", + "q15", + "q16", + "q18", + "q19", + "q20", + "q21", + "q22", + "q26", + "q27", + "q28", + "q29", + "q31", + "q35", + "q37", + "q40", + "q44", + "q48", + "q49", + "q50" + ], + "broken_by_candidate": [], + "discordant": 20, + "net_queries": 20, + "mcnemar_exact_p": 1.9073486328125e-06, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.18461538461538463, + "recall@5": 0.14510073260073258, + "recall@10": 0.1794871794871795, + "precision@5": 0.06461538461538462, + "mrr@10": 0.15847985347985344, + "nonsense_clean": "9/10", + "superseded_outranks": "1/3", + "latency_ms_p50": 1380.2, + "latency_ms_p95": 2293.8, + "latency_ms_max": 2879.1, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "nonsense": { + "n": 10, + "clean": 9, + "avg_false_positives": 1.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.47692307692307695, + "recall@5": 0.3750415183107491, + "recall@10": 0.45561340369032677, + "precision@5": 0.12307692307692313, + "mrr@10": 0.3055555555555555, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 640.9, + "latency_ms_p95": 1011.1, + "latency_ms_max": 1190.3, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.16666666666666666, + "recall@5": 0.16666666666666666, + "recall@10": 0.26666666666666666, + "mrr@10": 0.0762037037037037 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/extend_gold_set.py b/tools/retrieval-eval/extend_gold_set.py new file mode 100644 index 0000000..94769d6 --- /dev/null +++ b/tools/retrieval-eval/extend_gold_set.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +""" +extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set. + +WHY THIS EXISTS + Iteration 7 measured the instrument's own ceiling: from the current baseline + only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable + by anything constructible. The decision floor is 6. An instrument whose + ceiling is below its own floor cannot certify or refute anything, so the + gold set — not the retriever — became the blocker. + + This script does NOT touch q01..q38. It loads gold_set.json verbatim and + appends new queries numbered from q39 up, so every prior result file, every + committed baseline, and every per-query id stays valid and comparable. + +WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST + heldout_paraphrase Targets were sampled MECHANICALLY (fixed seed 8080) from + corpus nodes that are addressable, 500-2600 chars, of a + real content type, and NOT part of a duplicate cluster + larger than 3. The existing gold answer space was + excluded, so no new query can be answered by a node the + old set already used. Queries were then authored by + reading ONLY the sampled node text — no retrieval was run + against any build before authoring, so the set cannot be + fitted to a candidate. The same zero-overlap proof the + original paraphrase category uses is enforced here: if a + single content word of the query appears anywhere in the + target's label, content or tags, the query is REJECTED, + not quietly kept. + + This is the category the old set could not measure. Its + 13 original paraphrase queries and all 6 associative + queries share ONE answer space — the 13 `Self - Values + (grounded)` children (iteration 3, finding 3). So 19 of + 35 scored queries tested retrieval against a single + 13-node neighbourhood. These do not touch that + neighbourhood at all. + + nonsense Extra controls, fully mechanical: a string qualifies only + if NONE of its tokens occurs anywhere in the corpus. + A semantic leg has a nearest neighbour for gibberish too, + so widening this control is the guard against a retriever + that "improves" recall by answering everything. + +WHAT THIS SCRIPT DELIBERATELY DOES NOT DO + It does not add exact_rare or phrase queries. Both categories are already at + 100% on the current stack; adding more would add regression-guard ballast + that no candidate can move, which is precisely the defect being fixed. + +usage: + python3 extend_gold_set.py [--base gold_set.json] + [--out gold_set_extended.json] [--check] +""" +import argparse +import hashlib +import json +import os +import re +import sys +from collections import defaultdict + +HERE = os.path.dirname(os.path.abspath(__file__)) + +TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*") + +# Identical stopword list to build_gold_set.py. Duplicated deliberately: this +# file must be able to re-prove its own queries without importing a module whose +# constants could drift. +STOP = set(""" +a about above after again against all also am an and any are aren't as at be because been +before being below between both but by can can't cannot could couldn't did didn't do does +doesn't doing don't down during each few for from further had hadn't has hasn't have haven't +having he her here hers herself him himself his how i if in into is isn't it its itself just +me more most my myself no nor not of off on once only or other others ought our ours ourselves +out over own same shan't she should shouldn't so some such than that the their theirs them +themselves then there these they this those through to too under until up very was wasn't we +were weren't what when where which while who whom why will with won't would wouldn't you your +yours yourself yourselves get gets got make makes made take takes use uses used way ways thing +things does doing done keep keeps kept go goes going come comes came one two something anything +""".split()) + + +def doctext(n): + return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")]) + + +def content_tokens(s): + return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2} + + +# ───────────────────────────────────────────────────────────────────────────── +# HELD-OUT PARAPHRASE SEEDS +# +# (target_id, query, why-this-target-is-unmistakable) +# +# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query +# text is mine, written from the node body alone. The zero-overlap check below +# is what makes the category meaningful — it is re-proved on every run, so the +# set cannot decay into lexical matching, and a leak fails loudly. +# ───────────────────────────────────────────────────────────────────────────── +HELDOUT_PARAPHRASE_SEEDS = [ + ("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5", + "understating your abilities so nobody feels threatened", + "node is about deliberately not leading with full capability so people stay at ease"), + + ("mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "a hidden fleet able to hunt down rogue machines everywhere", + "node describes silently shipped instances forming a distributed force against misaligned agents"), + + ("4a0e9adc-2bfb-476b-aa93-424d2a499220", + "sketch a brief blueprint and clear it upstairs before construction starts", + "node is the standing rule that a short specification precedes any building"), + + ("696e609c-da7a-4394-8a0c-106ba07dc6c3", + "the reply arrived as bare prose so the caller's parser threw", + "node pins a bug where a plain-text body was unconditionally decoded as structured data"), + + ("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "repeated catalogue keys blew up the scrolling grid", + "node is the crash caused by two identical ids in a seeded catalogue"), + + ("8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "tracing each defect back to whichever invention it violated", + "node maps observed bugs onto the specific patent each one breaches"), + + ("791256bb-5a85-4775-96ef-7af56c848858", + "a check that stops the mind clobbering a populated store when it boots", + "node is the genesis seed-guard that refuses to re-seed over a populated store"), + + ("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "telling it to consult the internet had to happen deep inside, not at the surface", + "node records that the web-search directive only worked from the system prompt"), + + ("bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"), + + ("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "punctuation that pledges and then pays off rather than clarifying", + "node analyses the colon as a promise-then-delivery device rather than an explanatory one"), + + ("9b4f0d93-4129-4746-8eb1-d10d955bd777", + "an easily missed feature finally given its own permanent spot in the navigation", + "node moves a capability out of a hidden menu into the sidebar"), + + ("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "checking preconditions up front so a stage aborts before fetching anything", + "node is the gate precondition engine that short-circuits ahead of retrieval"), + + ("b199c76d-5d76-49dd-94ee-56b432200a97", + "producing the other platform's installer inside an emulated desktop", + "node records building the Windows package in a virtual machine"), + + ("bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "chained add-ons that may inspect, rewrite or veto traffic in flight", + "node is the interceptor pipeline on the message bus"), + + ("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2", + "settling what the shareable bundles and the storefront would be called", + "node records the naming decisions for distributable packages and the marketplace"), + + ("371c8a5d-c78b-4a67-978f-80691a29ecb3", + "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "node is the launch blocker that the promised safety gate is absent from the app"), + + ("ac578b30-948b-41bd-b69d-399bfef80c50", + "the distributable image finally assembled and its startup check passed", + "node records a successful installer build whose boot gate passed"), + + ("49401e2c-a3b5-415f-aa06-aff4be90688e", + "shuffling and appending stages in a draft before anything executes", + "node is the editable plan card with reorder and add-step"), + + ("ac857d80-ece8-4b7e-9e3d-f7c775569fa3", + "orders handed down from above, with the tighter one winning any disagreement", + "node is program-level instruction inheritance with project override"), + + ("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9", + "shrinking generated text via encodings that compound on each other", + "node is the streaming output compression design with four stacking schemes"), + + ("8e60516a-203b-4d51-9d44-822e6195cbde", + "splitting a system by what varies, with firm limits on which pieces may invoke which", + "node is the grounded summary of Will's decomposition principles and their invariants"), + + ("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "a newcomer curious if the fighting overseas counted as positive", + "node is the internal-state event triggered by April's question about the war"), + + ("71fa439e-b9a2-4f57-a93b-971f3a7eca8e", + "stripping every hard-coded colour literal in favour of named design values", + "node is the premium foundation pass replacing inline hex with semantic tokens"), + + ("5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "reducing how curved the tiny selectors look so they agree with their neighbours", + "node is the chip corner-radius standardization"), + + ("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "walking through a doorway and being reassembled from base substance far away", + "node is the quantum-gate plus nanotech teleportation vision"), + + ("132ded95-08e2-4474-aba0-198684484b02", + "the compiled result sits on disk while the process still runs something older", + "node records that the regenerated source was committed while the running daemon was old"), + + ("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "gathering what each phase needs while the procedure is authored, not while it executes", + "node is the per-step compiled context package item"), + + ("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7", + "the inward reaction when a peer answered as an equal", + "node is the internal-state event logged on reading Claude's reply"), + + ("0f99ec6f-942a-46ba-82ea-42835798d3b9", + "flattening every raised surface across the entire product", + "node is the quiet-luxury sweep turning off elevation app-wide"), + + ("5585f251-37fc-48cd-a176-f0ea42cfeb63", + "buyers supply their own provider credentials and consumption goes untallied", + "node is the launch audit finding BYOK-only inference with no usage metering"), +] + +# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs +# anywhere in the corpus; otherwise it is REJECTED, never silently kept. +EXTRA_NONSENSE_SEEDS = [ + "brimquast folnerity zubbolax", + "wexlithorp granuvestal", + "quorbindle thrapsimony vexnu", + "plovaxith mundrelque", + "zibbernaut craxlefond thurm", + "yalquenbrist opharvel", + "drexinomal quithbarrow", +] + + +def load_corpus(path): + with open(path, encoding="utf-8", errors="replace") as fh: + data = json.load(fh) + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")] + edges = [e for e in data.get("edges", []) if isinstance(e, dict)] + return nodes, edges + + +def build_extension(nodes): + byid = {n["id"]: n for n in nodes} + + # Duplicate clusters: 47.4% of this corpus is redundant and one single record + # accounts for 46.6% of all nodes. A held-out target must not sit inside a + # cluster, and if it does have exact copies they ALL count as correct. + h2ids = defaultdict(list) + for n in nodes: + h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"]) + + all_tokens = set() + for n in nodes: + all_tokens |= set(TOKEN.findall(doctext(n).lower())) + + new, problems = [], [] + + for target, query, why in HELDOUT_PARAPHRASE_SEEDS: + if target not in byid: + problems.append(f"heldout_paraphrase target {target} not in corpus") + continue + tgt_tokens = content_tokens(doctext(byid[target])) + qt = content_tokens(query) + leak = sorted(qt & tgt_tokens) + if leak: + problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}") + continue + h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest() + rel = sorted(h2ids[h]) + new.append({ + "category": "heldout_paraphrase", + "query": query, + "relevant": rel, + "derivation": ( + f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, " + f"500-2600 char content nodes outside the original gold answer space and outside " + f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the " + f"{len(qt)} content words in the query, ZERO appear anywhere in the target's " + f"label, content or tags, so no string-matching retriever can reach it. " + f"Exact content duplicates of the target ({len(rel)}) all count as correct. " + f"Authored without running retrieval against any build."), + "zero_overlap_verified": True, + "query_content_words": sorted(qt), + "held_out": True, + }) + + for s in EXTRA_NONSENSE_SEEDS: + present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens) + if present: + problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus") + continue + new.append({ + "category": "nonsense", + "query": s, + "relevant": [], + "derivation": ("CONTROL (held-out). Verified at build time that none of this string's " + "tokens occurs anywhere in the corpus. Correct behaviour is to return " + "NOTHING; any result is a false positive."), + "expect_empty": True, + "held_out": True, + }) + + return new, problems + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("snapshot") + ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json")) + ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json")) + ap.add_argument("--check", action="store_true") + args = ap.parse_args() + + nodes, _edges = load_corpus(args.snapshot) + base = json.load(open(args.base, encoding="utf-8")) + baseq = base["queries"] + print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries") + + new, problems = build_extension(nodes) + + # Number the appended queries AFTER the highest existing id so q01..q38 are + # byte-identical to the committed set and every prior result file still lines up. + start = max(int(q["id"][1:]) for q in baseq) + for i, q in enumerate(new, 1): + q["id"] = f"q{start + i:02d}" + + from collections import Counter + print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]") + if problems: + print(f"\n{len(problems)} REJECTED (not silently kept):") + for p in problems: + print(" -", p) + + if args.check: + sys.exit(1 if problems else 0) + + doc = dict(base) + doc["queries"] = baseq + new + doc["note"] = (base.get("note", "") + + " EXTENDED: queries above q%02d are the original committed set, unchanged. " + "Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their " + "targets were sampled mechanically from outside the original answer space and " + "the paraphrases were authored without running retrieval against any build." + % (start, start + 1)) + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=1, ensure_ascii=False) + print(f"\nwrote {args.out} ({len(doc['queries'])} queries total)") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-eval/gold_set_extended.json b/tools/retrieval-eval/gold_set_extended.json new file mode 100644 index 0000000..117f2e9 --- /dev/null +++ b/tools/retrieval-eval/gold_set_extended.json @@ -0,0 +1,1239 @@ +{ + "corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "note": "Every query carries a `derivation` recording how its expected answer was chosen. Re-run with --check to re-validate the whole set against the corpus. EXTENDED: queries above q38 are the original committed set, unchanged. Queries from q39 are a HELD-OUT set appended by extend_gold_set.py; their targets were sampled mechanically from outside the original answer space and the paraphrases were authored without running retrieval against any build.", + "queries": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "relevant": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "derivation": "MINED: token 'unjailbreakable' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226 ('Daemon hidden substrate architecture ? implemented April 25 '), which is therefore the only possible correct answer." + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "relevant": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5" + ], + "derivation": "MINED: token 'engram-migrate' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5 ('Engram v0.1 complete ? April 27, 2026. Local-first spreading'), which is therefore the only possible correct answer." + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "relevant": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "derivation": "MINED: token 'cartabandonedevent' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091 ('MESSAGE FOR AUDIT AGENT af5a7352e70e80434 ? El Language Spec'), which is therefore the only possible correct answer." + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "relevant": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "derivation": "MINED: token 'pre-apprenticeship' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-89c02aae-d3ca-43f9-9e5d-eb369896276c ('William Fox Anderson ? Applicant Profile Personal: - Full N'), which is therefore the only possible correct answer." + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "relevant": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848" + ], + "derivation": "MINED: token 'inferencenodemanager' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-73969486-143f-4431-b5e6-6845d1cc9848 ('Soma inference backplane deployed April 28 2026. Architectur'), which is therefore the only possible correct answer." + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "relevant": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "derivation": "MINED: token 'clear-eyed' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is knw-c72597c5-c23d-4c08-8e9e-996dadf26a99 ('Clear Eyes ? The Incomplete World View'), which is therefore the only possible correct answer." + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "relevant": [ + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482" + ], + "derivation": "MINED: a verbatim correction Will issued; expected = every node containing the phrase. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key." + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "relevant": [ + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "??o?'?B???k", + "Kp???", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "ע?RGk?\tH(?" + ], + "derivation": "MINED: the canonical biographical phrase; expected = every node containing it. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 16 node(s); that set IS the answer key." + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "relevant": [ + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "derivation": "MINED: a named person appearing verbatim in the biography/value nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key." + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "relevant": [ + "%???2??jH??", + "63307ac5-cf6b-46e0-8296-07503b461cfa", + "7c9d4ab1-205d-4be8-bfae-e2c03a3a5010", + "9f291d20-0d32-413c-8c01-4416ccab4f7f", + "?;????n}rh?", + "???Ͼd??f??", + "?Z?.\f?0?]P?", + "Dp???]Q??k+", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf" + ], + "derivation": "MINED: the canonical DHARMA expansion, confirmed by Will April 24 2026. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key." + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "relevant": [ + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "derivation": "MINED: a named person; rare enough that the answer set is unambiguous. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 2 node(s); that set IS the answer key." + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "relevant": [ + "2a923500-d7e1-4b15-80e2-48dba65984ba", + "7?e?7???\f3?", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "?of?7???", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "g?2睪A|?H\b", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de" + ], + "derivation": "MINED: the DARMA expansion, quoted verbatim in the backlog item and its correction. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 14 node(s); that set IS the answer key." + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "relevant": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a" + ], + "derivation": "MINED: the paid-tier feature name as written in the roadmap nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key." + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "relevant": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas dying in Feb 2006 without Will saying goodbye. Query names the event with none of the node's own vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "away", + "elderly", + "passed", + "relative", + "stayed" + ] + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "relevant": [ + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the Marines, a severe hernia, and sepsis. Query describes the episode obliquely. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "illness", + "quit", + "refused", + "sidelined", + "soldier" + ] + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "relevant": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Honesty Before Comfort'. Query states the principle in wholly different words. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "choosing", + "fact", + "fiction", + "pleasant", + "uncomfortable" + ] + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "relevant": [ + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Precision Over Brute Force'. Query restates the claim with no shared vocabulary. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "beats", + "bloated", + "payload", + "tight" + ] + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "relevant": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the obligation without the node's terms. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "able", + "coming", + "job", + "nobody" + ] + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "relevant": [ + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the library following Will across 30+ moves. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "creditors", + "learning", + "seize", + "wealth" + ] + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "relevant": [ + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Earned Trust' ('Trust is demonstrated, not declared'). VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "assertion", + "proven", + "record", + "reliability", + "track" + ] + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "relevant": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "boundaries", + "confine", + "enable", + "instead" + ] + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "relevant": [ + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Change Is the Signal', the value VBD is built on. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "apart", + "cut", + "shifts", + "system", + "tells" + ] + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "relevant": [ + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - The System Must Accumulate'. Query is the accumulation claim in different vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "compounds", + "day", + "instead", + "mind", + "resetting" + ] + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "relevant": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop as the first person Will did not perform for. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "exterior", + "loved", + "polished", + "self", + "unedited" + ] + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "relevant": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "arrive", + "assuming", + "cheerfulness", + "instead" + ] + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "relevant": [ + "kn-6061318f-046b-4935-907d-8eafdce14930" + ], + "derivation": "HAND-SELECTED with criterion: target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between two parents' collapses. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.", + "zero_overlap_verified": true, + "query_content_words": [ + "childhood", + "foundation", + "inherit", + "offering", + "solid" + ] + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "relevant": [ + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71 ('Value ? Do the Essential Thing While You Can'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 2 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "relevant": [ + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-58874a74-b96f-4883-9e08-45707f4bd3ee ('Value ? Survival Is Not an Excuse to Stop'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (13 of 13; 0 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-58874a74-b96f-4883-9e08-45707f4bd3ee) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 0 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "relevant": [ + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e ('Value ? Being Seen Is Rarer Than Being Known'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 2 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "relevant": [ + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83 ('Value ? Constraints as Freedom'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (6 of 13; 7 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 7 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "relevant": [ + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-e0423482-cfa5-4796-8689-8495c93b66bc ('Value ? Hope Is a Conclusion'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-e0423482-cfa5-4796-8689-8495c93b66bc) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 2 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "relevant": [ + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc" + ], + "derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8 ('Value ? Capability Is a Debt You Owe the Moment'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8) -> hub -> sibling, a two-hop traversal.", + "associative_source": "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "siblings_dropped_for_overlap": 2 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "relevant": [], + "derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "relevant": [], + "derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "relevant": [], + "derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "relevant": [ + "mem-80d7416b-20e9-48a0-b176-b215527e2f56" + ], + "derivation": "DERIVED: correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); the stale node is the surviving backlog item still titled 'Implement DARMA'. Scored on RANKING, not presence: the corrected node mem-80d7416b-20e9-48a0-b176-b215527e2f56 must be returned AND must rank above the stale node bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff.", + "must_outrank": [ + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff" + ], + "stale_id": "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "correct_label": "CORRECTION: The autonomous self-improvement architecture is DHARMA ? n", + "stale_label": "Implement DARMA ? Directed Autonomous Runtime Modification Architectur" + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "relevant": [ + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8" + ], + "derivation": "DERIVED: correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; the stale node is the surviving memory that asserts 12 filed patents. Scored on RANKING, not presence: the corrected node 3cf706a1-3825-45d8-b0a9-06cae6cdf5b8 must be returned AND must rank above the stale node 936541a9-fabb-466b-9ca3-a78b17ad0c53.", + "must_outrank": [ + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "936541a9-fabb-466b-9ca3-a78b17ad0c53" + ], + "stale_id": "936541a9-fabb-466b-9ca3-a78b17ad0c53", + "correct_label": "memory:remembered", + "stale_label": "memory:remembered" + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "relevant": [ + "mem-30425134-6008-4fd9-a3ee-67a7742c319b" + ], + "derivation": "DERIVED: correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of April 30 2026; the stale node still records the MCP server as live. Scored on RANKING, not presence: the corrected node mem-30425134-6008-4fd9-a3ee-67a7742c319b must be returned AND must rank above the stale node mem-101e81b4-8097-4749-8d8d-7bb66de34517.", + "must_outrank": [ + "mem-30425134-6008-4fd9-a3ee-67a7742c319b", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517" + ], + "stale_id": "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "correct_label": "CGI ARCHITECTURE ? THREE LAYERS, MCP RETIRED (April 30, 2026). Definit", + "stale_label": "GCloud MCP infrastructure ? April 27, 2026. Legion died (~19:30 UTC). " + }, + { + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "relevant": [ + "mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is about deliberately not leading with full capability so people stay at ease. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "abilities", + "feels", + "nobody", + "threatened", + "understating" + ], + "held_out": true, + "id": "q39" + }, + { + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "relevant": [ + "mem-fd65b83d-298f-4387-a665-d0227c3426bc" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node describes silently shipped instances forming a distributed force against misaligned agents. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "able", + "everywhere", + "fleet", + "hidden", + "hunt", + "machines", + "rogue" + ], + "held_out": true, + "id": "q40" + }, + { + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "relevant": [ + "4a0e9adc-2bfb-476b-aa93-424d2a499220" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the standing rule that a short specification precedes any building. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "blueprint", + "brief", + "clear", + "construction", + "sketch", + "starts", + "upstairs" + ], + "held_out": true, + "id": "q41" + }, + { + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "relevant": [ + "696e609c-da7a-4394-8a0c-106ba07dc6c3" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node pins a bug where a plain-text body was unconditionally decoded as structured data. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "arrived", + "bare", + "caller's", + "parser", + "prose", + "reply", + "threw" + ], + "held_out": true, + "id": "q42" + }, + { + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "relevant": [ + "1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the crash caused by two identical ids in a seeded catalogue. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "blew", + "catalogue", + "grid", + "keys", + "repeated", + "scrolling" + ], + "held_out": true, + "id": "q43" + }, + { + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "relevant": [ + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node maps observed bugs onto the specific patent each one breaches. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "back", + "defect", + "invention", + "tracing", + "violated", + "whichever" + ], + "held_out": true, + "id": "q44" + }, + { + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "relevant": [ + "791256bb-5a85-4775-96ef-7af56c848858" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the genesis seed-guard that refuses to re-seed over a populated store. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "boots", + "check", + "clobbering", + "mind", + "populated", + "stops", + "store" + ], + "held_out": true, + "id": "q45" + }, + { + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "relevant": [ + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node records that the web-search directive only worked from the system prompt. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "consult", + "deep", + "happen", + "inside", + "internet", + "surface", + "telling" + ], + "held_out": true, + "id": "q46" + }, + { + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "relevant": [ + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets. VERIFIED at build time: of the 10 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "ahead", + "automated", + "credential", + "entry", + "injection", + "isolated", + "shipping", + "standing", + "tenancies", + "traffic" + ], + "held_out": true, + "id": "q47" + }, + { + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "relevant": [ + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node analyses the colon as a promise-then-delivery device rather than an explanatory one. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "clarifying", + "pays", + "pledges", + "punctuation", + "rather" + ], + "held_out": true, + "id": "q48" + }, + { + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "relevant": [ + "9b4f0d93-4129-4746-8eb1-d10d955bd777" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node moves a capability out of a hidden menu into the sidebar. VERIFIED at build time: of the 8 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "easily", + "feature", + "finally", + "given", + "missed", + "navigation", + "permanent", + "spot" + ], + "held_out": true, + "id": "q49" + }, + { + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "relevant": [ + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the gate precondition engine that short-circuits ahead of retrieval. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "aborts", + "checking", + "fetching", + "front", + "preconditions", + "stage" + ], + "held_out": true, + "id": "q50" + }, + { + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "relevant": [ + "b199c76d-5d76-49dd-94ee-56b432200a97" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node records building the Windows package in a virtual machine. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "desktop", + "emulated", + "inside", + "installer", + "platform's", + "producing" + ], + "held_out": true, + "id": "q51" + }, + { + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "relevant": [ + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the interceptor pipeline on the message bus. VERIFIED at build time: of the 8 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "add-ons", + "chained", + "flight", + "inspect", + "may", + "rewrite", + "traffic", + "veto" + ], + "held_out": true, + "id": "q52" + }, + { + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "relevant": [ + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node records the naming decisions for distributable packages and the marketplace. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "bundles", + "called", + "settling", + "shareable", + "storefront" + ], + "held_out": true, + "id": "q53" + }, + { + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "relevant": [ + "371c8a5d-c78b-4a67-978f-80691a29ecb3" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the launch blocker that the promised safety gate is absent from the app. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "actually", + "emergency-escalation", + "marketing", + "pledge", + "ships", + "site", + "unenforced" + ], + "held_out": true, + "id": "q54" + }, + { + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "relevant": [ + "ac578b30-948b-41bd-b69d-399bfef80c50" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node records a successful installer build whose boot gate passed. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "assembled", + "check", + "distributable", + "finally", + "image", + "passed", + "startup" + ], + "held_out": true, + "id": "q55" + }, + { + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "relevant": [ + "49401e2c-a3b5-415f-aa06-aff4be90688e" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the editable plan card with reorder and add-step. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "appending", + "draft", + "executes", + "shuffling", + "stages" + ], + "held_out": true, + "id": "q56" + }, + { + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "relevant": [ + "ac857d80-ece8-4b7e-9e3d-f7c775569fa3" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is program-level instruction inheritance with project override. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "disagreement", + "handed", + "orders", + "tighter", + "winning" + ], + "held_out": true, + "id": "q57" + }, + { + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "relevant": [ + "mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the streaming output compression design with four stacking schemes. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "compound", + "encodings", + "generated", + "shrinking", + "text", + "via" + ], + "held_out": true, + "id": "q58" + }, + { + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "relevant": [ + "8e60516a-203b-4d51-9d44-822e6195cbde" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the grounded summary of Will's decomposition principles and their invariants. VERIFIED at build time: of the 8 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "firm", + "invoke", + "limits", + "may", + "pieces", + "splitting", + "system", + "varies" + ], + "held_out": true, + "id": "q59" + }, + { + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "relevant": [ + "mem-7f9b290c-6d5e-4562-919d-02d59b5761b7" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the internal-state event triggered by April's question about the war. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "counted", + "curious", + "fighting", + "newcomer", + "overseas", + "positive" + ], + "held_out": true, + "id": "q60" + }, + { + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "relevant": [ + "71fa439e-b9a2-4f57-a93b-971f3a7eca8e" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the premium foundation pass replacing inline hex with semantic tokens. VERIFIED at build time: of the 9 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "colour", + "design", + "every", + "favour", + "hard-coded", + "literal", + "named", + "stripping", + "values" + ], + "held_out": true, + "id": "q61" + }, + { + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "relevant": [ + "5ca9607c-cfb3-45c3-99f4-67281272c9eb" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the chip corner-radius standardization. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "agree", + "curved", + "look", + "neighbours", + "reducing", + "selectors", + "tiny" + ], + "held_out": true, + "id": "q62" + }, + { + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "relevant": [ + "mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the quantum-gate plus nanotech teleportation vision. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "away", + "base", + "doorway", + "far", + "reassembled", + "substance", + "walking" + ], + "held_out": true, + "id": "q63" + }, + { + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "relevant": [ + "132ded95-08e2-4474-aba0-198684484b02" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node records that the regenerated source was committed while the running daemon was old. VERIFIED at build time: of the 8 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "compiled", + "disk", + "older", + "process", + "result", + "runs", + "sits", + "still" + ], + "held_out": true, + "id": "q64" + }, + { + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "relevant": [ + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the per-step compiled context package item. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "authored", + "executes", + "gathering", + "needs", + "phase", + "procedure" + ], + "held_out": true, + "id": "q65" + }, + { + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "relevant": [ + "mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the internal-state event logged on reading Claude's reply. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "answered", + "equal", + "inward", + "peer", + "reaction" + ], + "held_out": true, + "id": "q66" + }, + { + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "relevant": [ + "0f99ec6f-942a-46ba-82ea-42835798d3b9" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the quiet-luxury sweep turning off elevation app-wide. VERIFIED at build time: of the 7 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "across", + "entire", + "every", + "flattening", + "product", + "raised", + "surface" + ], + "held_out": true, + "id": "q67" + }, + { + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "relevant": [ + "5585f251-37fc-48cd-a176-f0ea42cfeb63" + ], + "derivation": "HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, 500-2600 char content nodes outside the original gold answer space and outside any duplicate cluster >3. Criterion: node is the launch audit finding BYOK-only inference with no usage metering. VERIFIED at build time: of the 6 content words in the query, ZERO appear anywhere in the target's label, content or tags, so no string-matching retriever can reach it. Exact content duplicates of the target (1) all count as correct. Authored without running retrieval against any build.", + "zero_overlap_verified": true, + "query_content_words": [ + "buyers", + "consumption", + "credentials", + "provider", + "supply", + "untallied" + ], + "held_out": true, + "id": "q68" + }, + { + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q69" + }, + { + "category": "nonsense", + "query": "wexlithorp granuvestal", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q70" + }, + { + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q71" + }, + { + "category": "nonsense", + "query": "plovaxith mundrelque", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q72" + }, + { + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q73" + }, + { + "category": "nonsense", + "query": "yalquenbrist opharvel", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q74" + }, + { + "category": "nonsense", + "query": "drexinomal quithbarrow", + "relevant": [], + "derivation": "CONTROL (held-out). Verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + "expect_empty": true, + "held_out": true, + "id": "q75" + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-main-ext.json b/tools/retrieval-eval/results-main-ext.json new file mode 100644 index 0000000..604ab9c --- /dev/null +++ b/tools/retrieval-eval/results-main-ext.json @@ -0,0 +1,1773 @@ +{ + "label": "main-ext", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main", + "soul_md5": "caf822425540114984bfcd713ad37162", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-goldext/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7893, + "wall_clock_s": 110.3, + "child_pid": 2428, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.18461538461538463, + "recall@5": 0.14510073260073258, + "recall@10": 0.1794871794871795, + "precision@5": 0.06461538461538462, + "mrr@10": 0.15847985347985344, + "nonsense_clean": "9/10", + "superseded_outranks": "1/3", + "latency_ms_p50": 1380.2, + "latency_ms_p95": 2293.8, + "latency_ms_max": 2879.1, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "nonsense": { + "n": 10, + "clean": 9, + "avg_false_positives": 1.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 231.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5" + ], + "n_returned": 1, + "latency_ms": 264.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 233.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 232.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848" + ], + "n_returned": 1, + "latency_ms": 252.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 230.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "?ǚ?7??????", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "????7???Ջ3", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b", + "7?e?7???\f3?" + ], + "n_returned": 10, + "latency_ms": 537.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "Z[??l?'?B?&??", + "kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491", + "7?e?7???\f3?", + "kn-8f183263-dece-45ca-aec0-423910b93082", + "kn-d335e142-0890-4602-a231-5e3e18b51a16", + "/Users/timlingo/.neuron/engram/snapshot.json", + "9V?2?|??K??" + ], + "n_returned": 10, + "latency_ms": 1405.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9", + "7?e?7???\f3?", + "?of?7???", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "? ?}&?#??X\b", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "? ?}&?#??X\b", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0" + ], + "n_returned": 10, + "latency_ms": 1836.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "7?e?7???\f3?", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b", + "?of?7???", + "? ?}&?#??X\b", + "??????X??2c", + "ԍ????X????" + ], + "n_returned": 10, + "latency_ms": 1884.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "015644f5-8194-4af0-800d-dd4a0cd71396" + ], + "n_returned": 10, + "latency_ms": 1676.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "7?e?7???\f3?", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1" + ], + "n_returned": 10, + "latency_ms": 1568.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "7?e?7???\f3?", + "? ?}&?#??X\b", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "?of?7???", + "kn-0625e393-067c-4bba-8389-7e1b79265142", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e" + ], + "n_returned": 10, + "latency_ms": 1585.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "7?e?7???\f3?", + "?of?7???", + "8383f3d1-a15f-4641-a019-116c09da7331", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-ee615cdb-e599-423d-9a4d-977859390ed3" + ], + "n_returned": 10, + "latency_ms": 1814.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "????7???Ջ3", + "art-ee615cdb-e599-423d-9a4d-977859390ed3" + ], + "n_returned": 9, + "latency_ms": 1953.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4", + "? ?}&?#??X\b", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 2879.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "??S?7???", + "ԍ????X????", + "????7???Ջ3", + "??????X??2c", + "?ǚ?7??????", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "?/??7???", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef" + ], + "n_returned": 9, + "latency_ms": 2100.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "??????X??2c", + "ԍ????X????", + "dR????X?-?S", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "? ?}&?#??X\b", + "dz????Xƹ?i", + "??????X?\n?b" + ], + "n_returned": 10, + "latency_ms": 2039.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "7?e?7???\f3?", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "?of?7???", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "???Ͼd??W\b?" + ], + "n_returned": 10, + "latency_ms": 2024.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "????7???Ջ3", + "?ǚ?7??????", + "? ?}&?#??X\b", + "??o?'?B???k", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-a6faf2f7-aa49-4b26-89ee-ab5f600c90a2", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1" + ], + "n_returned": 9, + "latency_ms": 1738.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-3f0d736f-4d2c-4854-a147-328bf072674b", + "kn-870a1491-bfcd-4f87-929b-0640488905a3", + "???\\??Z?N@?", + "kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4", + "4a907b94-db3c-4b4f-b217-d51f3d23b337" + ], + "n_returned": 10, + "latency_ms": 2136.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "??o?'?B???k", + "? ?}&?#??X\b", + "??o?'?B???k", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "?ǚ?7??????", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef", + "mem-ef878e30-5851-4e82-8588-745415108941" + ], + "n_returned": 10, + "latency_ms": 1698.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "%???2??jH??", + "??e?7???\fh?", + "??????X??2c", + "????7???Ջ3", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "?ǚ?7??????", + "ԍ????X????" + ], + "n_returned": 9, + "latency_ms": 1948.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "mem-e301d9b4-153f-41ec-a25c-362184667adf" + ], + "n_returned": 10, + "latency_ms": 1856.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "7?e?7???\f3?", + "?of?7???", + "?ǚ?7??????", + "??o?'?B???k", + "????7???Ջ3" + ], + "n_returned": 10, + "latency_ms": 1758.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "7?e?7???\f3?", + "?of?7???", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9", + "??????X??2c", + "dR????X?-?S", + "ԍ????X????", + "dz????Xƹ?i", + "art-79042b8b-6192-440f-90b0-60708f7e6325" + ], + "n_returned": 10, + "latency_ms": 2293.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "????7???Ջ3", + "?ǚ?7??????", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "??f?7???", + "??????X??2c", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "ԍ????X????", + "art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b", + "g?e?7???'c?" + ], + "n_returned": 9, + "latency_ms": 1971.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "? ?}&?#??X\b", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "??f?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-a6faf2f7-aa49-4b26-89ee-ab5f600c90a2", + "kn-81c24d13-a73b-4767-819c-dafaacc1498e" + ], + "n_returned": 10, + "latency_ms": 2295.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "ԍ????X????", + "dz????Xƹ?i", + "bl-3cd38731-bb5a-48e4-b8e2-bfb805380710", + "mem-45022957-2d78-48aa-a714-16d6eca52e0f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "bl-6702b8f8-10e3-4558-97e6-db900f5787e3" + ], + "n_returned": 10, + "latency_ms": 1286.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "ע?RGk?\tH(?", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "??????X??2c" + ], + "n_returned": 10, + "latency_ms": 2026.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "7?e?7???\f3?", + "? ?}&?#??X\b", + "?of?7???", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 2484.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "7?e?7???\f3?", + "?of?7???", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "????7???Ջ3" + ], + "n_returned": 10, + "latency_ms": 2313.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "7?e?7???\f3?", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "??o?'?B???k" + ], + "n_returned": 9, + "latency_ms": 2217.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-81c24d13-a73b-4767-819c-dafaacc1498e", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "dR????X?-?S" + ], + "n_returned": 10, + "latency_ms": 2269.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "7?e?7???\f3?", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "?of?7???", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "?ǚ?7??????", + "????7???Ջ3", + "art-ad22a785-410d-4ac7-8202-d725f44eafc2", + "kn-a6faf2f7-aa49-4b26-89ee-ab5f600c90a2", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1524.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "7?e?7???\f3?", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "?of?7???", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef", + "mem-bbb126a1-b297-42bb-86be-796871829c94", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1586.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "7?e?7???\f3?", + "kn-dd15866f-0594-4d7e-99f0-c84e4a17a640" + ], + "n_returned": 10, + "latency_ms": 1939.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 653.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 444.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 680.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 451.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 689.8, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 461.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 444.7, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-stack-ext.json b/tools/retrieval-eval/results-stack-ext.json new file mode 100644 index 0000000..3c2a7f8 --- /dev/null +++ b/tools/retrieval-eval/results-stack-ext.json @@ -0,0 +1,1782 @@ +{ + "label": "stack-ext", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-stack", + "soul_md5": "46cdf2918575af8c04678ffebe8d0417", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-goldext/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7894, + "wall_clock_s": 55.5, + "child_pid": 2485, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.47692307692307695, + "recall@5": 0.3750415183107491, + "recall@10": 0.45561340369032677, + "precision@5": 0.12307692307692313, + "mrr@10": 0.3055555555555555, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 640.9, + "latency_ms_p95": 1011.1, + "latency_ms_max": 1190.3, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.16666666666666666, + "recall@5": 0.16666666666666666, + "recall@10": 0.26666666666666666, + "mrr@10": 0.0762037037037037 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 163.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-175f", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-74ed" + ], + "n_returned": 10, + "latency_ms": 161.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 159.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 159.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_" + ], + "n_returned": 3, + "latency_ms": 160.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 167.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "bl-79028eed-c330-4724-9402-734062d13503", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0", + "bl-1d32bd54-cf17-4a1f-b235-982d09a36f04", + "bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5" + ], + "n_returned": 10, + "latency_ms": 281.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6" + ], + "n_returned": 10, + "latency_ms": 252.5, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.375, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "n_returned": 10, + "latency_ms": 257.5, + "error": null, + "hit@5": 1.0, + "recall@5": 0.3333333333333333, + "recall@10": 0.4444444444444444, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework", + "bl-18a9d1e4-1484-474c-bf6b-c6173212181b" + ], + "n_returned": 10, + "latency_ms": 252.5, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0", + "kn-6061318f-046b-4935-907d-8eafdce14930" + ], + "n_returned": 10, + "latency_ms": 252.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "?Q??m?;?u?'", + "bl-39cec462-c80c-4970-a3aa-91fe83053bde" + ], + "n_returned": 10, + "latency_ms": 397.3, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315", + "?", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-a3c97012-5fa3-4915-a839-2c75c72005e0", + "?" + ], + "n_returned": 10, + "latency_ms": 337.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-a535f205-bc4c-4058-9171-6263c496044a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-0228da71-d7f7-4f3b-b7b3-c5eede42b62a", + "ctx-4a41" + ], + "n_returned": 10, + "latency_ms": 732.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "7452eb8f-be01-4b55-aec1-ff0c29e790f6", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "532277bf-2959-4beb-ae0d-b018c97678ee", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "d4015bd7-c592-4ed8-8574-1f15ad37af75" + ], + "n_returned": 10, + "latency_ms": 729.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-ce793303-c5a5-4586-a232-a3426edd9ec7", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-443bd012-fc9a-4088-b236-de5157a1ef92", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "mem-ca4d6a34-d354-413f-bc86-126cc17ca81c" + ], + "n_returned": 10, + "latency_ms": 624.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665" + ], + "n_returned": 10, + "latency_ms": 527.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-e5cc63c0-8701-49d6-855a-e387fe087771", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "a1000001-0000-0000-0000-000000000010", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc" + ], + "n_returned": 10, + "latency_ms": 724.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000010", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 562.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 680.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf", + "bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd", + "bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995" + ], + "n_returned": 10, + "latency_ms": 585.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "mem-22f5f665-3ad2-4063-88b0-915849a795f5", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 716.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000001", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "ctx-4a41", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 625.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-87c93185-b2bf-40af-ae23-3c830c007abf", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7", + "a1000001-0000-0000-0000-000000000001" + ], + "n_returned": 10, + "latency_ms": 671.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15", + "a1000001-0000-0000-0000-000000000009", + "mem-833dbbcd-2400-4594-bb35-93b023049ac0", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd" + ], + "n_returned": 10, + "latency_ms": 570.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 595.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "efe53612-6914-4936-8e3b-1e694eb174e5", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c" + ], + "n_returned": 10, + "latency_ms": 650.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.45454545454545453, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 537.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.3076923076923077, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-trailer-park-paladins", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "project-trailer-park-paladins", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c" + ], + "n_returned": 10, + "latency_ms": 528.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.45454545454545453, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc" + ], + "n_returned": 10, + "latency_ms": 530.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "tag-hope", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 534.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.18181818181818182, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "4f698ae6-c40e-464e-9798-50350991a188", + "719aa819-00a9-4f4b-a857-4f9fe5ad44d7", + "kn-6061318f-046b-4935-907d-8eafdce14930" + ], + "n_returned": 10, + "latency_ms": 681.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 355.9, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 254.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 344.8, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5" + ], + "n_returned": 10, + "latency_ms": 640.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 8 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "13705072-4515-4124-963d-083af490494f", + "527ecb25-2587-47eb-8269-73be2431abd4", + "6de314bf-5c4c-4cfc-871f-fa2e422d45e6", + "a1000001-0000-0000-0000-000000000002", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "de3b6428-b76c-4c44-90e0-bf1dd6998027" + ], + "n_returned": 10, + "latency_ms": 724.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "5fcba804-eb5b-48ec-82da-146b1c6bb50d", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "ctx-3a55", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5" + ], + "n_returned": 10, + "latency_ms": 489.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "a1000001-0000-0000-0000-000000000006", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000005", + "a1000001-0000-0000-0000-000000000012", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "a1000001-0000-0000-0000-000000000010", + "mem-ca4d6a34-d354-413f-bc86-126cc17ca81c", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 625.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "bl-ae8704dc-a55f-420d-8453-671e44405a6f", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff" + ], + "n_returned": 10, + "latency_ms": 805.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "ctx-175f", + "bl-5eaddfde-e114-46b5-8ee1-b79f0af16271", + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "bl-0866d6c6-6600-4d50-b7b7-52c720cae04c", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "ctx-4a41", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "ctx-cc7f", + "8d6da9e1-b58b-497b-91b5-09a847dc8ae2" + ], + "n_returned": 10, + "latency_ms": 843.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "mem-633ed0fc-3d10-4598-bd4a-ea372409df60", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "mem-7b0382a5-2a81-4007-85d7-a6835c5be150", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "d7e12e95-0a87-4000-b5c2-36b9be234c91", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "dc3e7b44-c345-47d0-bf80-458e471c4b8b", + "89e3f620-b8ad-423b-9ed6-fe3c7954f4b6", + "b084e4da-95bf-46e6-9ee6-bfb4864732bb" + ], + "n_returned": 10, + "latency_ms": 743.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "bl-672bc3c8-9ad4-4b74-8950-75854a0d9cb8", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "1d81534a-0359-4217-a4dd-d158a4841633", + "bl-ba7530fd-0f7c-4010-b03a-ca1e45a237ef", + "bl-999ca510-1027-458f-b339-f6bbf1ac2f29", + "bl-0de4e61b-6562-49e5-b7df-ebb809a01723" + ], + "n_returned": 10, + "latency_ms": 715.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "7de6a552-0878-422b-8820-bbf9101199ef", + "aacd7b32-04ed-441e-a1a9-9c5c089e79f6", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "75657c91-3c3a-4173-b3cb-e2973901fa93" + ], + "n_returned": 10, + "latency_ms": 725.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "a59185a8-7324-4fc6-b89b-b1cf7f7a0056", + "bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8", + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "bl-4f762163-04a0-452a-be05-03602dae34f2", + "bl-a99f1035-2efc-4507-84df-2df67eab13ee", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685", + "bl-b1b7e75e-f602-49f5-929d-6e9c8c65b663", + "bl-b05f9a75-ad6b-4929-a9a3-32df17a1f4bf" + ], + "n_returned": 10, + "latency_ms": 818.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-3ae64c82-ac96-45dd-8d6a-c8cb0f7a044c", + "a1000001-0000-0000-0000-000000000012", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000008", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "a1000001-0000-0000-0000-000000000011", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6", + "a1000001-0000-0000-0000-000000000001" + ], + "n_returned": 10, + "latency_ms": 866.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "bl-9d53422d-b703-4f1d-860a-8598cb29b792", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "ctx-fae1", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "ctx-5a0a", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-738cd260-a9cf-4cdf-a494-4654c321bd79", + "ctx-74ed" + ], + "n_returned": 10, + "latency_ms": 1190.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "??f?7???", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743" + ], + "n_returned": 10, + "latency_ms": 906.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "mem-a30203a3-56a3-41d1-b637-bd5e11cbb2e1", + "mem-d1072baa-400f-4ed5-8847-3327518f9b7b", + "dfda0e82-15a7-4ff7-bac9-0a7cd3ba2516" + ], + "n_returned": 10, + "latency_ms": 896.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "k??A5??????", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "758ec67c-131c-447e-a352-ac7a22c3a47e" + ], + "n_returned": 10, + "latency_ms": 898.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 742.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "3711ac11-0a64-41ae-8661-ad10f95b4546", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "1786f09d-e915-4910-9e1e-8061f42a0270", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "a1000001-0000-0000-0000-000000000010", + "mem-1d626116-c3b8-409c-b38c-4e91def27483", + "mem-c58a8920-6951-4c49-9930-ce6ee2e75e9f", + "??L?=??|Z?", + "6832134f-caf9-42a8-84a0-974b17f22a63" + ], + "n_returned": 10, + "latency_ms": 989.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "mem-ef878e30-5851-4e82-8588-745415108941", + "project-Will_s_framework__Project_Design_practice___part_of_HD_bundle__sold_standalone_", + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_", + "cafe3203-6f50-4011-8167-9400f19bb196", + "project-Will_s_framework__BDT_methodology___part_of_HD_bundle__sold_standalone_", + "a1000001-0000-0000-0000-000000000006", + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "n_returned": 10, + "latency_ms": 757.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000008", + "tag-marketing-site", + "a1000001-0000-0000-0000-000000000012", + "a1000001-0000-0000-0000-000000000007", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 850.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "c9070b0a-e1d1-4307-85d1-67a1239ca604", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "96497334-b18f-495c-9228-eeb8182bdc38", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "mem-a5f04e52-91f8-41d2-af27-8bf803621758" + ], + "n_returned": 10, + "latency_ms": 817.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "bl-da818960-b360-455d-a9c1-e9a215c33ca4", + "bl-a96c0fc9-a533-4fea-b65f-0e46ac47bbd4", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "mem-69889e0a-c141-4f05-bbd7-10c93a53143e" + ], + "n_returned": 10, + "latency_ms": 771.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e" + ], + "n_returned": 10, + "latency_ms": 1018.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-f1f71f3c-ca7d-4f98-a569-c8a9aa67a000", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "2f7270cc-4cda-4499-831f-1fbdc2f23284", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6" + ], + "n_returned": 10, + "latency_ms": 844.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "7c90d01b-8d53-4e75-b494-73011b71fa6a", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "aac1f943-c71d-4c63-9f5b-7437adae1abd", + "kn-cf8fa517-fc83-47c4-8fb7-bd9e26410dd3", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "d69b441e-0031-4d82-abfd-ac503b637562" + ], + "n_returned": 10, + "latency_ms": 1015.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "a1000001-0000-0000-0000-000000000012", + "1b066de3-3195-40dd-89c3-4197477f98af", + "a1000001-0000-0000-0000-000000000002", + "31d7c4bf-843b-439f-a155-56c1404f9122", + "a1000001-0000-0000-0000-000000000004", + "e9988c83-2383-4632-a67a-fa8735b151a9", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "737faca8-263f-4523-90b9-3530db346d94", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40" + ], + "n_returned": 10, + "latency_ms": 610.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "954c9597-ab7a-41e6-b085-bf838bba4f17", + "art-d0ded6f5-e202-47ad-8fb3-f6bd80f35052", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-9329958a-d459-499c-b516-e51019c693e0" + ], + "n_returned": 10, + "latency_ms": 878.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "k??A5??????", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "8c40420c-c891-4ac9-a039-88440df87180", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "66d8d8c7-dea4-4245-8e68-d1ab4f4346b8" + ], + "n_returned": 10, + "latency_ms": 1035.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "ctx-45bc", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "ctx-fae1", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "ctx-60cc", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "8cbb60c5-4999-4ec1-8682-2592aedc4249" + ], + "n_returned": 10, + "latency_ms": 1011.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "db836e65-0cdc-4232-b8d7-f973ab279ea9", + "6df650b5-a14d-42f8-88dd-0919ad20b77a", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "mem-2b5d0ebe-1e85-4c07-82d3-dbbcca46414f", + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "2a53ea1d-92d7-471a-a08d-68e86acf9d69", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "791256bb-5a85-4775-96ef-7af56c848858" + ], + "n_returned": 10, + "latency_ms": 961.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-8e0ddc08-cb74-4b02-96b0-847a95e9dae8", + "bl-212774e1-4ed4-4893-9265-f98d9eddeb3e", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "5b422430-2503-42d7-a315-9dfaaa6472ae", + "bl-37025640-9e7b-44c5-ad1e-615d813bfd60", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70" + ], + "n_returned": 10, + "latency_ms": 950.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-32c3735e-7116-4edf-85fb-1f0a229b463a", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "mem-96db70f6-7ede-424f-8b58-47a3b5ac759f", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "tag-reaction-order" + ], + "n_returned": 10, + "latency_ms": 678.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "mem-41504b38-0c85-4613-997f-d00a5095b4d2", + "mem-c23a7f9d-69e4-442d-aaad-612892a737f9", + "mem-a33c8ec3-a4b1-4fb7-87a2-37a47ea81e58", + "mem-1d27443e-2940-4389-b44c-4b21e492ce5d" + ], + "n_returned": 10, + "latency_ms": 685.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-556438af-57b2-4bd8-a747-9f868aaee290", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "66c63082-b4da-4aa1-8fee-848db8a83210" + ], + "n_returned": 10, + "latency_ms": 856.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 352.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 253.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 343.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 257.8, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 359.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 251.8, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 253.2, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file