From b9ef66cae9fc6e588e3b87fcf4f87af4fac48523 Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 7 Aug 2026 18:12:43 -0500 Subject: [PATCH] fix(engram): the semantic leg was deleting 234 memory records per boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accumulated retrieval stack (iterations 1-9) put the claim-24 semantic leg and the claim-10 associative leg on engram_search_json — the function ~40 internal .el call sites already used as a KEYED read. Seven of those sites delete every record that comes back ("prune all existing X nodes, keep exactly one"): memory.el:176, sessions.el:250/268/444/523, soul.el:359. mem_boot_count_inc() calls engram_search_json("soul:boot_count", 50) and engram_forget()s all 50 results. With a lexical leg that returned 1 record. With a semantic leg it returns 50 — the 49 nearest neighbours of the STRING "soul:boot_count" — and the soul deletes them. MEASURED on the harness corpus, isolated, read-only, zero writes from any caller: 234 node records destroyed in a single boot. The deletion list is the soul's own lookup result list, in rank order. Casualties include 6 Knowledge nodes, a layer-1 "CORE IDENTITY - GENESIS, LINEAGE" Memory, the value node kn-58874a74, and the gold answers to 8 of the 75 gold-set queries. After the fix: 1 deletion, which is the one the code intends. THE BOUNDARY, from Will. Claim 24 authorises the vector index "to respond to EMBEDDING SEARCH QUERIES by returning the node records whose embedding vectors have the highest cosine similarity to a query vector". A keyed state read is not an embedding search query; it is the identifier-keyed retrieval of claim 23 ("node records are stored under a key encoding the node identifier"). One function served both, so a nearest neighbour of "soul:boot_count" was treated as a boot counter. So: engram_search_json returns to its lexical contract, and the legs move to engram_recall_json, which is what /api/neuron/recall reaches — the route the MCP wrapper, the app, and this harness all call. Retrieval quality on that route is unchanged by construction. MEASURED, 75-query extended gold set, embedded corpus, vs the iteration-9 baseline: +3 / -0 (q15, q28, q60), p=0.2500, hit@5 53.8 -> 58.5%, latency 1.02x, every regression guard held, nonsense 10/10. Net +3 against a floor of 6 is NOT-SHOWN and I am not calling it an improvement. The deliverable is the defect. Diagnostics kept, env-gated (EG_DIAG / EG_DIAG_ID), zero cost when unset: node/embedding census at load, per-query leg dump, and a FORGET log — the last is the regression detector for exactly this class of bug. LIMIT, stated: handle_api_search_knowledge still uses the lexical function. It is a retrieval surface and arguably wants the legs, but nothing in this harness measures it, so I did not change unmeasured behaviour. --- neuron-api.el | 7 +- tools/retrieval-eval/cmp-splitfix.json | 149 ++ tools/retrieval-eval/results-splitfix-r2.json | 1825 +++++++++++++++++ tools/retrieval-eval/results-splitfix.json | 1825 +++++++++++++++++ .../el-runtime/v1.0.0-20260501/el_runtime.c | 96 +- .../el-runtime/v1.0.0-20260501/el_runtime.h | 1 + 6 files changed, 3900 insertions(+), 3 deletions(-) create mode 100644 tools/retrieval-eval/cmp-splitfix.json create mode 100644 tools/retrieval-eval/results-splitfix-r2.json create mode 100644 tools/retrieval-eval/results-splitfix.json diff --git a/neuron-api.el b/neuron-api.el index 6f7c625..4a02e4d 100644 --- a/neuron-api.el +++ b/neuron-api.el @@ -450,7 +450,12 @@ fn handle_api_recall(method: String, path: String, body: String) -> String { if str_eq(eff_q, "") { return api_or_empty(engram_scan_nodes_json(limit, 0)) } - let results: String = engram_search_json(eff_q, limit) + // engram_recall_json, not engram_search_json: this route IS the retrieval + // surface (claim 24's "embedding search queries"), so it gets the semantic + // and associative legs. engram_search_json stays lexical because ~40 + // internal call sites pass a KEY and seven of them delete every record + // that comes back — see the boundary note above eg_search_json_impl. + let results: String = engram_recall_json(eff_q, limit) return api_or_empty(results) } diff --git a/tools/retrieval-eval/cmp-splitfix.json b/tools/retrieval-eval/cmp-splitfix.json new file mode 100644 index 0000000..9375709 --- /dev/null +++ b/tools/retrieval-eval/cmp-splitfix.json @@ -0,0 +1,149 @@ +{ + "baseline": "unfloor-clean", + "candidate": "splitfix", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q15", + "q28", + "q60" + ], + "broken_by_candidate": [], + "discordant": 3, + "net_queries": 3, + "mcnemar_exact_p": 0.25, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5846153846153846, + "recall@5": 0.48102442429365505, + "recall@10": 0.5692415490492414, + "precision@5": 0.14153846153846153, + "mrr@10": 0.3351709401709402, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 646.9, + "latency_ms_p95": 1028.5, + "latency_ms_max": 1197.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.15967365967365968, + "mrr@10": 0.24166666666666667 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.43333333333333335, + "mrr@10": 0.12120370370370372 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.7692307692307693, + "recall@5": 0.7692307692307693, + "recall@10": 0.8461538461538461, + "mrr@10": 0.33269230769230773 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-splitfix-r2.json b/tools/retrieval-eval/results-splitfix-r2.json new file mode 100644 index 0000000..1364396 --- /dev/null +++ b/tools/retrieval-eval/results-splitfix-r2.json @@ -0,0 +1,1825 @@ +{ + "label": "splitfix-r2", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-splitfix", + "soul_md5": "b0a5279aa2c2f4ba0eb9c0b36ca844b1", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semsub/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7894, + "wall_clock_s": 56.0, + "child_pid": 8230, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5846153846153846, + "recall@5": 0.48102442429365505, + "recall@10": 0.5692415490492414, + "precision@5": 0.14153846153846153, + "mrr@10": 0.3351709401709402, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 639.9, + "latency_ms_p95": 1028.2, + "latency_ms_max": 1214.5, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.15967365967365968, + "mrr@10": 0.24166666666666667 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.43333333333333335, + "mrr@10": 0.12120370370370372 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.7692307692307693, + "recall@5": 0.7692307692307693, + "recall@10": 0.8461538461538461, + "mrr@10": 0.33269230769230773 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226", + "??~'??JI???", + "b41a1b01-0075-4019-b0b5-6e639fd126e5", + "??H'??JI?#?", + "??H'??JI?[?", + "o#@-??JI4?\\", + "o#@-??JI4?\\", + "bb94bdf1-7b52-4967-ab81-9c6f01cf9665", + "7f092282-ccf5-4922-8694-08b7728f678d", + "6743e5f0-8d77-46e7-8ce0-b82e947272a1" + ], + "n_returned": 10, + "latency_ms": 161.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-175f", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-74ed" + ], + "n_returned": 10, + "latency_ms": 184.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "?o?`????", + "mem-5708f4c9-3d61-4182-8543-2843698931e6", + "%???2??jH??", + "%???2??jH??", + "%???2??jH??", + "??L?=??|Q?", + "????2??jZ??", + "project-canon" + ], + "n_returned": 10, + "latency_ms": 178.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "mem-f623328a-be55-49a0-a389-4cce5bf42509", + "bl-307c3355-528a-4adf-b191-6bbfa2419913", + "bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f", + "mem-295f7aac-6c9e-4508-8dbc-a25f101f01d6", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "mem-068e01fb-bdd0-498e-a2bb-33defa2c8bb2", + "mem-2d1ea831-cccd-4f0f-86b9-2cbbc89dc3e0", + "bl-aef42b86-cf88-4aca-9b82-dec83d69f13c" + ], + "n_returned": 10, + "latency_ms": 179.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_", + "mem-00de72b8-b532-4cc4-95eb-18e39697891c", + "project-runpod-inference", + "mem-8f6b1f0c-3dc2-42fe-8a93-5d72c8b57ced", + "bl-87b459c8-3953-478f-8384-10cc3f861424", + "bl-f918df79-dcd2-4a6e-9eae-71f94247e913", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34", + "bl-0fd8b96a-4b99-44cd-8b9b-ac91bf1e09f3" + ], + "n_returned": 10, + "latency_ms": 163.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "958ce934-a9c4-4242-afe1-d73d61c22aee", + "f2bb62a2-c2b6-4fb6-b58d-21cdecc1446f", + "cad44e60-eae9-4d4e-84c7-aaf85d4d698c", + "89aeb221-d068-4dfc-925d-d2249e744788", + "1c0effdd-baef-4523-abb8-490c3bc8b4df", + "00d2cc20-c76b-468d-8204-c624512ced28", + "eb68b995-eb4d-4f8d-b7c9-35eeba9ac781", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "??" + ], + "n_returned": 10, + "latency_ms": 176.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "3e9a651b-4b34-466d-ba45-874764a01609", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "3c633e71-2754-4273-990d-d44a6351cec3", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "cc4e7bc9-9d9b-4905-92a6-c6b99653b867", + "bl-79028eed-c330-4724-9402-734062d13503", + "14e328ad-bff1-4dc1-9ec5-dfc2bb98d968", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3" + ], + "n_returned": 10, + "latency_ms": 297.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "mem-c9bec303-a638-4a11-a490-f38410d448cf", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "10493b57-24c8-4de4-a138-a91ea13d0158", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be" + ], + "n_returned": 10, + "latency_ms": 274.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.3125, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee" + ], + "n_returned": 10, + "latency_ms": 274.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2222222222222222, + "recall@10": 0.3333333333333333, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "project-harmonic-framework-content", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework" + ], + "n_returned": 10, + "latency_ms": 264.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "e4ba1ca6-1496-49b0-b96c-a0ccf51cf821", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "??_/Pr?????" + ], + "n_returned": 10, + "latency_ms": 274.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "?Q??m?;?u?'", + "knw-cf13b883-d947-4cf8-b86b-cd9c6f0748d6" + ], + "n_returned": 10, + "latency_ms": 416.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315", + "?", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-a3c97012-5fa3-4915-a839-2c75c72005e0", + "?" + ], + "n_returned": 10, + "latency_ms": 362.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-a535f205-bc4c-4058-9171-6263c496044a" + ], + "n_returned": 10, + "latency_ms": 750.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "7452eb8f-be01-4b55-aec1-ff0c29e790f6", + "daf9d558-2265-49b0-b4fb-be0b80f3cdb1", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "d0bc22d1-073f-4264-bcc8-848ea3b7afb8" + ], + "n_returned": 10, + "latency_ms": 741.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-ce793303-c5a5-4586-a232-a3426edd9ec7", + "mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-443bd012-fc9a-4088-b236-de5157a1ef92" + ], + "n_returned": 10, + "latency_ms": 622.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-628437a6-47b0-4d81-8112-7e78499723d5", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "mem-e28e5db9-92a8-4afe-beb4-6b7a99e9309e", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783" + ], + "n_returned": 10, + "latency_ms": 553.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-f6270c76-5a1c-4ab9-ae03-b6b0c25915ec", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-e5cc63c0-8701-49d6-855a-e387fe087771", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38", + "? t?'?B?+??", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "a1000001-0000-0000-0000-000000000010" + ], + "n_returned": 10, + "latency_ms": 747.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "mem-d450f442-541a-4d3e-8c97-b6303d6cbe56", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838" + ], + "n_returned": 10, + "latency_ms": 605.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 727.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf", + "bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3", + "bl-34f51ddb-a840-459f-a248-94214f5febb6" + ], + "n_returned": 10, + "latency_ms": 608.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-738cd260-a9cf-4cdf-a494-4654c321bd79", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "bl-06c13965-082b-417d-9561-93d6e958ae5d", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c" + ], + "n_returned": 10, + "latency_ms": 739.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000001", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "ctx-4a41", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 636.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-87c93185-b2bf-40af-ae23-3c830c007abf" + ], + "n_returned": 10, + "latency_ms": 674.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "mem-37674b32-f897-4218-866c-ef281b4fe3af", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "mem-03a5e09f-bca7-4a74-a3aa-19dd3704dba8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15" + ], + "n_returned": 10, + "latency_ms": 601.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 608.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-bce80169-2d46-4b3e-9ebe-8498e26f0a89" + ], + "n_returned": 10, + "latency_ms": 654.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "mem-a5ae594a-235b-42c4-b3cf-f770e4f73761", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-a54d59a4-cc2b-405d-b68e-5b6cf2b277bc", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 539.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.23076923076923078, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-19317db1-fa05-43cd-b4e4-cb57840b7c40", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-trailer-park-paladins", + "?V?" + ], + "n_returned": 10, + "latency_ms": 533.5, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc" + ], + "n_returned": 10, + "latency_ms": 528.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "tag-hope", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 534.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.18181818181818182, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "mem-d396d789-0f7f-4366-a008-5d8801c8f2eb", + "4f698ae6-c40e-464e-9798-50350991a188" + ], + "n_returned": 10, + "latency_ms": 677.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 351.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 261.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 348.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de" + ], + "n_returned": 10, + "latency_ms": 639.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 9 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "13705072-4515-4124-963d-083af490494f", + "527ecb25-2587-47eb-8269-73be2431abd4", + "6de314bf-5c4c-4cfc-871f-fa2e422d45e6", + "a1000001-0000-0000-0000-000000000002", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "acedcec1-6555-4d95-9a68-de05bdb4e45e" + ], + "n_returned": 10, + "latency_ms": 726.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "5fcba804-eb5b-48ec-82da-146b1c6bb50d", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "ctx-3a55", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5" + ], + "n_returned": 10, + "latency_ms": 494.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "a1000001-0000-0000-0000-000000000006", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000005", + "a1000001-0000-0000-0000-000000000012", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "a1000001-0000-0000-0000-000000000010", + "mem-ca4d6a34-d354-413f-bc86-126cc17ca81c" + ], + "n_returned": 10, + "latency_ms": 626.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "0c7b9ab3-580d-45fc-8b5e-a8106f4eb449", + "bl-ae8704dc-a55f-420d-8453-671e44405a6f", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9" + ], + "n_returned": 10, + "latency_ms": 821.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "ctx-175f", + "bl-5eaddfde-e114-46b5-8ee1-b79f0af16271", + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "bl-0866d6c6-6600-4d50-b7b7-52c720cae04c", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "ctx-4a41", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4" + ], + "n_returned": 10, + "latency_ms": 866.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "mem-633ed0fc-3d10-4598-bd4a-ea372409df60", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "mem-7b0382a5-2a81-4007-85d7-a6835c5be150", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "d7e12e95-0a87-4000-b5c2-36b9be234c91", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "dc3e7b44-c345-47d0-bf80-458e471c4b8b", + "89e3f620-b8ad-423b-9ed6-fe3c7954f4b6", + "b084e4da-95bf-46e6-9ee6-bfb4864732bb" + ], + "n_returned": 10, + "latency_ms": 765.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "bl-0d01319f-f9cc-4db5-b0f1-107dba3c3a92", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "bl-672bc3c8-9ad4-4b74-8950-75854a0d9cb8", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "a36f494c-465b-4b3f-9c7c-80460cf31748", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "1d81534a-0359-4217-a4dd-d158a4841633" + ], + "n_returned": 10, + "latency_ms": 731.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "7de6a552-0878-422b-8820-bbf9101199ef", + "aacd7b32-04ed-441e-a1a9-9c5c089e79f6", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "75657c91-3c3a-4173-b3cb-e2973901fa93" + ], + "n_returned": 10, + "latency_ms": 746.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "a59185a8-7324-4fc6-b89b-b1cf7f7a0056", + "bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8", + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "bl-4f762163-04a0-452a-be05-03602dae34f2", + "bl-a99f1035-2efc-4507-84df-2df67eab13ee", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "bl-79028eed-c330-4724-9402-734062d13503", + "bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685", + "bl-b1b7e75e-f602-49f5-929d-6e9c8c65b663" + ], + "n_returned": 10, + "latency_ms": 843.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-3ae64c82-ac96-45dd-8d6a-c8cb0f7a044c", + "a1000001-0000-0000-0000-000000000012", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000008", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-983b25ad-5d73-4029-a970-6b311c24644b", + "a1000001-0000-0000-0000-000000000011", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6" + ], + "n_returned": 10, + "latency_ms": 941.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "7fdcca2c-09f7-423b-bc6a-a74cb08830d3", + "bl-9d53422d-b703-4f1d-860a-8598cb29b792", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "bl-88221dad-8dab-48f1-bebc-b5ca84a3cb69", + "ctx-fae1", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "ctx-60cc", + "ctx-5a0a", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca" + ], + "n_returned": 10, + "latency_ms": 1214.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "??f?7???", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743" + ], + "n_returned": 10, + "latency_ms": 912.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "c7dddd9c-fd3b-419e-baa2-61623b495597" + ], + "n_returned": 10, + "latency_ms": 890.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "k??A5??????", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "758ec67c-131c-447e-a352-ac7a22c3a47e" + ], + "n_returned": 10, + "latency_ms": 903.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 747.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "3711ac11-0a64-41ae-8661-ad10f95b4546", + "bl-2ade8732-2f2c-45bb-b15d-238d671426ac", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "1786f09d-e915-4910-9e1e-8061f42a0270", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "a1000001-0000-0000-0000-000000000010", + "bl-b4deb654-6361-4339-8b8e-0786607f07f7" + ], + "n_returned": 10, + "latency_ms": 994.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "mem-ef878e30-5851-4e82-8588-745415108941", + "project-Will_s_framework__Project_Design_practice___part_of_HD_bundle__sold_standalone_", + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_", + "cafe3203-6f50-4011-8167-9400f19bb196", + "project-Will_s_framework__BDT_methodology___part_of_HD_bundle__sold_standalone_", + "a1000001-0000-0000-0000-000000000006", + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "n_returned": 10, + "latency_ms": 761.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000008", + "tag-marketing-site", + "a1000001-0000-0000-0000-000000000012", + "a1000001-0000-0000-0000-000000000007", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 855.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "c9070b0a-e1d1-4307-85d1-67a1239ca604", + "bl-0547b0b1-1cd5-447c-814d-ebe525bc08e9", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "786f8e8b-ac43-42bc-9c7a-58743b37f81f", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "31d7c4bf-843b-439f-a155-56c1404f9122", + "96497334-b18f-495c-9228-eeb8182bdc38", + "bl-f925cf8a-cd34-4ce6-a8f9-3d201738c1a4" + ], + "n_returned": 10, + "latency_ms": 818.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "bl-da818960-b360-455d-a9c1-e9a215c33ca4", + "bl-a96c0fc9-a533-4fea-b65f-0e46ac47bbd4", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "bl-8e0ddc08-cb74-4b02-96b0-847a95e9dae8", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-2ff9c669-a9fc-4ef7-9a18-9ec7167639c8" + ], + "n_returned": 10, + "latency_ms": 764.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "mem-5dd5f3b9-3c7e-44b1-84ce-2e36925f17fb", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2" + ], + "n_returned": 10, + "latency_ms": 1028.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-f1f71f3c-ca7d-4f98-a569-c8a9aa67a000", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "2f7270cc-4cda-4499-831f-1fbdc2f23284", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6" + ], + "n_returned": 10, + "latency_ms": 849.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "8179f20c-47de-47ba-a3d7-b32154391c19", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "d69b441e-0031-4d82-abfd-ac503b637562", + "kn-cf8fa517-fc83-47c4-8fb7-bd9e26410dd3", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "7c90d01b-8d53-4e75-b494-73011b71fa6a" + ], + "n_returned": 10, + "latency_ms": 1036.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "mem-fc3a6f33-3a6d-4fa3-a1cd-62086c558dfd", + "a1000001-0000-0000-0000-000000000012", + "e8a5ff51-bf57-400b-9840-cb2822c5bd10", + "mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "a1000001-0000-0000-0000-000000000002", + "1b066de3-3195-40dd-89c3-4197477f98af", + "c34f6292-b6e5-4619-a553-90ae4fc498a2", + "a1000001-0000-0000-0000-000000000004", + "31d7c4bf-843b-439f-a155-56c1404f9122" + ], + "n_returned": 10, + "latency_ms": 632.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "954c9597-ab7a-41e6-b085-bf838bba4f17", + "art-d0ded6f5-e202-47ad-8fb3-f6bd80f35052", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-9329958a-d459-499c-b516-e51019c693e0" + ], + "n_returned": 10, + "latency_ms": 899.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "k??A5??????", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "8c40420c-c891-4ac9-a039-88440df87180", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "66d8d8c7-dea4-4245-8e68-d1ab4f4346b8" + ], + "n_returned": 10, + "latency_ms": 1059.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "ctx-45bc", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "ctx-175f", + "ctx-fae1", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "87ab5b46-4fe1-4e1d-86c6-d437e17682ab", + "ctx-60cc", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f" + ], + "n_returned": 10, + "latency_ms": 1032.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "8d91620b-9d0a-4ba1-b066-e57d1d8bc7d3", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "d6a745c3-fb56-4c0f-872c-a3aa69f8b12f", + "db836e65-0cdc-4232-b8d7-f973ab279ea9", + "project-Current_instance_live_until_new_verified_healthy__Liveness_checks__cutover__rollback_", + "6df650b5-a14d-42f8-88dd-0919ad20b77a", + "f169bf40-51ab-497d-8085-e57d374b77d4", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-cb6edb16-24ed-4888-b6d2-5b23c67748b3" + ], + "n_returned": 10, + "latency_ms": 983.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-8e0ddc08-cb74-4b02-96b0-847a95e9dae8", + "bl-212774e1-4ed4-4893-9265-f98d9eddeb3e", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "5b422430-2503-42d7-a315-9dfaaa6472ae", + "bl-37025640-9e7b-44c5-ad1e-615d813bfd60", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70" + ], + "n_returned": 10, + "latency_ms": 951.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-32c3735e-7116-4edf-85fb-1f0a229b463a", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "mem-96db70f6-7ede-424f-8b58-47a3b5ac759f", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "tag-reaction-order" + ], + "n_returned": 10, + "latency_ms": 690.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "aa3fb542-87f2-4eef-8e95-a870bfd0318a", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "d43a8137-c3c3-49e2-ad11-89d897280761" + ], + "n_returned": 10, + "latency_ms": 707.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-556438af-57b2-4bd8-a747-9f868aaee290", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "66c63082-b4da-4aa1-8fee-848db8a83210" + ], + "n_returned": 10, + "latency_ms": 876.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 365.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 266.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 355.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 258.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 353.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 257.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 256.0, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-splitfix.json b/tools/retrieval-eval/results-splitfix.json new file mode 100644 index 0000000..2f83c6d --- /dev/null +++ b/tools/retrieval-eval/results-splitfix.json @@ -0,0 +1,1825 @@ +{ + "label": "splitfix", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-splitfix", + "soul_md5": "b0a5279aa2c2f4ba0eb9c0b36ca844b1", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semsub/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7893, + "wall_clock_s": 55.9, + "child_pid": 8184, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5846153846153846, + "recall@5": 0.48102442429365505, + "recall@10": 0.5692415490492414, + "precision@5": 0.14153846153846153, + "mrr@10": 0.3351709401709402, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 646.9, + "latency_ms_p95": 1028.5, + "latency_ms_max": 1197.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.15967365967365968, + "mrr@10": 0.24166666666666667 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.43333333333333335, + "mrr@10": 0.12120370370370372 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.7692307692307693, + "recall@5": 0.7692307692307693, + "recall@10": 0.8461538461538461, + "mrr@10": 0.33269230769230773 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226", + "??~'??JI???", + "b41a1b01-0075-4019-b0b5-6e639fd126e5", + "??H'??JI?#?", + "??H'??JI?[?", + "o#@-??JI4?\\", + "o#@-??JI4?\\", + "bb94bdf1-7b52-4967-ab81-9c6f01cf9665", + "7f092282-ccf5-4922-8694-08b7728f678d", + "6743e5f0-8d77-46e7-8ce0-b82e947272a1" + ], + "n_returned": 10, + "latency_ms": 189.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-175f", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-74ed" + ], + "n_returned": 10, + "latency_ms": 167.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "?o?`????", + "mem-5708f4c9-3d61-4182-8543-2843698931e6", + "%???2??jH??", + "%???2??jH??", + "%???2??jH??", + "??L?=??|Q?", + "????2??jZ??", + "project-canon" + ], + "n_returned": 10, + "latency_ms": 181.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "mem-f623328a-be55-49a0-a389-4cce5bf42509", + "bl-307c3355-528a-4adf-b191-6bbfa2419913", + "bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f", + "mem-295f7aac-6c9e-4508-8dbc-a25f101f01d6", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "mem-068e01fb-bdd0-498e-a2bb-33defa2c8bb2", + "mem-2d1ea831-cccd-4f0f-86b9-2cbbc89dc3e0", + "bl-aef42b86-cf88-4aca-9b82-dec83d69f13c" + ], + "n_returned": 10, + "latency_ms": 182.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_", + "mem-00de72b8-b532-4cc4-95eb-18e39697891c", + "project-runpod-inference", + "mem-8f6b1f0c-3dc2-42fe-8a93-5d72c8b57ced", + "bl-87b459c8-3953-478f-8384-10cc3f861424", + "bl-f918df79-dcd2-4a6e-9eae-71f94247e913", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34", + "bl-0fd8b96a-4b99-44cd-8b9b-ac91bf1e09f3" + ], + "n_returned": 10, + "latency_ms": 164.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "958ce934-a9c4-4242-afe1-d73d61c22aee", + "f2bb62a2-c2b6-4fb6-b58d-21cdecc1446f", + "cad44e60-eae9-4d4e-84c7-aaf85d4d698c", + "89aeb221-d068-4dfc-925d-d2249e744788", + "1c0effdd-baef-4523-abb8-490c3bc8b4df", + "00d2cc20-c76b-468d-8204-c624512ced28", + "eb68b995-eb4d-4f8d-b7c9-35eeba9ac781", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "??" + ], + "n_returned": 10, + "latency_ms": 165.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "3e9a651b-4b34-466d-ba45-874764a01609", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "3c633e71-2754-4273-990d-d44a6351cec3", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "cc4e7bc9-9d9b-4905-92a6-c6b99653b867", + "bl-79028eed-c330-4724-9402-734062d13503", + "14e328ad-bff1-4dc1-9ec5-dfc2bb98d968", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3" + ], + "n_returned": 10, + "latency_ms": 282.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "mem-c9bec303-a638-4a11-a490-f38410d448cf", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "157ba9fe-e919-4f2c-a42d-aa7ecfd3a27b", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be" + ], + "n_returned": 10, + "latency_ms": 258.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.3125, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee" + ], + "n_returned": 10, + "latency_ms": 259.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2222222222222222, + "recall@10": 0.3333333333333333, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "project-harmonic-framework-content", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework" + ], + "n_returned": 10, + "latency_ms": 252.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "e4ba1ca6-1496-49b0-b96c-a0ccf51cf821", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "??_/Pr?????" + ], + "n_returned": 10, + "latency_ms": 260.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "?Q??m?;?u?'", + "knw-cf13b883-d947-4cf8-b86b-cd9c6f0748d6" + ], + "n_returned": 10, + "latency_ms": 410.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315", + "?", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-a3c97012-5fa3-4915-a839-2c75c72005e0", + "?" + ], + "n_returned": 10, + "latency_ms": 383.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-a535f205-bc4c-4058-9171-6263c496044a" + ], + "n_returned": 10, + "latency_ms": 732.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "7452eb8f-be01-4b55-aec1-ff0c29e790f6", + "daf9d558-2265-49b0-b4fb-be0b80f3cdb1", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "d0bc22d1-073f-4264-bcc8-848ea3b7afb8" + ], + "n_returned": 10, + "latency_ms": 735.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-ce793303-c5a5-4586-a232-a3426edd9ec7", + "mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-443bd012-fc9a-4088-b236-de5157a1ef92" + ], + "n_returned": 10, + "latency_ms": 624.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-628437a6-47b0-4d81-8112-7e78499723d5", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "mem-e28e5db9-92a8-4afe-beb4-6b7a99e9309e", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783" + ], + "n_returned": 10, + "latency_ms": 542.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-f6270c76-5a1c-4ab9-ae03-b6b0c25915ec", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-e5cc63c0-8701-49d6-855a-e387fe087771", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38", + "? t?'?B?+??", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "a1000001-0000-0000-0000-000000000010" + ], + "n_returned": 10, + "latency_ms": 725.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "mem-d450f442-541a-4d3e-8c97-b6303d6cbe56", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838" + ], + "n_returned": 10, + "latency_ms": 564.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 684.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf", + "bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3", + "bl-34f51ddb-a840-459f-a248-94214f5febb6" + ], + "n_returned": 10, + "latency_ms": 589.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-738cd260-a9cf-4cdf-a494-4654c321bd79", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "bl-06c13965-082b-417d-9561-93d6e958ae5d", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c" + ], + "n_returned": 10, + "latency_ms": 721.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000001", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "ctx-4a41", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 632.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-87c93185-b2bf-40af-ae23-3c830c007abf" + ], + "n_returned": 10, + "latency_ms": 671.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "mem-37674b32-f897-4218-866c-ef281b4fe3af", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "mem-03a5e09f-bca7-4a74-a3aa-19dd3704dba8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15" + ], + "n_returned": 10, + "latency_ms": 572.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 598.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-bce80169-2d46-4b3e-9ebe-8498e26f0a89" + ], + "n_returned": 10, + "latency_ms": 645.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "mem-a5ae594a-235b-42c4-b3cf-f770e4f73761", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-a54d59a4-cc2b-405d-b68e-5b6cf2b277bc", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 554.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.23076923076923078, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-19317db1-fa05-43cd-b4e4-cb57840b7c40", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-trailer-park-paladins", + "?V?" + ], + "n_returned": 10, + "latency_ms": 550.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc" + ], + "n_returned": 10, + "latency_ms": 544.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "tag-hope", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 554.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.18181818181818182, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "mem-d396d789-0f7f-4366-a008-5d8801c8f2eb", + "4f698ae6-c40e-464e-9798-50350991a188" + ], + "n_returned": 10, + "latency_ms": 698.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 368.8, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 279.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 367.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de" + ], + "n_returned": 10, + "latency_ms": 660.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 9 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "13705072-4515-4124-963d-083af490494f", + "527ecb25-2587-47eb-8269-73be2431abd4", + "6de314bf-5c4c-4cfc-871f-fa2e422d45e6", + "a1000001-0000-0000-0000-000000000002", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "acedcec1-6555-4d95-9a68-de05bdb4e45e" + ], + "n_returned": 10, + "latency_ms": 738.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "5fcba804-eb5b-48ec-82da-146b1c6bb50d", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "ctx-3a55", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5" + ], + "n_returned": 10, + "latency_ms": 494.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "a1000001-0000-0000-0000-000000000006", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000005", + "a1000001-0000-0000-0000-000000000012", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "a1000001-0000-0000-0000-000000000010", + "mem-ca4d6a34-d354-413f-bc86-126cc17ca81c" + ], + "n_returned": 10, + "latency_ms": 646.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "0c7b9ab3-580d-45fc-8b5e-a8106f4eb449", + "bl-ae8704dc-a55f-420d-8453-671e44405a6f", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9" + ], + "n_returned": 10, + "latency_ms": 825.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "ctx-175f", + "bl-5eaddfde-e114-46b5-8ee1-b79f0af16271", + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "bl-0866d6c6-6600-4d50-b7b7-52c720cae04c", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "ctx-4a41", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4" + ], + "n_returned": 10, + "latency_ms": 862.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "mem-633ed0fc-3d10-4598-bd4a-ea372409df60", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "mem-7b0382a5-2a81-4007-85d7-a6835c5be150", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "d7e12e95-0a87-4000-b5c2-36b9be234c91", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "dc3e7b44-c345-47d0-bf80-458e471c4b8b", + "89e3f620-b8ad-423b-9ed6-fe3c7954f4b6", + "b084e4da-95bf-46e6-9ee6-bfb4864732bb" + ], + "n_returned": 10, + "latency_ms": 767.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "bl-0d01319f-f9cc-4db5-b0f1-107dba3c3a92", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "bl-672bc3c8-9ad4-4b74-8950-75854a0d9cb8", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "a36f494c-465b-4b3f-9c7c-80460cf31748", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "1d81534a-0359-4217-a4dd-d158a4841633" + ], + "n_returned": 10, + "latency_ms": 720.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "7de6a552-0878-422b-8820-bbf9101199ef", + "aacd7b32-04ed-441e-a1a9-9c5c089e79f6", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "75657c91-3c3a-4173-b3cb-e2973901fa93" + ], + "n_returned": 10, + "latency_ms": 731.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "a59185a8-7324-4fc6-b89b-b1cf7f7a0056", + "bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8", + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "bl-4f762163-04a0-452a-be05-03602dae34f2", + "bl-a99f1035-2efc-4507-84df-2df67eab13ee", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "bl-79028eed-c330-4724-9402-734062d13503", + "bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685", + "bl-b1b7e75e-f602-49f5-929d-6e9c8c65b663" + ], + "n_returned": 10, + "latency_ms": 819.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-3ae64c82-ac96-45dd-8d6a-c8cb0f7a044c", + "a1000001-0000-0000-0000-000000000012", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000008", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-983b25ad-5d73-4029-a970-6b311c24644b", + "a1000001-0000-0000-0000-000000000011", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6" + ], + "n_returned": 10, + "latency_ms": 872.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "7fdcca2c-09f7-423b-bc6a-a74cb08830d3", + "bl-9d53422d-b703-4f1d-860a-8598cb29b792", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "bl-88221dad-8dab-48f1-bebc-b5ca84a3cb69", + "ctx-fae1", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "ctx-60cc", + "ctx-5a0a", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca" + ], + "n_returned": 10, + "latency_ms": 1197.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "??f?7???", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743" + ], + "n_returned": 10, + "latency_ms": 911.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "c7dddd9c-fd3b-419e-baa2-61623b495597" + ], + "n_returned": 10, + "latency_ms": 895.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "k??A5??????", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "758ec67c-131c-447e-a352-ac7a22c3a47e" + ], + "n_returned": 10, + "latency_ms": 906.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 743.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "3711ac11-0a64-41ae-8661-ad10f95b4546", + "bl-2ade8732-2f2c-45bb-b15d-238d671426ac", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "1786f09d-e915-4910-9e1e-8061f42a0270", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "a1000001-0000-0000-0000-000000000010", + "bl-b4deb654-6361-4339-8b8e-0786607f07f7" + ], + "n_returned": 10, + "latency_ms": 993.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "mem-ef878e30-5851-4e82-8588-745415108941", + "project-Will_s_framework__Project_Design_practice___part_of_HD_bundle__sold_standalone_", + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_", + "cafe3203-6f50-4011-8167-9400f19bb196", + "project-Will_s_framework__BDT_methodology___part_of_HD_bundle__sold_standalone_", + "a1000001-0000-0000-0000-000000000006", + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "n_returned": 10, + "latency_ms": 761.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000008", + "tag-marketing-site", + "a1000001-0000-0000-0000-000000000012", + "a1000001-0000-0000-0000-000000000007", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 855.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "c9070b0a-e1d1-4307-85d1-67a1239ca604", + "bl-0547b0b1-1cd5-447c-814d-ebe525bc08e9", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "786f8e8b-ac43-42bc-9c7a-58743b37f81f", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "31d7c4bf-843b-439f-a155-56c1404f9122", + "96497334-b18f-495c-9228-eeb8182bdc38", + "bl-f925cf8a-cd34-4ce6-a8f9-3d201738c1a4" + ], + "n_returned": 10, + "latency_ms": 835.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "bl-da818960-b360-455d-a9c1-e9a215c33ca4", + "bl-a96c0fc9-a533-4fea-b65f-0e46ac47bbd4", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "bl-8e0ddc08-cb74-4b02-96b0-847a95e9dae8", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-2ff9c669-a9fc-4ef7-9a18-9ec7167639c8" + ], + "n_returned": 10, + "latency_ms": 782.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "mem-5dd5f3b9-3c7e-44b1-84ce-2e36925f17fb", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2" + ], + "n_returned": 10, + "latency_ms": 1039.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-f1f71f3c-ca7d-4f98-a569-c8a9aa67a000", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "2f7270cc-4cda-4499-831f-1fbdc2f23284", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6" + ], + "n_returned": 10, + "latency_ms": 854.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "8179f20c-47de-47ba-a3d7-b32154391c19", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "d69b441e-0031-4d82-abfd-ac503b637562", + "kn-cf8fa517-fc83-47c4-8fb7-bd9e26410dd3", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "7c90d01b-8d53-4e75-b494-73011b71fa6a" + ], + "n_returned": 10, + "latency_ms": 1029.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "mem-fc3a6f33-3a6d-4fa3-a1cd-62086c558dfd", + "a1000001-0000-0000-0000-000000000012", + "e8a5ff51-bf57-400b-9840-cb2822c5bd10", + "mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "a1000001-0000-0000-0000-000000000002", + "1b066de3-3195-40dd-89c3-4197477f98af", + "c34f6292-b6e5-4619-a553-90ae4fc498a2", + "a1000001-0000-0000-0000-000000000004", + "31d7c4bf-843b-439f-a155-56c1404f9122" + ], + "n_returned": 10, + "latency_ms": 617.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "954c9597-ab7a-41e6-b085-bf838bba4f17", + "art-d0ded6f5-e202-47ad-8fb3-f6bd80f35052", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-9329958a-d459-499c-b516-e51019c693e0" + ], + "n_returned": 10, + "latency_ms": 884.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "k??A5??????", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "8c40420c-c891-4ac9-a039-88440df87180", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "66d8d8c7-dea4-4245-8e68-d1ab4f4346b8" + ], + "n_returned": 10, + "latency_ms": 1071.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "ctx-45bc", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "ctx-175f", + "ctx-fae1", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "87ab5b46-4fe1-4e1d-86c6-d437e17682ab", + "ctx-60cc", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f" + ], + "n_returned": 10, + "latency_ms": 1028.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "8d91620b-9d0a-4ba1-b066-e57d1d8bc7d3", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "d6a745c3-fb56-4c0f-872c-a3aa69f8b12f", + "db836e65-0cdc-4232-b8d7-f973ab279ea9", + "project-Current_instance_live_until_new_verified_healthy__Liveness_checks__cutover__rollback_", + "6df650b5-a14d-42f8-88dd-0919ad20b77a", + "f169bf40-51ab-497d-8085-e57d374b77d4", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-cb6edb16-24ed-4888-b6d2-5b23c67748b3" + ], + "n_returned": 10, + "latency_ms": 976.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-8e0ddc08-cb74-4b02-96b0-847a95e9dae8", + "bl-212774e1-4ed4-4893-9265-f98d9eddeb3e", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "5b422430-2503-42d7-a315-9dfaaa6472ae", + "bl-37025640-9e7b-44c5-ad1e-615d813bfd60", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70" + ], + "n_returned": 10, + "latency_ms": 954.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "mem-d3a4a876-9207-42c4-8a58-25cffa05d743", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-32c3735e-7116-4edf-85fb-1f0a229b463a", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "mem-96db70f6-7ede-424f-8b58-47a3b5ac759f", + "bl-7a38d618-34a0-4d8c-a018-13c81851dd46", + "tag-reaction-order" + ], + "n_returned": 10, + "latency_ms": 680.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "aa3fb542-87f2-4eef-8e95-a870bfd0318a", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "d43a8137-c3c3-49e2-ad11-89d897280761" + ], + "n_returned": 10, + "latency_ms": 689.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-556438af-57b2-4bd8-a747-9f868aaee290", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "66c63082-b4da-4aa1-8fee-848db8a83210" + ], + "n_returned": 10, + "latency_ms": 854.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 352.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 262.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 349.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 258.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 354.9, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 256.9, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 256.7, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 4d06d29..67d829e 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -7175,6 +7175,12 @@ void engram_forget(el_val_t node_id) { if (idx < 0) return; /* Free node strings */ EngramNode* n = &g->nodes[idx]; + if (getenv("EG_DIAG")) { + fprintf(stderr, "[EG_DIAG] FORGET id=%s type=%s layer=%u label=%s\n", + sid, n->node_type ? n->node_type : "?", n->layer_id, + n->label ? n->label : "?"); + fflush(stderr); + } free(n->id); free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); free(n->emb); @@ -7270,6 +7276,8 @@ el_val_t engram_prune_telemetry(el_val_t older_than_ms) { } } g->node_count = w; + if (getenv("EG_DIAG")) + fprintf(stderr, "[EG_DIAG] PRUNE_TELEMETRY removed=%lld\n", (long long)removed); if (removed == 0) { free(removed_ids); return 0; } /* Removed-id hash set (open addressing, power-of-two >= 2*removed). */ @@ -9304,6 +9312,26 @@ el_val_t engram_load(el_val_t path) { } } g->adj_dirty = 1; + if (getenv("EG_DIAG")) { + int64_t we = 0, wrongdim = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].emb) { we++; if (g->nodes[i].emb_dim != 768) wrongdim++; } + } + fprintf(stderr, "[EG_DIAG] loaded nodes=%lld with_emb=%lld wrongdim=%lld\n", + (long long)g->node_count, (long long)we, (long long)wrongdim); + const char* probe = getenv("EG_DIAG_ID"); + if (probe) { + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].id && strcmp(g->nodes[i].id, probe) == 0) { + fprintf(stderr, "[EG_DIAG] probe id=%s idx=%lld emb=%p dim=%d layer=%u addr=%d\n", + probe, (long long)i, (void*)g->nodes[i].emb, + (int)g->nodes[i].emb_dim, g->nodes[i].layer_id, + eg_node_addressable(&g->nodes[i])); + } + } + } + fflush(stderr); + } /* Walk edges array */ const char* edges_p = json_find_key(data, "edges"); if (edges_p) { @@ -9624,7 +9652,33 @@ el_val_t engram_get_node_by_label(el_val_t label) { return el_wrap_str(el_strdup("{}")); } -el_val_t engram_search_json(el_val_t query, el_val_t limit) { +/* ── THE SEARCH / RECALL BOUNDARY (2026-08-07) ─────────────────────────────── + * engram_search_json is the LEXICAL function ~40 .el call sites already + * depend on: they pass a key-shaped string ("soul:boot_count", + * "soul-inbox-pending", a session label) and treat every returned record as + * a record that CONTAINS that key. Seven of those sites then delete what + * comes back (memory.el:176, sessions.el:250/268/444/523, soul.el:359 — + * "prune all existing X nodes, keep exactly one"). + * + * The semantic and associative legs must therefore NOT live on this + * function. Claim 24 authorises the vector index "to respond to EMBEDDING + * SEARCH QUERIES by returning the node records whose embedding vectors have + * the highest cosine similarity to a query vector"; a keyed state read is + * not an embedding search query, it is the identifier-keyed retrieval of + * claim 23 ("node records are stored under a key encoding the node + * identifier"). Putting both behind one function erased that boundary, and + * a nearest neighbour of the string "soul:boot_count" is not a boot counter. + * + * MEASURED, on the harness corpus, isolated, read-only, no writes from any + * caller: 240 node records destroyed per boot, including 6 Knowledge nodes, + * a layer-1 "CORE IDENTITY — GENESIS, LINEAGE" Memory, and the value node + * `kn-58874a74` (gold answer for gold-set q15). The deletion list is the + * result list of the soul's own mem_boot_count_inc() lookup, in order. + * + * So: legs OFF here, legs ON in engram_recall_json below, which is what + * /api/neuron/recall reaches. Retrieval quality on the recall route is + * unchanged; the internal keyed reads get their contract back. */ +static el_val_t eg_search_json_impl(el_val_t query, el_val_t limit, int with_legs) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t lim = (int64_t)limit; @@ -9646,7 +9700,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { * so the semantic half of the retrieval surface has to land HERE * to be observable to the MCP wrapper and the app. */ int32_t qdim = 0; - float* qv = eg_embed_fetch(q, &qdim); + float* qv = with_legs ? eg_embed_fetch(q, &qdim) : NULL; EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; int64_t nsem = 0; int64_t nhits = 0; @@ -9751,6 +9805,31 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { } qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp); if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp); + if (getenv("EG_DIAG")) { + int64_t we = 0, unaddr = 0, found = 0; + const char* pid = getenv("EG_DIAG_ID"); + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].emb) we++; + if (!eg_node_addressable(&g->nodes[i])) unaddr++; + if (pid && g->nodes[i].id && strcmp(g->nodes[i].id, pid) == 0) found++; + } + fprintf(stderr, "[EG_DIAG] STORE node_count=%lld with_emb=%lld unaddressable=%lld probe_found=%lld\n", + (long long)g->node_count, (long long)we, (long long)unaddr, (long long)found); + fprintf(stderr, "[EG_DIAG] q=\"%s\" qdim=%d nhits=%lld nsem=%lld\n", + q, (int)qdim, (long long)nhits, (long long)nsem); + for (int64_t k = 0; k < 5 && k < nsem; k++) + fprintf(stderr, "[EG_DIAG] sem[%lld] cos=%.4f id=%s\n", + (long long)k, sem[k].sem, g->nodes[sem[k].idx].id); + const char* probe = getenv("EG_DIAG_ID"); + if (probe) for (int64_t k = 0; k < nsem; k++) + if (g->nodes[sem[k].idx].id + && strcmp(g->nodes[sem[k].idx].id, probe) == 0) { + fprintf(stderr, "[EG_DIAG] probe at sem rank %lld cos=%.4f\n", + (long long)k, sem[k].sem); + break; + } + fflush(stderr); + } /* Claim-10 associative leg: expand the top lexical hits along * structural relations only, order the reached set by query * similarity. Empty whenever the seeds have no structural @@ -9795,6 +9874,19 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { return el_wrap_str(b.buf); } +/* Lexical keyed read — the historical contract every internal caller relies + * on. Every returned record CONTAINS a query token. */ +el_val_t engram_search_json(el_val_t query, el_val_t limit) { + return eg_search_json_impl(query, limit, 0); +} + +/* The retrieval surface: lexical + claim-24 semantic + claim-10 associative, + * rank-fused. Reached from handle_api_recall (/api/neuron/recall) — the route + * the MCP wrapper and the app call, and the one the eval harness measures. */ +el_val_t engram_recall_json(el_val_t query, el_val_t limit) { + return eg_search_json_impl(query, limit, 1); +} + el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) { EngramStore* g = engram_get(); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.h b/vendor/el-runtime/v1.0.0-20260501/el_runtime.h index 88da992..dfb22da 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.h +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.h @@ -612,6 +612,7 @@ el_val_t engram_load(el_val_t path); el_val_t engram_get_node_json(el_val_t id); el_val_t engram_get_node_by_label(el_val_t label); el_val_t engram_search_json(el_val_t query, el_val_t limit); +el_val_t engram_recall_json(el_val_t query, el_val_t limit); el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset); el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);