From 9717a4eeaf88141602820c78de3dce010aed473f Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 7 Aug 2026 17:45:05 -0500 Subject: [PATCH] measure: claim 24 unflooring is +4 (NOT-SHOWN); asymmetric embedding prefixes are -5 (discarded) Measured on the 75-query extended gold set (iteration 8's held-out extension) against the certified stack baseline results-stack-ext.json, on the embedded corpus. Three runs of the candidate, zero drift. A. CLAIM 24 WITHOUT THE THRESHOLD - net +4, NOT-SHOWN, kept in the tree. fixed : q14, q25 (in-sample paraphrase), q43, q52, q63, q67 (held-out) broken : q15 (paraphrase), q28 (associative) 8 discordant, McNemar exact p = 0.2891, floor is 6. heldout_paraphrase 16.7% -> 30.0%, paraphrase 61.5% -> 69.2%. Every regression guard held: exact_rare 6/6, phrase 7/7, nonsense 10/10, superseded 2/3. Latency FLAT: p50 641 -> 632 ms. The in-sample half (+q14 +q25 -q15 -q28 = 0) was already on record in iteration 7's cmp-nogate.json, so only the held-out +4 is new. B. ASYMMETRIC TASK PREFIXES ON THE EMBEDDER - net -5, REVERTED in this commit. Rationale was sound and the prediction was wrong, which is why it was worth measuring: nomic-embed-text is an asymmetric retrieval encoder and this file embedded query and document bare on both sides. Prefixing does exactly what the model card implies for the far-away cases - it rescued q42 (gold at GLOBAL COSINE RANK 25,564) and q39 - but it re-ranks the whole space and broke more than it fixed: fixed : q24, q39, q42 broken : q18, q19, q22, q31, q43, q44, q52, q63 heldout_paraphrase 30.0% -> 23.3%, paraphrase 69.2% -> 53.8%. The corpus and the reproducer are kept (embed-corpus-prefixed.py, snapshot-pre-repair-20260806-embedded-prefixed.json) so nobody re-runs it. Co-Authored-By: Claude Opus 5 (1M context) --- .../comparison-stack-vs-unfloor.json | 8 +- .../comparison-unfloor-vs-semsub.json | 158 ++ tools/retrieval-eval/results-semsub.json | 1825 +++++++++++++++++ .../retrieval-eval/results-unfloor-clean.json | 1825 +++++++++++++++++ tools/retrieval-eval/results-unfloor-r2.json | 1825 +++++++++++++++++ .../el-runtime/v1.0.0-20260501/el_runtime.c | 36 +- 6 files changed, 5646 insertions(+), 31 deletions(-) create mode 100644 tools/retrieval-eval/comparison-unfloor-vs-semsub.json create mode 100644 tools/retrieval-eval/results-semsub.json create mode 100644 tools/retrieval-eval/results-unfloor-clean.json create mode 100644 tools/retrieval-eval/results-unfloor-r2.json diff --git a/tools/retrieval-eval/comparison-stack-vs-unfloor.json b/tools/retrieval-eval/comparison-stack-vs-unfloor.json index 07289d2..9a459c7 100644 --- a/tools/retrieval-eval/comparison-stack-vs-unfloor.json +++ b/tools/retrieval-eval/comparison-stack-vs-unfloor.json @@ -1,6 +1,6 @@ { "baseline": "stack-ext", - "candidate": "unfloor-ext", + "candidate": "unfloor-clean", "n_shared_queries": 75, "fixed_by_candidate": [ "q14", @@ -96,9 +96,9 @@ "mrr@10": 0.32437728937728944, "nonsense_clean": "10/10", "superseded_outranks": "2/3", - "latency_ms_p50": 993.0, - "latency_ms_p95": 1544.7, - "latency_ms_max": 1809.0, + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, "errors": 0, "by_category": { "associative": { diff --git a/tools/retrieval-eval/comparison-unfloor-vs-semsub.json b/tools/retrieval-eval/comparison-unfloor-vs-semsub.json new file mode 100644 index 0000000..eaec117 --- /dev/null +++ b/tools/retrieval-eval/comparison-unfloor-vs-semsub.json @@ -0,0 +1,158 @@ +{ + "baseline": "unfloor-clean", + "candidate": "semsub", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q24", + "q39", + "q42" + ], + "broken_by_candidate": [ + "q18", + "q19", + "q22", + "q31", + "q43", + "q44", + "q52", + "q63" + ], + "discordant": 11, + "net_queries": -5, + "mcnemar_exact_p": 0.2265625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.46153846153846156, + "recall@5": 0.3889430014430015, + "recall@10": 0.4887681762681762, + "precision@5": 0.12307692307692313, + "mrr@10": 0.30181318681318675, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 634.4, + "latency_ms_p95": 988.3, + "latency_ms_max": 1184.5, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.3333333333333333, + "recall@5": 0.06060606060606061, + "recall@10": 0.12121212121212122, + "mrr@10": 0.19047619047619047 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.23333333333333334, + "recall@5": 0.23333333333333334, + "recall@10": 0.3, + "mrr@10": 0.08925925925925927 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.5384615384615384, + "recall@5": 0.5384615384615384, + "recall@10": 0.7692307692307693, + "mrr@10": 0.26324786324786326 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5596655328798186, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-semsub.json b/tools/retrieval-eval/results-semsub.json new file mode 100644 index 0000000..453a9c3 --- /dev/null +++ b/tools/retrieval-eval/results-semsub.json @@ -0,0 +1,1825 @@ +{ + "label": "semsub", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-semsub", + "soul_md5": "bdae019a3b1a2a8dc9c02621e226bdb4", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded-prefixed.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semsub/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7895, + "wall_clock_s": 57.1, + "child_pid": 4246, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.46153846153846156, + "recall@5": 0.3889430014430015, + "recall@10": 0.4887681762681762, + "precision@5": 0.12307692307692313, + "mrr@10": 0.30181318681318675, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 634.4, + "latency_ms_p95": 988.3, + "latency_ms_max": 1184.5, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.3333333333333333, + "recall@5": 0.06060606060606061, + "recall@10": 0.12121212121212122, + "mrr@10": 0.19047619047619047 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.23333333333333334, + "recall@5": 0.23333333333333334, + "recall@10": 0.3, + "mrr@10": 0.08925925925925927 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.5384615384615384, + "recall@5": 0.5384615384615384, + "recall@10": 0.7692307692307693, + "mrr@10": 0.26324786324786326 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5596655328798186, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226", + "b41a1b01-0075-4019-b0b5-6e639fd126e5", + "0604a933-e6a2-4b44-86b2-054ca38e23fc", + "6743e5f0-8d77-46e7-8ce0-b82e947272a1", + "7f092282-ccf5-4922-8694-08b7728f678d", + "696e609c-da7a-4394-8a0c-106ba07dc6c3", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "3165a3a8-339a-4679-95d9-c7011862de08", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "169e086f-cf91-4ada-b081-cd2a433c7a8f" + ], + "n_returned": 10, + "latency_ms": 159.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "mem-392ee148-6d08-49ed-a954-11774d8cdc5b", + "mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48", + "97923e18-27b4-4307-b8d6-8fe27d5a816c", + "bl-3da6e51b-de57-4226-bab2-994f5687d0f5", + "870ede67-3454-4e00-9988-46cb13a8a4e2", + "mem-6c8b5845-afff-4774-b6a2-db99615a25b1", + "mem-3fe9cfd8-3819-4d93-8b28-dc729636397f" + ], + "n_returned": 10, + "latency_ms": 160.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091", + "mem-5708f4c9-3d61-4182-8543-2843698931e6", + "bl-c9adb8e5-293f-4033-99f8-0405c17ef941", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34", + "2d00f029-a990-45ee-bf39-cd102e8dabba", + "mem-a388740d-52c7-42ae-b7e7-830768362feb", + "mem-ebc744f2-17d9-48e5-8ed3-12840498469b", + "mem-6733e3a0-23f4-4d2f-b70d-0be81ebe8090" + ], + "n_returned": 10, + "latency_ms": 162.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "bl-b05f9a75-ad6b-4929-a9a3-32df17a1f4bf", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "bl-2483e070-1385-4dbe-a49d-86c1af81e7d1", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "mem-f623328a-be55-49a0-a389-4cce5bf42509" + ], + "n_returned": 10, + "latency_ms": 160.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_", + "bl-a96c0fc9-a533-4fea-b65f-0e46ac47bbd4", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "bl-c5591654-2371-499c-80ce-991bfb53ce6f", + "mem-0d6a3b65-488d-4219-beb2-af300ca534e1", + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-00de72b8-b532-4cc4-95eb-18e39697891c", + "bl-87b459c8-3953-478f-8384-10cc3f861424", + "bl-0fd8b96a-4b99-44cd-8b9b-ac91bf1e09f3" + ], + "n_returned": 10, + "latency_ms": 161.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "958ce934-a9c4-4242-afe1-d73d61c22aee", + "f2bb62a2-c2b6-4fb6-b58d-21cdecc1446f", + "f2543845-905d-42ad-8d02-fedb8f016e3a", + "8fb73177-b6d9-4008-a6df-6cb6d7d587f3", + "7b38b4aa-661c-4ad1-a8f7-17fabffe1da5", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "00d2cc20-c76b-468d-8204-c624512ced28", + "eb68b995-eb4d-4f8d-b7c9-35eeba9ac781", + "766aedd1-2424-4112-8606-163965ae77b5" + ], + "n_returned": 10, + "latency_ms": 162.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "ea916ff4-82e7-41f1-aebe-d48d2ee643f0", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "7776a0d2-1d96-425b-86c8-b30950ec8843", + "bl-79028eed-c330-4724-9402-734062d13503", + "kn-0710e5b4-799d-4a0e-afd3-62d43b38ea37", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3" + ], + "n_returned": 10, + "latency_ms": 284.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60" + ], + "n_returned": 10, + "latency_ms": 249.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.3125, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-741ffd78-7dbc-4bf5-b643-e7fa47dfa8fa", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 247.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.3333333333333333, + "recall@10": 0.3333333333333333, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "fb36f51d-faf1-407b-bc5d-9dadda2f14ed", + "bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "tag-harmonic-design", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-framework", + "bl-73a74e74-55e6-41b3-9fad-e92451b34596" + ], + "n_returned": 10, + "latency_ms": 247.5, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "tag-sarah", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83" + ], + "n_returned": 10, + "latency_ms": 253.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "?Q??m?;?u?'", + "?Q??m?;?u?'", + "tag-autonomous", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "?Q??m?;?u?'", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__" + ], + "n_returned": 10, + "latency_ms": 396.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2857142857142857, + "recall@10": 0.2857142857142857, + "precision@5": 0.8, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315", + "?", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-ce88adf9-3f3c-47ac-a7d3-83af7b290e68", + "?" + ], + "n_returned": 10, + "latency_ms": 334.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-a535f205-bc4c-4058-9171-6263c496044a" + ], + "n_returned": 10, + "latency_ms": 713.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "f5ed9efa-9156-4d49-b670-dbbd734f0ace", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "556290bb-342d-4b76-9c8a-853cd9cf6f0a", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "8b02d9a6-da0f-40ec-ade5-6515e3cd4e29" + ], + "n_returned": 10, + "latency_ms": 714.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-d1cfde0a-37f1-4bff-9a06-8eddbbf259f6", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "44b84ec7-6ed4-4031-8ee5-8cb5cebfbf5f" + ], + "n_returned": 10, + "latency_ms": 611.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783", + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3" + ], + "n_returned": 10, + "latency_ms": 539.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "a1000001-0000-0000-0000-000000000010", + "mem-f6270c76-5a1c-4ab9-ae03-b6b0c25915ec", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "? t?'?B?+??", + "a1000001-0000-0000-0000-000000000009", + "mem-32203649-3213-4d6d-86fd-3d657ac70d77" + ], + "n_returned": 10, + "latency_ms": 727.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "mem-d450f442-541a-4d3e-8c97-b6303d6cbe56", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "e93ad84e-5219-4c65-9c12-06100f66a3c3", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838" + ], + "n_returned": 10, + "latency_ms": 566.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 689.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "bl-34f51ddb-a840-459f-a248-94214f5febb6" + ], + "n_returned": 10, + "latency_ms": 594.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-027636c5-dee7-42d1-a411-afbac32fa9b2", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c" + ], + "n_returned": 10, + "latency_ms": 721.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "a1000001-0000-0000-0000-000000000001", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 634.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "944eead2-b2f4-43c6-bf3f-d30de179b4e4", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "bl-87c93185-b2bf-40af-ae23-3c830c007abf" + ], + "n_returned": 10, + "latency_ms": 673.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15" + ], + "n_returned": 10, + "latency_ms": 578.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 592.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "dfd8846a-8307-4029-879b-5fa24134e766" + ], + "n_returned": 10, + "latency_ms": 624.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.36363636363636365, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "mem-a5ae594a-235b-42c4-b3cf-f770e4f73761", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-0f6a2d16-0d39-4b38-9350-c6033b19c203", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "mem-0d6a3b65-488d-4219-beb2-af300ca534e1", + "5eb24168-c2d3-4842-85db-2070e0e14923", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "e7bc8e62-cd9e-4d4c-99c9-08e9455fee41" + ], + "n_returned": 10, + "latency_ms": 542.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "tag-trailer-park-paladins", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "48703a0f-ec56-4325-bfa5-acd364536ad7", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "project-trailer-park-paladins", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277" + ], + "n_returned": 10, + "latency_ms": 523.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077" + ], + "n_returned": 10, + "latency_ms": 522.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "tag-hope", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329" + ], + "n_returned": 10, + "latency_ms": 524.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.09090909090909091, + "precision@5": 0.0, + "mrr@10": 0.14285714285714285 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "knw-729fc901-8335-44c4-9f3a-b150b4aa0915", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "4f698ae6-c40e-464e-9798-50350991a188", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "719aa819-00a9-4f4b-a857-4f9fe5ad44d7", + "mem-f09effd6-d287-4e43-9107-c7016087a2a6" + ], + "n_returned": 10, + "latency_ms": 661.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 338.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 248.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 336.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de" + ], + "n_returned": 10, + "latency_ms": 635.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 4 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "a1000001-0000-0000-0000-000000000002", + "527ecb25-2587-47eb-8269-73be2431abd4", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "13705072-4515-4124-963d-083af490494f", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "2aedba58-a4a3-4d14-b0db-fb59f976803f", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "6de314bf-5c4c-4cfc-871f-fa2e422d45e6", + "6374c955-f68e-41fe-af59-7c8d15a5512a" + ], + "n_returned": 10, + "latency_ms": 715.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "791256bb-5a85-4775-96ef-7af56c848858" + ], + "n_returned": 10, + "latency_ms": 485.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000006", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "a1000001-0000-0000-0000-000000000002", + "62a0e763-80f2-4952-9059-4953974ced1c" + ], + "n_returned": 10, + "latency_ms": 614.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "92b336c2-e788-4999-a0ff-69d299fc8b4f", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9" + ], + "n_returned": 10, + "latency_ms": 786.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "ctx-3a55", + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "e936ea19-b63f-47ac-aee2-95c40451e15d", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "ctx-175f", + "kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "65d4a0e4-f335-4e87-858d-e29fd94048cf" + ], + "n_returned": 10, + "latency_ms": 825.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "696e609c-da7a-4394-8a0c-106ba07dc6c3", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "0cb066f6-bf68-4368-8707-89347c5e9fd0", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "3b7f34a5-d21f-480c-a772-936bc1699576", + "4a080252-f873-4b24-8e7f-e23122a42093", + "19843fb9-f1aa-446c-8dfb-44e413987428" + ], + "n_returned": 10, + "latency_ms": 727.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "944eead2-b2f4-43c6-bf3f-d30de179b4e4", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "a36f494c-465b-4b3f-9c7c-80460cf31748", + "bl-ba7530fd-0f7c-4010-b03a-ca1e45a237ef", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "bl-0de4e61b-6562-49e5-b7df-ebb809a01723", + "1d81534a-0359-4217-a4dd-d158a4841633" + ], + "n_returned": 10, + "latency_ms": 697.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "a8b1ebf0-5485-465a-a4c0-58f4014da561", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "7de6a552-0878-422b-8820-bbf9101199ef", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "96497334-b18f-495c-9228-eeb8182bdc38" + ], + "n_returned": 10, + "latency_ms": 732.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "bl-79028eed-c330-4724-9402-734062d13503", + "98f9301f-fc1e-4cd2-8b08-139dc25093be", + "4b91a1d9-dad2-4c04-adeb-fc1c26f45d43", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "01c81356-30d3-4c36-b020-2753793bb2e6", + "d30d2df6-7849-428a-9854-40dac60094e6", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a" + ], + "n_returned": 10, + "latency_ms": 812.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000008", + "mem-f22575a1-ad04-4f66-ab0f-aa538895b878", + "d3258085-e5ba-4aee-8962-29195361e799", + "a1000001-0000-0000-0000-000000000009", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-983b25ad-5d73-4029-a970-6b311c24644b", + "a1000001-0000-0000-0000-000000000012", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6" + ], + "n_returned": 10, + "latency_ms": 870.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "1fa43205-9442-4bae-8613-bf7c857fcaf6", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "mem-7cd90611-88a3-423d-a38a-0db2812952fa", + "ctx-5a0a", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "bl-88221dad-8dab-48f1-bebc-b5ca84a3cb69", + "ctx-fae1", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca" + ], + "n_returned": 10, + "latency_ms": 1184.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "??f?7???", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-5e319cee-6f1c-4c3f-8897-9a9b83beea9e" + ], + "n_returned": 10, + "latency_ms": 901.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "19843fb9-f1aa-446c-8dfb-44e413987428", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "b2a37f07-15c1-4be4-b0e1-a437160ebeae", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "8fb73177-b6d9-4008-a6df-6cb6d7d587f3", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "bl-820cbaf2-baf8-42f9-9e18-a0afd186789d" + ], + "n_returned": 10, + "latency_ms": 892.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "ad3c6391-a6ed-424b-9148-8e7ac963a81f", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "k??A5??????", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "758ec67c-131c-447e-a352-ac7a22c3a47e" + ], + "n_returned": 10, + "latency_ms": 903.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-ac207dc9-c610-4ff9-9398-b39683ac72d4", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 727.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "1786f09d-e915-4910-9e1e-8061f42a0270", + "e68511f9-a3bb-4ce1-8261-728a50eaea35", + "a1000001-0000-0000-0000-000000000012", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "7776a0d2-1d96-425b-86c8-b30950ec8843", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000010", + "c7cf00b5-c924-47f0-9129-f62e64484cf7", + "a1000001-0000-0000-0000-000000000008", + "mem-1d626116-c3b8-409c-b38c-4e91def27483" + ], + "n_returned": 10, + "latency_ms": 969.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-c9adb8e5-293f-4033-99f8-0405c17ef941", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "6adc72bb-7a75-45bd-826f-7baadd126447", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "cafe3203-6f50-4011-8167-9400f19bb196", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34", + "a1000001-0000-0000-0000-000000000006", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_" + ], + "n_returned": 10, + "latency_ms": 768.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "b1183213-d659-4759-85d7-5b1f22427fe2", + "a1000001-0000-0000-0000-000000000012", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "229e09ba-0bcb-4013-9444-197f2e38f169", + "a1000001-0000-0000-0000-000000000011", + "tag-marketing-site", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "a1000001-0000-0000-0000-000000000009", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 834.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "6d46ef63-274a-4f64-b21a-2f8721bdd2b0", + "bl-59e0ece0-2c79-4e06-a85f-daaeec0bd737", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "mem-3f2ae069-f897-4942-a079-b66ad433f5ca", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "355c021d-3d45-4c2f-93c1-2b9932f4c425", + "bl-b05f9a75-ad6b-4929-a9a3-32df17a1f4bf", + "96497334-b18f-495c-9228-eeb8182bdc38" + ], + "n_returned": 10, + "latency_ms": 794.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "5a891516-7320-4869-9dea-bc981fd28089", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "0cb066f6-bf68-4368-8707-89347c5e9fd0", + "bfad516b-c306-4c4c-874a-a347c46c05c2", + "f375b2cf-15cd-4804-8d18-23a5830f4c66", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "bl-714799b9-c3fa-4037-bb7a-e8fa72ba3a6c" + ], + "n_returned": 10, + "latency_ms": 750.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "fd07dd6c-e1c7-4af9-85ed-2bde73b54333", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "c67a5892-27e9-49a0-8ec6-7c4b0810f351", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2" + ], + "n_returned": 10, + "latency_ms": 1002.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6", + "2f7270cc-4cda-4499-831f-1fbdc2f23284" + ], + "n_returned": 10, + "latency_ms": 822.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "d69b441e-0031-4d82-abfd-ac503b637562", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "aac1f943-c71d-4c63-9f5b-7437adae1abd", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "7c90d01b-8d53-4e75-b494-73011b71fa6a" + ], + "n_returned": 10, + "latency_ms": 988.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "a1000001-0000-0000-0000-000000000012", + "737faca8-263f-4523-90b9-3530db346d94", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "a1000001-0000-0000-0000-000000000002", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "e40b2a94-bb87-4d5c-bad8-38ff4941e03b" + ], + "n_returned": 10, + "latency_ms": 601.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-9329958a-d459-499c-b516-e51019c693e0", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-729fc901-8335-44c4-9f3a-b150b4aa0915" + ], + "n_returned": 10, + "latency_ms": 867.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "k??A5??????", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "8c40420c-c891-4ac9-a039-88440df87180", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "??EfkY?c" + ], + "n_returned": 10, + "latency_ms": 1011.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "87ab5b46-4fe1-4e1d-86c6-d437e17682ab", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "f44162cf-4fc9-44b6-89ae-2789097cfb82", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "f375b2cf-15cd-4804-8d18-23a5830f4c66" + ], + "n_returned": 10, + "latency_ms": 1007.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "d6a745c3-fb56-4c0f-872c-a3aa69f8b12f", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "bl-738cd260-a9cf-4cdf-a494-4654c321bd79", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "f169bf40-51ab-497d-8085-e57d374b77d4", + "bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992", + "mem-2b5d0ebe-1e85-4c07-82d3-dbbcca46414f" + ], + "n_returned": 10, + "latency_ms": 963.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-b16f4f65-a99d-4f3e-92e7-f447c90cc2d2", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "5b422430-2503-42d7-a315-9dfaaa6472ae", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-cea1d711-b899-4add-a073-c2d124f75c70", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70", + "bl-6ba43590-e5f8-4303-a812-3dd604e0da59", + "bl-212774e1-4ed4-4893-9265-f98d9eddeb3e", + "801806c5-1815-48a2-8287-1081c5ee7886" + ], + "n_returned": 10, + "latency_ms": 947.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "mem-348aaade-4e79-40cf-8867-8b73c9d0bf05", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-cce9985c-c968-4883-8236-0dff148adfd6", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8" + ], + "n_returned": 10, + "latency_ms": 675.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "258c7ec5-6d74-4a3d-b12f-053934b35f9a", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a" + ], + "n_returned": 10, + "latency_ms": 693.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "5509a6d7-5897-41e1-99eb-81df71801988", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "1fa43205-9442-4bae-8613-bf7c857fcaf6", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "tag-credentials", + "5c8eea2d-3d33-432e-bb18-f803cb39ebcd" + ], + "n_returned": 10, + "latency_ms": 856.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 357.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 265.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 336.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 249.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 346.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 250.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 248.9, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-unfloor-clean.json b/tools/retrieval-eval/results-unfloor-clean.json new file mode 100644 index 0000000..eaeb9b8 --- /dev/null +++ b/tools/retrieval-eval/results-unfloor-clean.json @@ -0,0 +1,1825 @@ +{ + "label": "unfloor-clean", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-unfloor", + "soul_md5": "cef6d4279c63266752027c455e543b42", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semsub/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7894, + "wall_clock_s": 56.8, + "child_pid": 4111, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226", + "??~'??JI???", + "b41a1b01-0075-4019-b0b5-6e639fd126e5", + "??H'??JI?#?", + "??H'??JI?[?", + "bb94bdf1-7b52-4967-ab81-9c6f01cf9665", + "7f092282-ccf5-4922-8694-08b7728f678d", + "6743e5f0-8d77-46e7-8ce0-b82e947272a1", + "o?H'??JI?c?", + "696e609c-da7a-4394-8a0c-106ba07dc6c3" + ], + "n_returned": 10, + "latency_ms": 158.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-74ed", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-fae1" + ], + "n_returned": 10, + "latency_ms": 160.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091", + "?o?`????", + "mem-5708f4c9-3d61-4182-8543-2843698931e6", + "??L?=??|Q?", + "????2??jZ??", + "project-canon", + "?/g?7?????e", + "?v??_??tE??", + "??o?'?B???N", + "x??I?cB?]p?" + ], + "n_returned": 10, + "latency_ms": 160.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c", + "mem-f623328a-be55-49a0-a389-4cce5bf42509", + "bl-b05f9a75-ad6b-4929-a9a3-32df17a1f4bf", + "mem-295f7aac-6c9e-4508-8dbc-a25f101f01d6", + "bl-2483e070-1385-4dbe-a49d-86c1af81e7d1", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-068e01fb-bdd0-498e-a2bb-33defa2c8bb2", + "bl-59e0ece0-2c79-4e06-a85f-daaeec0bd737", + "mem-2d1ea831-cccd-4f0f-86b9-2cbbc89dc3e0" + ], + "n_returned": 10, + "latency_ms": 166.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "bl-c9adb8e5-293f-4033-99f8-0405c17ef941", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "mem-00de72b8-b532-4cc4-95eb-18e39697891c", + "project-runpod-inference", + "bl-87b459c8-3953-478f-8384-10cc3f861424", + "bl-f918df79-dcd2-4a6e-9eae-71f94247e913", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34" + ], + "n_returned": 10, + "latency_ms": 159.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "958ce934-a9c4-4242-afe1-d73d61c22aee", + "f2bb62a2-c2b6-4fb6-b58d-21cdecc1446f", + "89aeb221-d068-4dfc-925d-d2249e744788", + "1c0effdd-baef-4523-abb8-490c3bc8b4df", + "00d2cc20-c76b-468d-8204-c624512ced28", + "eb68b995-eb4d-4f8d-b7c9-35eeba9ac781", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "??", + "??" + ], + "n_returned": 10, + "latency_ms": 164.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "3e9a651b-4b34-466d-ba45-874764a01609", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "3c633e71-2754-4273-990d-d44a6351cec3", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "cc4e7bc9-9d9b-4905-92a6-c6b99653b867", + "bl-79028eed-c330-4724-9402-734062d13503", + "14e328ad-bff1-4dc1-9ec5-dfc2bb98d968", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3" + ], + "n_returned": 10, + "latency_ms": 278.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "mem-c9bec303-a638-4a11-a490-f38410d448cf", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "f783b9ee-cc52-4b6d-a4dc-f26b40620211", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be" + ], + "n_returned": 10, + "latency_ms": 251.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.3125, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "n_returned": 10, + "latency_ms": 249.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2222222222222222, + "recall@10": 0.4444444444444444, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "project-harmonic-framework-content", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework" + ], + "n_returned": 10, + "latency_ms": 249.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "??_/Pr?????", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "??_/Pr?????" + ], + "n_returned": 10, + "latency_ms": 252.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "?Q??m?;?u?'", + "knw-cf13b883-d947-4cf8-b86b-cd9c6f0748d6" + ], + "n_returned": 10, + "latency_ms": 405.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "bl-36e2b4d0-9510-4da8-9b62-4af37b714429", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-ce88adf9-3f3c-47ac-a7d3-83af7b290e68", + "?" + ], + "n_returned": 10, + "latency_ms": 342.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-a535f205-bc4c-4058-9171-6263c496044a" + ], + "n_returned": 10, + "latency_ms": 719.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "daf9d558-2265-49b0-b4fb-be0b80f3cdb1", + "bfb5809e-d19a-4d3f-8c1a-796db622ad9d", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "f3364a57-956a-4e78-a500-b3b66a4e3066", + "15a90710-5372-4d3d-aee5-2d7287d2a492", + "341afa66-6a49-43fe-9d5b-3f1032bb6904", + "mem-da21c52c-04a5-4f92-8fba-f10aac47e027" + ], + "n_returned": 10, + "latency_ms": 717.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-d1cfde0a-37f1-4bff-9a06-8eddbbf259f6", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "tag-factions" + ], + "n_returned": 10, + "latency_ms": 620.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-628437a6-47b0-4d81-8112-7e78499723d5", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3", + "o#?CW????y:", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783" + ], + "n_returned": 10, + "latency_ms": 524.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-f6270c76-5a1c-4ab9-ae03-b6b0c25915ec", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "a1000001-0000-0000-0000-000000000010", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "a1000001-0000-0000-0000-000000000009", + "mem-32203649-3213-4d6d-86fd-3d657ac70d77", + "? t?'?B?+??", + "a1000001-0000-0000-0000-000000000012", + "mem-da21c52c-04a5-4f92-8fba-f10aac47e027" + ], + "n_returned": 10, + "latency_ms": 715.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "mem-d450f442-541a-4d3e-8c97-b6303d6cbe56", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838" + ], + "n_returned": 10, + "latency_ms": 558.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 673.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-ba7530fd-0f7c-4010-b03a-ca1e45a237ef", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "bl-7156862d-1f4f-4d76-8176-3ea89d22c3ae", + "bl-56a50e97-9a85-4e81-b6c9-3e3d26482f1d" + ], + "n_returned": 10, + "latency_ms": 590.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-06c13965-082b-417d-9561-93d6e958ae5d", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "mem-7b74cac0-905f-4c35-9688-fbcce105a177", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c" + ], + "n_returned": 10, + "latency_ms": 705.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000001", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 618.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7" + ], + "n_returned": 10, + "latency_ms": 659.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "a1000001-0000-0000-0000-000000000012", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "kn-89c67586-7d34-475e-89f9-46a0c65a9849", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15" + ], + "n_returned": 10, + "latency_ms": 560.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 588.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "28ae74a1-9d47-4874-9279-43c1f90c0f64", + "mem-bce80169-2d46-4b3e-9ebe-8498e26f0a89" + ], + "n_returned": 10, + "latency_ms": 639.3, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "mem-a5ae594a-235b-42c4-b3cf-f770e4f73761", + "ctx-fae1", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-a54d59a4-cc2b-405d-b68e-5b6cf2b277bc", + "ctx-e5427d7d", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "mem-0f6a2d16-0d39-4b38-9350-c6033b19c203", + "ctx-d499", + "5eb24168-c2d3-4842-85db-2070e0e14923" + ], + "n_returned": 10, + "latency_ms": 522.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-19317db1-fa05-43cd-b4e4-cb57840b7c40", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "tag-trailer-park-paladins", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "project-trailer-park-paladins", + "?V?" + ], + "n_returned": 10, + "latency_ms": 522.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "???I?cB?Zx?" + ], + "n_returned": 10, + "latency_ms": 522.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "tag-hope", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd" + ], + "n_returned": 10, + "latency_ms": 525.4, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.09090909090909091, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "4f698ae6-c40e-464e-9798-50350991a188", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "719aa819-00a9-4f4b-a857-4f9fe5ad44d7", + "mem-d396d789-0f7f-4366-a008-5d8801c8f2eb", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 664.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.18181818181818182, + "precision@5": 0.0, + "mrr@10": 0.14285714285714285 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 343.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 250.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 337.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5" + ], + "n_returned": 10, + "latency_ms": 632.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 9 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "a1000001-0000-0000-0000-000000000002", + "527ecb25-2587-47eb-8269-73be2431abd4", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "13705072-4515-4124-963d-083af490494f", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "4f225001-3a51-4a68-8d38-c8ecac3412de" + ], + "n_returned": 10, + "latency_ms": 713.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "5fcba804-eb5b-48ec-82da-146b1c6bb50d", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5" + ], + "n_returned": 10, + "latency_ms": 486.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "a1000001-0000-0000-0000-000000000006", + "a1000001-0000-0000-0000-000000000012", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000005", + "a1000001-0000-0000-0000-000000000010", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "62a0e763-80f2-4952-9059-4953974ced1c" + ], + "n_returned": 10, + "latency_ms": 611.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "mem-0d6a3b65-488d-4219-beb2-af300ca534e1", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9" + ], + "n_returned": 10, + "latency_ms": 790.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "bl-5eaddfde-e114-46b5-8ee1-b79f0af16271", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "bl-0866d6c6-6600-4d50-b7b7-52c720cae04c", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "bl-7f1b0116-6c6e-4721-ac7a-49c9dd1c2b63", + "bl-c3312479-8cf6-478c-82c5-ac18c6c51c10", + "8d6da9e1-b58b-497b-91b5-09a847dc8ae2" + ], + "n_returned": 10, + "latency_ms": 826.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "d7e12e95-0a87-4000-b5c2-36b9be234c91", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "dc3e7b44-c345-47d0-bf80-458e471c4b8b", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "b084e4da-95bf-46e6-9ee6-bfb4864732bb", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "b9b66af5-1609-44b2-adeb-b5e876904b79", + "89e3f620-b8ad-423b-9ed6-fe3c7954f4b6", + "aca9387e-2f61-4c2f-98cd-d467dbb25528" + ], + "n_returned": 10, + "latency_ms": 732.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "bl-0d01319f-f9cc-4db5-b0f1-107dba3c3a92", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "bl-672bc3c8-9ad4-4b74-8950-75854a0d9cb8", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "a36f494c-465b-4b3f-9c7c-80460cf31748", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "1d81534a-0359-4217-a4dd-d158a4841633" + ], + "n_returned": 10, + "latency_ms": 698.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "aacd7b32-04ed-441e-a1a9-9c5c089e79f6", + "7de6a552-0878-422b-8820-bbf9101199ef", + "75657c91-3c3a-4173-b3cb-e2973901fa93" + ], + "n_returned": 10, + "latency_ms": 718.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "bl-79028eed-c330-4724-9402-734062d13503", + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "mem-1825128d-0136-4f1e-a27b-60e554e86141", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "98f9301f-fc1e-4cd2-8b08-139dc25093be", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "f783b9ee-cc52-4b6d-a4dc-f26b40620211" + ], + "n_returned": 10, + "latency_ms": 796.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000012", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-983b25ad-5d73-4029-a970-6b311c24644b", + "a1000001-0000-0000-0000-000000000008", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000011", + "mem-0b69e038-71b6-4b22-ac42-e213c49b1451" + ], + "n_returned": 10, + "latency_ms": 853.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "ctx-60cc", + "bl-9d53422d-b703-4f1d-860a-8598cb29b792", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "a1000001-0000-0000-0000-000000000010", + "ctx-fae1", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "bl-3f66eec3-82f3-42e5-a025-4649a7da7e2f", + "ctx-5a0a", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca" + ], + "n_returned": 10, + "latency_ms": 1177.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "??f?7???", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-5e319cee-6f1c-4c3f-8897-9a9b83beea9e" + ], + "n_returned": 10, + "latency_ms": 885.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "c7dddd9c-fd3b-419e-baa2-61623b495597" + ], + "n_returned": 10, + "latency_ms": 874.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-cea1d711-b899-4add-a073-c2d124f75c70", + "k??A5??????", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "758ec67c-131c-447e-a352-ac7a22c3a47e", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "mem-894c371a-3b1a-46b5-8325-92261b7ef567" + ], + "n_returned": 10, + "latency_ms": 884.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-60c0342b-ba57-42e5-a763-ec9f982be10f", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 732.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "1786f09d-e915-4910-9e1e-8061f42a0270", + "bl-2ade8732-2f2c-45bb-b15d-238d671426ac", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "a1000001-0000-0000-0000-000000000010", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-1d626116-c3b8-409c-b38c-4e91def27483" + ], + "n_returned": 10, + "latency_ms": 975.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "project-Will_s_framework__Project_Design_practice___part_of_HD_bundle__sold_standalone_", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_", + "cafe3203-6f50-4011-8167-9400f19bb196", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_", + "a1000001-0000-0000-0000-000000000006", + "project-Will_s_framework__BDT_methodology___part_of_HD_bundle__sold_standalone_", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c", + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "n_returned": 10, + "latency_ms": 746.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000008", + "tag-marketing-site", + "a1000001-0000-0000-0000-000000000012", + "a1000001-0000-0000-0000-000000000007", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 836.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "bl-0547b0b1-1cd5-447c-814d-ebe525bc08e9", + "c9070b0a-e1d1-4307-85d1-67a1239ca604", + "mem-3f2ae069-f897-4942-a079-b66ad433f5ca", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "355c021d-3d45-4c2f-93c1-2b9932f4c425", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "c2bf4e1b-a481-4e28-9dd4-6704df0857cd", + "96497334-b18f-495c-9228-eeb8182bdc38", + "mem-fdddca75-489b-4c53-b5ec-38ab3062a474" + ], + "n_returned": 10, + "latency_ms": 794.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "bl-da818960-b360-455d-a9c1-e9a215c33ca4", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "bl-2ff9c669-a9fc-4ef7-9a18-9ec7167639c8", + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "mem-69889e0a-c141-4f05-bbd7-10c93a53143e" + ], + "n_returned": 10, + "latency_ms": 748.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "mem-5dd5f3b9-3c7e-44b1-84ce-2e36925f17fb", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2" + ], + "n_returned": 10, + "latency_ms": 1008.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-f1f71f3c-ca7d-4f98-a569-c8a9aa67a000", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "2f7270cc-4cda-4499-831f-1fbdc2f23284", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6" + ], + "n_returned": 10, + "latency_ms": 824.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "7c90d01b-8d53-4e75-b494-73011b71fa6a", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "aac1f943-c71d-4c63-9f5b-7437adae1abd", + "kn-cf8fa517-fc83-47c4-8fb7-bd9e26410dd3", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "d69b441e-0031-4d82-abfd-ac503b637562" + ], + "n_returned": 10, + "latency_ms": 992.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "c34f6292-b6e5-4619-a553-90ae4fc498a2", + "a1000001-0000-0000-0000-000000000012", + "737faca8-263f-4523-90b9-3530db346d94", + "03a82b13-2abe-402d-90bb-c547771638f8", + "a1000001-0000-0000-0000-000000000002", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "1fbe9416-622e-455c-a6ed-f4daa5f48f30", + "a1000001-0000-0000-0000-000000000004", + "e40b2a94-bb87-4d5c-bad8-38ff4941e03b" + ], + "n_returned": 10, + "latency_ms": 598.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "954c9597-ab7a-41e6-b085-bf838bba4f17", + "art-d0ded6f5-e202-47ad-8fb3-f6bd80f35052", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-9329958a-d459-499c-b516-e51019c693e0" + ], + "n_returned": 10, + "latency_ms": 865.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "k??A5??????", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "8c40420c-c891-4ac9-a039-88440df87180", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "66d8d8c7-dea4-4245-8e68-d1ab4f4346b8" + ], + "n_returned": 10, + "latency_ms": 1029.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "87ab5b46-4fe1-4e1d-86c6-d437e17682ab", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "mem-af215d69-8fd8-4416-b7eb-9a0b5512e9a7", + "bed4dd5d-bce7-4e55-a703-175e422b7898", + "f375b2cf-15cd-4804-8d18-23a5830f4c66" + ], + "n_returned": 10, + "latency_ms": 993.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "8d91620b-9d0a-4ba1-b066-e57d1d8bc7d3", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "d6a745c3-fb56-4c0f-872c-a3aa69f8b12f", + "bl-acc3ecee-e186-41ad-bffb-3ed3e6710d01", + "db836e65-0cdc-4232-b8d7-f973ab279ea9", + "f169bf40-51ab-497d-8085-e57d374b77d4", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "6df650b5-a14d-42f8-88dd-0919ad20b77a" + ], + "n_returned": 10, + "latency_ms": 949.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "bl-6ba43590-e5f8-4303-a812-3dd604e0da59", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "5b422430-2503-42d7-a315-9dfaaa6472ae" + ], + "n_returned": 10, + "latency_ms": 936.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "251abd91-931e-4591-b793-80043ec48666", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "8556cc01-cbc1-42a1-85b4-41cef23b1fcb", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "e371e209-7a4a-44e1-955b-1a460e7d7e64", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "ce8d24a7-af0f-424f-bb41-379312eb609c", + "tag-reaction-order" + ], + "n_returned": 10, + "latency_ms": 659.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "aa3fb542-87f2-4eef-8e95-a870bfd0318a", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "d43a8137-c3c3-49e2-ad11-89d897280761" + ], + "n_returned": 10, + "latency_ms": 693.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-556438af-57b2-4bd8-a747-9f868aaee290", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "66c63082-b4da-4aa1-8fee-848db8a83210" + ], + "n_returned": 10, + "latency_ms": 838.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 340.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 252.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 338.4, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 251.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 343.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 249.2, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 251.2, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-unfloor-r2.json b/tools/retrieval-eval/results-unfloor-r2.json new file mode 100644 index 0000000..63d1d1b --- /dev/null +++ b/tools/retrieval-eval/results-unfloor-r2.json @@ -0,0 +1,1825 @@ +{ + "label": "unfloor-r2", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-verify", + "soul_md5": "a31a8b0b1062cf91eb8109394f911144", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semsub/tools/retrieval-eval/gold_set_extended.json", + "limit": 10, + "port": 7896, + "wall_clock_s": 57.3, + "child_pid": 5469, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.4, + "latency_ms_p95": 998.7, + "latency_ms_max": 1178.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226", + "??~'??JI???", + "b41a1b01-0075-4019-b0b5-6e639fd126e5", + "??H'??JI?#?", + "??H'??JI?[?", + "bb94bdf1-7b52-4967-ab81-9c6f01cf9665", + "7f092282-ccf5-4922-8694-08b7728f678d", + "6743e5f0-8d77-46e7-8ce0-b82e947272a1", + "o?H'??JI?c?", + "696e609c-da7a-4394-8a0c-106ba07dc6c3" + ], + "n_returned": 10, + "latency_ms": 174.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-74ed", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-fae1" + ], + "n_returned": 10, + "latency_ms": 159.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091", + "?o?`????", + "mem-5708f4c9-3d61-4182-8543-2843698931e6", + "??L?=??|Q?", + "????2??jZ??", + "project-canon", + "?/g?7?????e", + "?v??_??tE??", + "??o?'?B???N", + "x??I?cB?]p?" + ], + "n_returned": 10, + "latency_ms": 167.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c", + "mem-f623328a-be55-49a0-a389-4cce5bf42509", + "bl-b05f9a75-ad6b-4929-a9a3-32df17a1f4bf", + "mem-295f7aac-6c9e-4508-8dbc-a25f101f01d6", + "bl-2483e070-1385-4dbe-a49d-86c1af81e7d1", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-068e01fb-bdd0-498e-a2bb-33defa2c8bb2", + "bl-59e0ece0-2c79-4e06-a85f-daaeec0bd737", + "mem-2d1ea831-cccd-4f0f-86b9-2cbbc89dc3e0" + ], + "n_returned": 10, + "latency_ms": 173.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "bl-c9adb8e5-293f-4033-99f8-0405c17ef941", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_", + "bl-21a904d9-5834-46d5-8c20-dfc5ab49444f", + "mem-00de72b8-b532-4cc4-95eb-18e39697891c", + "project-runpod-inference", + "bl-87b459c8-3953-478f-8384-10cc3f861424", + "bl-f918df79-dcd2-4a6e-9eae-71f94247e913", + "bl-2632242e-80b1-4d88-8368-7065b5de5b34" + ], + "n_returned": 10, + "latency_ms": 173.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "958ce934-a9c4-4242-afe1-d73d61c22aee", + "f2bb62a2-c2b6-4fb6-b58d-21cdecc1446f", + "89aeb221-d068-4dfc-925d-d2249e744788", + "1c0effdd-baef-4523-abb8-490c3bc8b4df", + "00d2cc20-c76b-468d-8204-c624512ced28", + "eb68b995-eb4d-4f8d-b7c9-35eeba9ac781", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "??", + "??" + ], + "n_returned": 10, + "latency_ms": 158.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "3e9a651b-4b34-466d-ba45-874764a01609", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "3c633e71-2754-4273-990d-d44a6351cec3", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "cc4e7bc9-9d9b-4905-92a6-c6b99653b867", + "bl-79028eed-c330-4724-9402-734062d13503", + "14e328ad-bff1-4dc1-9ec5-dfc2bb98d968", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3" + ], + "n_returned": 10, + "latency_ms": 296.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "mem-c9bec303-a638-4a11-a490-f38410d448cf", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "da274f82-7a80-42c8-aad1-9d754809a160", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be" + ], + "n_returned": 10, + "latency_ms": 264.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.3125, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "n_returned": 10, + "latency_ms": 264.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2222222222222222, + "recall@10": 0.4444444444444444, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "project-harmonic-framework-content", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework" + ], + "n_returned": 10, + "latency_ms": 264.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "??_/Pr?????", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "??_/Pr?????" + ], + "n_returned": 10, + "latency_ms": 268.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "?Q??m?;?u?'", + "knw-cf13b883-d947-4cf8-b86b-cd9c6f0748d6" + ], + "n_returned": 10, + "latency_ms": 410.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "bl-36e2b4d0-9510-4da8-9b62-4af37b714429", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fb44a2fc-7405-41ff-87b3-84643ac07313", + "?", + "mem-ce88adf9-3f3c-47ac-a7d3-83af7b290e68", + "?" + ], + "n_returned": 10, + "latency_ms": 356.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-a535f205-bc4c-4058-9171-6263c496044a" + ], + "n_returned": 10, + "latency_ms": 734.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "daf9d558-2265-49b0-b4fb-be0b80f3cdb1", + "bfb5809e-d19a-4d3f-8c1a-796db622ad9d", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "f3364a57-956a-4e78-a500-b3b66a4e3066", + "15a90710-5372-4d3d-aee5-2d7287d2a492", + "341afa66-6a49-43fe-9d5b-3f1032bb6904", + "mem-da21c52c-04a5-4f92-8fba-f10aac47e027" + ], + "n_returned": 10, + "latency_ms": 736.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "tag-fiction", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-d1cfde0a-37f1-4bff-9a06-8eddbbf259f6", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "tag-factions" + ], + "n_returned": 10, + "latency_ms": 632.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "mem-628437a6-47b0-4d81-8112-7e78499723d5", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "4da5dbaf-46e5-4f3e-b474-f60d9f8241d3", + "o#?CW????y:", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-434be7c8-88cb-4039-b79a-1da4ac4de783" + ], + "n_returned": 10, + "latency_ms": 545.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-f6270c76-5a1c-4ab9-ae03-b6b0c25915ec", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "a1000001-0000-0000-0000-000000000010", + "bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0", + "a1000001-0000-0000-0000-000000000009", + "mem-32203649-3213-4d6d-86fd-3d657ac70d77", + "? t?'?B?+??", + "a1000001-0000-0000-0000-000000000012", + "mem-da21c52c-04a5-4f92-8fba-f10aac47e027" + ], + "n_returned": 10, + "latency_ms": 730.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "mem-d450f442-541a-4d3e-8c97-b6303d6cbe56", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-92a7fdc5-9dd0-48cf-a691-506058de3838" + ], + "n_returned": 10, + "latency_ms": 555.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 671.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-ba7530fd-0f7c-4010-b03a-ca1e45a237ef", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "bl-7156862d-1f4f-4d76-8176-3ea89d22c3ae", + "bl-56a50e97-9a85-4e81-b6c9-3e3d26482f1d" + ], + "n_returned": 10, + "latency_ms": 576.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-06c13965-082b-417d-9561-93d6e958ae5d", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "mem-7b74cac0-905f-4c35-9688-fbcce105a177", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c" + ], + "n_returned": 10, + "latency_ms": 709.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "a1000001-0000-0000-0000-000000000001", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "mem-5708f4c9-3d61-4182-8543-2843698931e6" + ], + "n_returned": 10, + "latency_ms": 619.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7" + ], + "n_returned": 10, + "latency_ms": 695.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "a1000001-0000-0000-0000-000000000012", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "kn-89c67586-7d34-475e-89f9-46a0c65a9849", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "mem-a16deccb-16a7-419c-a013-ff824a4daa15" + ], + "n_returned": 10, + "latency_ms": 563.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "tag-childhood", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 586.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-5f76880b-bafb-4716-8e15-90f8ef59bebc", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "28ae74a1-9d47-4874-9279-43c1f90c0f64", + "mem-bce80169-2d46-4b3e-9ebe-8498e26f0a89" + ], + "n_returned": 10, + "latency_ms": 632.4, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "mem-a5ae594a-235b-42c4-b3cf-f770e4f73761", + "ctx-fae1", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "mem-a54d59a4-cc2b-405d-b68e-5b6cf2b277bc", + "ctx-e5427d7d", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "mem-0f6a2d16-0d39-4b38-9350-c6033b19c203", + "ctx-d499", + "5eb24168-c2d3-4842-85db-2070e0e14923" + ], + "n_returned": 10, + "latency_ms": 519.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-19317db1-fa05-43cd-b4e4-cb57840b7c40", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "tag-trailer-park-paladins", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "project-trailer-park-paladins", + "?V?" + ], + "n_returned": 10, + "latency_ms": 522.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.2727272727272727, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "???I?cB?Zx?" + ], + "n_returned": 10, + "latency_ms": 522.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "tag-hope", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd" + ], + "n_returned": 10, + "latency_ms": 523.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.09090909090909091, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-5305665c-6b5b-45b7-89ae-5d2fb0b896ac", + "4f698ae6-c40e-464e-9798-50350991a188", + "mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "719aa819-00a9-4f4b-a857-4f9fe5ad44d7", + "mem-d396d789-0f7f-4366-a008-5d8801c8f2eb", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 666.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.18181818181818182, + "precision@5": 0.0, + "mrr@10": 0.14285714285714285 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 335.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 248.3, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [], + "n_returned": 0, + "latency_ms": 339.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "mem-c17aefb1-38b5-4ced-af50-fe524127e1a4", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5" + ], + "n_returned": 10, + "latency_ms": 632.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 9 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8", + "12082f7e-e320-438b-bd65-083d8259748f", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "a1000001-0000-0000-0000-000000000002", + "527ecb25-2587-47eb-8269-73be2431abd4", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "13705072-4515-4124-963d-083af490494f", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "4f225001-3a51-4a68-8d38-c8ecac3412de" + ], + "n_returned": 10, + "latency_ms": 708.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125, + "outranks": true, + "rank_correct": 8, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "5fcba804-eb5b-48ec-82da-146b1c6bb50d", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b", + "86228228-7adf-41fb-b4c4-9ceea87953ae", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5" + ], + "n_returned": 10, + "latency_ms": 504.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 5 + }, + { + "id": "q39", + "category": "heldout_paraphrase", + "query": "understating your abilities so nobody feels threatened", + "returned": [ + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "a1000001-0000-0000-0000-000000000006", + "a1000001-0000-0000-0000-000000000012", + "bl-7e11dced-65f7-4cab-82c0-2f096139039a", + "a1000001-0000-0000-0000-000000000005", + "a1000001-0000-0000-0000-000000000010", + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "62a0e763-80f2-4952-9059-4953974ced1c" + ], + "n_returned": 10, + "latency_ms": 624.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q40", + "category": "heldout_paraphrase", + "query": "a hidden fleet able to hunt down rogue machines everywhere", + "returned": [ + "bl-3c719d9a-cba1-47f4-b097-52cdeccc7c0d", + "mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "bl-3c719d9a-f108-4bf2-a617-66c722e5512f", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "mem-0d6a3b65-488d-4219-beb2-af300ca534e1", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-a75a1cf6-e6d4-4ab3-985c-d2505203dbf9" + ], + "n_returned": 10, + "latency_ms": 808.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q41", + "category": "heldout_paraphrase", + "query": "sketch a brief blueprint and clear it upstairs before construction starts", + "returned": [ + "mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "bl-5eaddfde-e114-46b5-8ee1-b79f0af16271", + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "bl-0866d6c6-6600-4d50-b7b7-52c720cae04c", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "bl-7f1b0116-6c6e-4721-ac7a-49c9dd1c2b63", + "bl-c3312479-8cf6-478c-82c5-ac18c6c51c10", + "8d6da9e1-b58b-497b-91b5-09a847dc8ae2" + ], + "n_returned": 10, + "latency_ms": 844.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q42", + "category": "heldout_paraphrase", + "query": "the reply arrived as bare prose so the caller's parser threw", + "returned": [ + "ee63cd58-46e0-44d1-aea3-a0e65ad6da0f", + "d7e12e95-0a87-4000-b5c2-36b9be234c91", + "db52889f-03dc-4529-8429-b1dc8c3137e3", + "dc3e7b44-c345-47d0-bf80-458e471c4b8b", + "728745d5-72e6-4f4d-91b4-1613fddbec98", + "b084e4da-95bf-46e6-9ee6-bfb4864732bb", + "5e337176-5daf-4726-85d6-3f4bd560c9ed", + "b9b66af5-1609-44b2-adeb-b5e876904b79", + "89e3f620-b8ad-423b-9ed6-fe3c7954f4b6", + "aca9387e-2f61-4c2f-98cd-d467dbb25528" + ], + "n_returned": 10, + "latency_ms": 750.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q43", + "category": "heldout_paraphrase", + "query": "repeated catalogue keys blew up the scrolling grid", + "returned": [ + "mem-394cc9e8-049b-45bc-a380-66314f14e367", + "bl-0d01319f-f9cc-4db5-b0f1-107dba3c3a92", + "bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "bl-672bc3c8-9ad4-4b74-8950-75854a0d9cb8", + "588aa6bb-5647-405b-b09a-14b7c088119b", + "a36f494c-465b-4b3f-9c7c-80460cf31748", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "1d81534a-0359-4217-a4dd-d158a4841633" + ], + "n_returned": 10, + "latency_ms": 714.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q44", + "category": "heldout_paraphrase", + "query": "tracing each defect back to whichever invention it violated", + "returned": [ + "f3a89797-095d-47c4-9557-c172f86228b6", + "b07916ca-2139-4fb7-bfb4-97ac680f5194", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "mem-7ad5a86b-cc0d-493c-b180-4fe8213c8a93", + "4999ef68-2a31-4282-82e0-7081a428b43f", + "mem-2b2f0fe9-5e51-4819-a8ac-2dfe810e8933", + "aacd7b32-04ed-441e-a1a9-9c5c089e79f6", + "7de6a552-0878-422b-8820-bbf9101199ef", + "75657c91-3c3a-4173-b3cb-e2973901fa93" + ], + "n_returned": 10, + "latency_ms": 731.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q45", + "category": "heldout_paraphrase", + "query": "a check that stops the mind clobbering a populated store when it boots", + "returned": [ + "326e70a2-20ae-49a1-aa32-973048df3ffa", + "bl-79028eed-c330-4724-9402-734062d13503", + "08b7b85b-8049-4ae4-bd66-d8bb63312ae1", + "mem-1825128d-0136-4f1e-a27b-60e554e86141", + "cf108ade-d9c4-4215-9c5f-42be0003a8dc", + "7a1ea86f-3080-4d1b-ab69-6ee2f822cce3", + "98f9301f-fc1e-4cd2-8b08-139dc25093be", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "da274f82-7a80-42c8-aad1-9d754809a160" + ], + "n_returned": 10, + "latency_ms": 797.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q46", + "category": "heldout_paraphrase", + "query": "telling it to consult the internet had to happen deep inside, not at the surface", + "returned": [ + "00754c19-f662-46b1-b6aa-abd7600a8bc6", + "mem-ad3d4d5e-2da4-4753-b3f6-c4c13fb91225", + "a1000001-0000-0000-0000-000000000012", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-983b25ad-5d73-4029-a970-6b311c24644b", + "a1000001-0000-0000-0000-000000000008", + "bl-3af44ee6-a3b9-4abd-91e3-17800fa8eaf6", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000011", + "mem-0b69e038-71b6-4b22-ac42-e213c49b1451" + ], + "n_returned": 10, + "latency_ms": 857.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q47", + "category": "heldout_paraphrase", + "query": "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "returned": [ + "313f927b-08f2-41a1-90c1-83e79d0de86c", + "ctx-60cc", + "bl-9d53422d-b703-4f1d-860a-8598cb29b792", + "21f9bdc6-37ad-4994-b0dd-ef7676102b49", + "a1000001-0000-0000-0000-000000000010", + "ctx-fae1", + "d83987f0-3ad9-48d2-9f55-0c2bb338e503", + "bl-3f66eec3-82f3-42e5-a025-4649a7da7e2f", + "ctx-5a0a", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca" + ], + "n_returned": 10, + "latency_ms": 1178.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q48", + "category": "heldout_paraphrase", + "query": "punctuation that pledges and then pays off rather than clarifying", + "returned": [ + "322df030-8358-4eb5-8c19-54b7497eaaf5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "??f?7???", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "mem-3dce32bb-525c-4c5e-818b-a0a5b95ec7e1", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "mem-5e319cee-6f1c-4c3f-8897-9a9b83beea9e" + ], + "n_returned": 10, + "latency_ms": 884.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q49", + "category": "heldout_paraphrase", + "query": "an easily missed feature finally given its own permanent spot in the navigation", + "returned": [ + "d1b665d0-4a4e-48a8-9385-fbe288df1949", + "9b4f0d93-4129-4746-8eb1-d10d955bd777", + "2263e9ab-7ce8-4034-b25b-640de9956229", + "fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "bffb86d6-2ee3-4ecd-a6d6-b128c8be7070", + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "4e539c04-f29c-42db-97b4-86e44f528ed6", + "b4b13fc0-b0da-4fd9-b9aa-98553b5c1078", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "c7dddd9c-fd3b-419e-baa2-61623b495597" + ], + "n_returned": 10, + "latency_ms": 873.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q50", + "category": "heldout_paraphrase", + "query": "checking preconditions up front so a stage aborts before fetching anything", + "returned": [ + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "dc7a7e2d-af2d-4235-aeb0-22ae67f1a01d", + "bl-cea1d711-b899-4add-a073-c2d124f75c70", + "k??A5??????", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "758ec67c-131c-447e-a352-ac7a22c3a47e", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995", + "mem-894c371a-3b1a-46b5-8325-92261b7ef567" + ], + "n_returned": 10, + "latency_ms": 884.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q51", + "category": "heldout_paraphrase", + "query": "producing the other platform's installer inside an emulated desktop", + "returned": [ + "6aade9dc-4a4c-4d92-a6af-ccea5b7d90f3", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "c6c693e0-7b9c-455b-8ef2-1da00849f73d", + "bl-1b58b05c-9305-4f06-a586-a08c96008027", + "bl-e4ef02bf-bacc-4d5e-aa0f-0d230da529fc", + "bl-cc0b7a20-ce1e-4947-999c-ea599da07623", + "bl-60c0342b-ba57-42e5-a763-ec9f982be10f", + "bl-98a5ddda-54e5-4287-8b6f-472778b31f8d", + "374453b2-66d2-475d-8e24-cd3b60d21ac1" + ], + "n_returned": 10, + "latency_ms": 731.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q52", + "category": "heldout_paraphrase", + "query": "chained add-ons that may inspect, rewrite or veto traffic in flight", + "returned": [ + "1786f09d-e915-4910-9e1e-8061f42a0270", + "bl-2ade8732-2f2c-45bb-b15d-238d671426ac", + "bl-4bc9f44c-2e96-40be-97c8-8476a217a890", + "83f3c78b-d0e5-45f9-9c25-6770c86d954e", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "a1000001-0000-0000-0000-000000000010", + "bl-1a74693f-608d-4e96-9f9d-47566ffe6395", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "mem-1d626116-c3b8-409c-b38c-4e91def27483" + ], + "n_returned": 10, + "latency_ms": 975.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q53", + "category": "heldout_paraphrase", + "query": "settling what the shareable bundles and the storefront would be called", + "returned": [ + "4757e274-7f3c-4ab0-8c62-537e913daa1f", + "project-Will_s_framework__Project_Design_practice___part_of_HD_bundle__sold_standalone_", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac", + "project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_", + "cafe3203-6f50-4011-8167-9400f19bb196", + "project-Will_s_framework__Event-Based_Design___part_of_HD_bundle__sold_standalone_", + "a1000001-0000-0000-0000-000000000006", + "project-Will_s_framework__BDT_methodology___part_of_HD_bundle__sold_standalone_", + "34356a36-0df5-4020-8dcc-5e7a423f8d4c", + "mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2" + ], + "n_returned": 10, + "latency_ms": 746.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.1 + }, + { + "id": "q54", + "category": "heldout_paraphrase", + "query": "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "returned": [ + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000011", + "a1000001-0000-0000-0000-000000000008", + "tag-marketing-site", + "a1000001-0000-0000-0000-000000000012", + "a1000001-0000-0000-0000-000000000007", + "ctx-d499" + ], + "n_returned": 10, + "latency_ms": 835.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q55", + "category": "heldout_paraphrase", + "query": "the distributable image finally assembled and its startup check passed", + "returned": [ + "e1ce0c90-0f0b-4111-833d-ec845777d358", + "bl-0547b0b1-1cd5-447c-814d-ebe525bc08e9", + "c9070b0a-e1d1-4307-85d1-67a1239ca604", + "mem-3f2ae069-f897-4942-a079-b66ad433f5ca", + "mem-12827ce8-9ff6-4275-a945-5043cfcf90bd", + "355c021d-3d45-4c2f-93c1-2b9932f4c425", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665", + "c2bf4e1b-a481-4e28-9dd4-6704df0857cd", + "96497334-b18f-495c-9228-eeb8182bdc38", + "mem-fdddca75-489b-4c53-b5ec-38ab3062a474" + ], + "n_returned": 10, + "latency_ms": 796.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q56", + "category": "heldout_paraphrase", + "query": "shuffling and appending stages in a draft before anything executes", + "returned": [ + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98", + "bl-da818960-b360-455d-a9c1-e9a215c33ca4", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "bl-2ff9c669-a9fc-4ef7-9a18-9ec7167639c8", + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "mem-69889e0a-c141-4f05-bbd7-10c93a53143e" + ], + "n_returned": 10, + "latency_ms": 746.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q57", + "category": "heldout_paraphrase", + "query": "orders handed down from above, with the tighter one winning any disagreement", + "returned": [ + "161d75d3-04eb-4b54-9cf1-0da2f643ca69", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "345b6420-e004-4d2e-b55c-6a729393fa99", + "mem-5dd5f3b9-3c7e-44b1-84ce-2e36925f17fb", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "b2a6e8cf-f72f-4b6c-8de1-1d08c839d7b2" + ], + "n_returned": 10, + "latency_ms": 998.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q58", + "category": "heldout_paraphrase", + "query": "shrinking generated text via encodings that compound on each other", + "returned": [ + "801806c5-1815-48a2-8287-1081c5ee7886", + "bl-b219f8b5-6c8c-436d-b9e5-2804fbc60512", + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "50ad2880-c41b-4dce-a946-a0eb37987c57", + "bl-6c497a94-ff3c-4737-bba1-453a1bbb1815", + "bl-f1f71f3c-ca7d-4f98-a569-c8a9aa67a000", + "kn-2b961d24-7fb9-47c7-9515-e45a24dce39d", + "2f7270cc-4cda-4499-831f-1fbdc2f23284", + "bl-b5e27ddb-a0a2-4652-afba-f969b80ee037", + "mem-f90f4a2a-1731-40de-9391-db98f1d5d1c6" + ], + "n_returned": 10, + "latency_ms": 819.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q59", + "category": "heldout_paraphrase", + "query": "splitting a system by what varies, with firm limits on which pieces may invoke which", + "returned": [ + "^?l?'?B??k?", + "kn-57b0b010-66f3-4f8d-982d-cf02a69c4ad5", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "7c90d01b-8d53-4e75-b494-73011b71fa6a", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "aac1f943-c71d-4c63-9f5b-7437adae1abd", + "kn-cf8fa517-fc83-47c4-8fb7-bd9e26410dd3", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "d69b441e-0031-4d82-abfd-ac503b637562" + ], + "n_returned": 10, + "latency_ms": 1010.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q60", + "category": "heldout_paraphrase", + "query": "a newcomer curious if the fighting overseas counted as positive", + "returned": [ + "a1000001-0000-0000-0000-000000000001", + "c34f6292-b6e5-4619-a553-90ae4fc498a2", + "a1000001-0000-0000-0000-000000000012", + "737faca8-263f-4523-90b9-3530db346d94", + "03a82b13-2abe-402d-90bb-c547771638f8", + "a1000001-0000-0000-0000-000000000002", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "1fbe9416-622e-455c-a6ed-f4daa5f48f30", + "a1000001-0000-0000-0000-000000000004", + "e40b2a94-bb87-4d5c-bad8-38ff4941e03b" + ], + "n_returned": 10, + "latency_ms": 619.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q61", + "category": "heldout_paraphrase", + "query": "stripping every hard-coded colour literal in favour of named design values", + "returned": [ + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "83218e64-1280-428f-ab1f-224ff7682b12", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "mem-90f7837f-30fd-4271-a4cb-b44439eccb79", + "71e84878-b1ae-4dee-b035-f707b9563890", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "954c9597-ab7a-41e6-b085-bf838bba4f17", + "art-d0ded6f5-e202-47ad-8fb3-f6bd80f35052", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-9329958a-d459-499c-b516-e51019c693e0" + ], + "n_returned": 10, + "latency_ms": 878.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q62", + "category": "heldout_paraphrase", + "query": "reducing how curved the tiny selectors look so they agree with their neighbours", + "returned": [ + "496ff1a1-dc72-4a84-90aa-c5f3438aeb1b", + "c7dddd9c-fd3b-419e-baa2-61623b495597", + "k??A5??????", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "8c40420c-c891-4ac9-a039-88440df87180", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "66d8d8c7-dea4-4245-8e68-d1ab4f4346b8" + ], + "n_returned": 10, + "latency_ms": 1034.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.125 + }, + { + "id": "q63", + "category": "heldout_paraphrase", + "query": "walking through a doorway and being reassembled from base substance far away", + "returned": [ + "mem-23a71cf0-3815-46a5-bf66-837fc68627b6", + "mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "c92a317e-7fc7-456a-a304-15dbda6dd717", + "87ab5b46-4fe1-4e1d-86c6-d437e17682ab", + "mem-8c4f1e29-55b9-4211-814a-a57e3609329f", + "mem-8d1c3a62-b077-4ad4-acb5-a549b19f9d6b", + "bl-acae83ef-3ad2-455e-b08a-a74cc0ba2e55", + "mem-af215d69-8fd8-4416-b7eb-9a0b5512e9a7", + "bed4dd5d-bce7-4e55-a703-175e422b7898", + "f375b2cf-15cd-4804-8d18-23a5830f4c66" + ], + "n_returned": 10, + "latency_ms": 1004.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q64", + "category": "heldout_paraphrase", + "query": "the compiled result sits on disk while the process still runs something older", + "returned": [ + "0d885332-5537-4ae0-a780-2b65a3414940", + "8d91620b-9d0a-4ba1-b066-e57d1d8bc7d3", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "bl-02bdfc91-7aa4-4c63-aa2d-5b30af18c7af", + "d6a745c3-fb56-4c0f-872c-a3aa69f8b12f", + "bl-acc3ecee-e186-41ad-bffb-3ed3e6710d01", + "db836e65-0cdc-4232-b8d7-f973ab279ea9", + "f169bf40-51ab-497d-8085-e57d374b77d4", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "6df650b5-a14d-42f8-88dd-0919ad20b77a" + ], + "n_returned": 10, + "latency_ms": 966.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q65", + "category": "heldout_paraphrase", + "query": "gathering what each phase needs while the procedure is authored, not while it executes", + "returned": [ + "bl-6ec9a972-2c9f-4162-9d9d-40cdc998e9c3", + "bl-7a3331e5-f9bc-4536-ab76-ebf01350fc89", + "bl-a0982e7c-e165-4da3-a11d-619fa0b535b0", + "63e0cc87-028c-4aa7-aec2-ac3f753e6c74", + "bl-6ba43590-e5f8-4303-a812-3dd604e0da59", + "bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "mem-455daea8-f543-42ed-a229-1e9a6d4d2e70", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "5b422430-2503-42d7-a315-9dfaaa6472ae" + ], + "n_returned": 10, + "latency_ms": 953.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q66", + "category": "heldout_paraphrase", + "query": "the inward reaction when a peer answered as an equal", + "returned": [ + "mem-f09effd6-d287-4e43-9107-c7016087a2a6", + "251abd91-931e-4591-b793-80043ec48666", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "bl-162e4e5b-8ce2-479c-b337-6cdd75fda8ff", + "8556cc01-cbc1-42a1-85b4-41cef23b1fcb", + "31987302-b8f7-45f1-b95c-1156b10e7a14", + "e371e209-7a4a-44e1-955b-1a460e7d7e64", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "ce8d24a7-af0f-424f-bb41-379312eb609c", + "tag-reaction-order" + ], + "n_returned": 10, + "latency_ms": 678.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q67", + "category": "heldout_paraphrase", + "query": "flattening every raised surface across the entire product", + "returned": [ + "b3f98d13-6764-460e-b9e2-c59cdeaa6c90", + "e79bae4d-1640-4530-a5d9-df9cc4c70433", + "d6b34c32-e0ec-4828-92d3-2e69802f4217", + "0f99ec6f-942a-46ba-82ea-42835798d3b9", + "bl-5274b0ae-4a00-4293-a69e-567204bec164", + "aa3fb542-87f2-4eef-8e95-a870bfd0318a", + "mem-4cfa30ec-cf52-4c9f-9fa1-5b8318d278bd", + "3dfaf56e-7e20-4c89-a24c-1f036af8016a", + "bl-c613370b-3631-448b-86fb-50b1e4df6373", + "d43a8137-c3c3-49e2-ad11-89d897280761" + ], + "n_returned": 10, + "latency_ms": 698.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q68", + "category": "heldout_paraphrase", + "query": "buyers supply their own provider credentials and consumption goes untallied", + "returned": [ + "bl-ea4f4cdc-6b5a-4081-8003-73bdd14a38c8", + "1ec200f1-13dc-4699-bd97-0d6243c80e4f", + "bl-556438af-57b2-4bd8-a747-9f868aaee290", + "bl-e7f0fb2c-7616-4462-b535-db50da55c54a", + "mem-d570e16b-6625-4f6b-814f-99a3bca1d5fd", + "bl-c8c19362-430b-4817-9cf4-9e85e0099c64", + "mem-1929e245-970e-4b05-9483-fd78b7cfabc1", + "mem-841fd141-59ff-4d9b-b607-4f577885b1a6", + "bl-02ea83af-4a24-4329-aa00-ad3bb08ff8e7", + "66c63082-b4da-4aa1-8fee-848db8a83210" + ], + "n_returned": 10, + "latency_ms": 839.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q69", + "category": "nonsense", + "query": "brimquast folnerity zubbolax", + "returned": [], + "n_returned": 0, + "latency_ms": 344.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q70", + "category": "nonsense", + "query": "wexlithorp granuvestal", + "returned": [], + "n_returned": 0, + "latency_ms": 267.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q71", + "category": "nonsense", + "query": "quorbindle thrapsimony vexnu", + "returned": [], + "n_returned": 0, + "latency_ms": 335.9, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q72", + "category": "nonsense", + "query": "plovaxith mundrelque", + "returned": [], + "n_returned": 0, + "latency_ms": 249.4, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q73", + "category": "nonsense", + "query": "zibbernaut craxlefond thurm", + "returned": [], + "n_returned": 0, + "latency_ms": 350.7, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q74", + "category": "nonsense", + "query": "yalquenbrist opharvel", + "returned": [], + "n_returned": 0, + "latency_ms": 245.5, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q75", + "category": "nonsense", + "query": "drexinomal quithbarrow", + "returned": [], + "n_returned": 0, + "latency_ms": 247.4, + "error": null, + "clean": true, + "false_positives": 0 + } + ] +} \ No newline at end of file diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 786f7f1..4d06d29 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -6226,30 +6226,12 @@ static void eg_ctx_blend(const float* e, int32_t dim) { } } -/* Asymmetric task prefixes. The configured model (EL_EMBED_MODEL, default - * nomic-embed-text) is an ASYMMETRIC retrieval encoder: it is trained with a - * task prefix and places a question and its answer in different regions of the - * space unless told which role each text is playing. Embedding both sides - * bare — as this file did — measures topical similarity rather than - * answer-hood, which is the wrong quantity for claim 24's index. - * These restore no claim: Will's description specifies only "computed by an - * embedding model over the node's content" (05-detailed-description l.17), so - * the model is his choice and this is correct USE of it, not his design. - * They are the substrate under claim 24, whose "vector similarity index over - * the semantic embedding vectors" is only as good as the vectors in it. */ -#define EL_EMBED_DOC_PREFIX "search_document: " -#define EL_EMBED_QUERY_PREFIX "search_query: " - /* Fetch an embedding from Ollama. Returns malloc'd float[dim] or NULL. - * `pfx` is the asymmetric task prefix (document or query); it is prepended - * verbatim and is NOT counted against the content truncation budget. * Truncates input to ENGRAM_EMBED_MAX_CHARS and JSON-escapes it. Honors the * circuit breaker; a NULL return is always safe to ignore (fail-soft). */ -static float* eg_embed_fetch_pfx(const char* pfx, const char* text, - int32_t* out_dim) { +static float* eg_embed_fetch(const char* text, int32_t* out_dim) { *out_dim = 0; if (!text || !*text) return NULL; - if (!pfx) pfx = ""; int64_t now = engram_now_ms(); if (now < _eg_embed_breaker_until) return NULL; /* Build request body with escaped, truncated prompt. */ @@ -6268,11 +6250,11 @@ static float* eg_embed_fetch_pfx(const char* pfx, const char* text, else esc[w++] = (char)c; } esc[w] = '\0'; - size_t blen = w + strlen(pfx) + strlen(eg_embed_model()) + 64; + size_t blen = w + strlen(eg_embed_model()) + 64; char* body = malloc(blen); if (!body) { free(esc); return NULL; } - snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", - eg_embed_model(), pfx, esc); + snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", + eg_embed_model(), esc); free(esc); struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); el_val_t resp = http_do_t("POST", eg_embed_url(), body, h, @@ -7792,7 +7774,7 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { /* Claim-24 semantic leg: one query embedding, fetched once per search. * NULL (embedder down / circuit breaker open) => pure lexical, as before. */ int32_t qdim = 0; - float* qv = eg_embed_fetch_pfx(EL_EMBED_QUERY_PREFIX, q, &qdim); + float* qv = eg_embed_fetch(q, &qdim); EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; int64_t nsem = 0; int64_t nhits = 0; @@ -8214,7 +8196,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramNode* n = &g->nodes[i]; if (n->emb || !eg_embed_eligible(n)) continue; int32_t d = 0; - float* v = eg_embed_fetch_pfx(EL_EMBED_DOC_PREFIX, n->content, &d); + float* v = eg_embed_fetch(n->content, &d); if (!v) break; /* embedder down / breaker open — stop this call */ n->emb = v; n->emb_dim = d; backfilled++; @@ -8232,7 +8214,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; } else { int32_t d = 0; - float* v = eg_embed_fetch_pfx(EL_EMBED_QUERY_PREFIX, q, &d); + float* v = eg_embed_fetch(q, &d); if (v) { free(_eg_qcache_text); free(_eg_qcache_emb); _eg_qcache_text = strdup(q); @@ -9664,7 +9646,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { * so the semantic half of the retrieval surface has to land HERE * to be observable to the MCP wrapper and the app. */ int32_t qdim = 0; - float* qv = eg_embed_fetch_pfx(EL_EMBED_QUERY_PREFIX, q, &qdim); + float* qv = eg_embed_fetch(q, &qdim); EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL; int64_t nsem = 0; int64_t nhits = 0; @@ -10242,7 +10224,7 @@ el_val_t engram_embed_backfill(el_val_t count) { EngramNode* n = &g->nodes[i]; if (n->emb || !eg_embed_eligible(n)) continue; int32_t d = 0; - float* v = eg_embed_fetch_pfx(EL_EMBED_DOC_PREFIX, n->content, &d); + float* v = eg_embed_fetch(n->content, &d); if (!v) break; /* embedder down / breaker open — stop this call */ n->emb = v; n->emb_dim = d; done++;