diff --git a/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json b/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json new file mode 100644 index 0000000..7f43706 --- /dev/null +++ b/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json @@ -0,0 +1,147 @@ +{ + "baseline": "semseed", + "candidate": "bm25lex", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q10", + "q11" + ], + "broken_by_candidate": [], + "discordant": 2, + "net_queries": 2, + "mcnemar_exact_p": 0.5, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-bm25lex-r2.json b/tools/retrieval-eval/results-bm25lex-r2.json new file mode 100644 index 0000000..2a60b4b --- /dev/null +++ b/tools/retrieval-eval/results-bm25lex-r2.json @@ -0,0 +1,959 @@ +{ + "label": "bm25lex-r2", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25lex", + "soul_md5": "dfbd0f8e3646212db5c60026f8ad906f", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json", + "limit": 10, + "port": 7894, + "wall_clock_s": 50.1, + "child_pid": 90371, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1183.8, + "latency_ms_p95": 1611.3, + "latency_ms_max": 1654.7, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 282.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "bl-ba764d70-e9d7-4f62-848f-719cb665f45e", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78", + "project-engram", + "ctx-45bc", + "project-engram-lang", + "ctx-175f", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-74ed" + ], + "n_returned": 10, + "latency_ms": 315.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 283.4, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 283.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_" + ], + "n_returned": 3, + "latency_ms": 303.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 283.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "tag-patterns", + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_", + "project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_", + "bl-79028eed-c330-4724-9402-734062d13503", + "bl-39dad13d-7105-4049-8224-dc3c34fdb1f3", + "bl-4ef4d914-da46-4e0f-be78-5219b9547e9f", + "bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0", + "bl-1d32bd54-cf17-4a1f-b235-982d09a36f04", + "bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5" + ], + "n_returned": 10, + "latency_ms": 587.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6" + ], + "n_returned": 10, + "latency_ms": 518.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.375, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "kn-f230b362-b201-4402-9833-4160c89ab3d4" + ], + "n_returned": 10, + "latency_ms": 511.3, + "error": null, + "hit@5": 1.0, + "recall@5": 0.3333333333333333, + "recall@10": 0.4444444444444444, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "project-harmonic-framework", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "project-harmonic-framework_com", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "tag-harmonic-design", + "bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76", + "tag-harmonic-framework", + "bl-18a9d1e4-1484-474c-bf6b-c6173212181b" + ], + "n_returned": 10, + "latency_ms": 497.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1111111111111111, + "recall@10": 0.1111111111111111, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-sarah", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0", + "kn-6061318f-046b-4935-907d-8eafdce14930" + ], + "n_returned": 10, + "latency_ms": 510.2, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.4, + "mrr@10": 1.0 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "bl-6f99e111-7055-4635-9831-a489747ce418", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae", + "bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25", + "2c56a7a9-5323-4ce4-ba09-35836ba15d54", + "bl-39cec462-c80c-4970-a3aa-91fe83053bde" + ], + "n_returned": 10, + "latency_ms": 870.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.21428571428571427, + "recall@10": 0.2857142857142857, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "?", + "bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315", + "?", + "830ca37a-d334-4e41-ba89-64893dc8d628", + "?", + "ce9636dc-85a5-4dae-9e07-74ea2fcc6307", + "?" + ], + "n_returned": 10, + "latency_ms": 723.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "cb070131-dfd4-4a38-91d7-22b1bde164d2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "knw-d788a210-613b-4c49-9486-88bbc9d4716f", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "66c63082-b4da-4aa1-8fee-848db8a83210", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-a535f205-bc4c-4058-9171-6263c496044a", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "ctx-4a41" + ], + "n_returned": 10, + "latency_ms": 1584.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "b1183213-d659-4759-85d7-5b1f22427fe2", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-16efddd1-c43d-4a42-9d78-f54fb82bd277", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "f0eb6b13-909c-4674-91ef-23301d3abc8b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "30a44d10-2487-420e-bf61-3892e4343c92", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "bfb5809e-d19a-4d3f-8c1a-796db622ad9d" + ], + "n_returned": 10, + "latency_ms": 1652.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "mem-ef878e30-5851-4e82-8588-745415108941", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "tag-fiction", + "knw-8fd9836c-cc39-49df-8d61-babda626cc88", + "mem-8d690e9d-a7e9-4062-b2f8-e2064294e463", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "mem-ce793303-c5a5-4586-a232-a3426edd9ec7", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "mem-443bd012-fc9a-4088-b236-de5157a1ef92" + ], + "n_returned": 10, + "latency_ms": 1341.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5", + "bl-798d135f-3987-4ccd-8de6-70ca2f358337", + "bl-680b24a9-edc3-4a9d-847a-bff0b46b568c", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "bl-164b520b-c503-49db-89f9-bd2fdf4215f5", + "knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "1219277c-1b95-45ec-95a2-07b4a47a4d92", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e", + "bl-79ce4464-5dd6-49bd-9b0c-9803549d0665" + ], + "n_returned": 10, + "latency_ms": 1064.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "bl-4476e856-c567-4b49-8ff7-d7dca3e5715e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "a1000001-0000-0000-0000-000000000010", + "a1000001-0000-0000-0000-000000000009", + "bl-448bc514-c2f1-4520-a9b1-1f3a73678d26", + "a1000001-0000-0000-0000-000000000012", + "43098881-e044-482b-8e92-471728a8ba8b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "mem-e5cc63c0-8701-49d6-855a-e387fe087771", + "a1000001-0000-0000-0000-000000000001" + ], + "n_returned": 10, + "latency_ms": 1398.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "tag-learning", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "d6b12ecf-702b-4101-b1bb-09ed9b220b29", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "c608a095-c98b-4bfa-bfe1-1611c1320290", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "451ae007-4219-4096-89fe-fa2e045fbeb1", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 1252.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "mem-cde58b77-50d3-4bac-9581-e70a4c02c015", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "mem-53d6adf0-cd08-4707-a237-daa5e65c7298", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "bl-ef2bac68-e119-4139-b529-c7a1404ae3ac" + ], + "n_returned": 10, + "latency_ms": 1581.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf", + "bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd", + "bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3", + "bl-34f51ddb-a840-459f-a248-94214f5febb6", + "bl-286b562a-5299-40e0-a32a-afa9cbdfe995" + ], + "n_returned": 10, + "latency_ms": 1363.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "5a2c118a-87bd-4239-97a7-9e02c5991983", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "mem-ef878e30-5851-4e82-8588-745415108941", + "knw-12b4b913-7a25-4b0d-844c-504c01d6725e", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "knw-9707256e-ed44-4042-bd88-f90fa514e1cf", + "kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "bl-4c5b385e-135a-4663-8521-96af0b491121", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21" + ], + "n_returned": 10, + "latency_ms": 1611.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "a0edad47-5f77-4fc3-a546-1e85f8c68e77", + "a1000001-0000-0000-0000-000000000001", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "96497334-b18f-495c-9228-eeb8182bdc38", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "knw-ed33e669-0790-44cb-a036-958d605c6fea" + ], + "n_returned": 10, + "latency_ms": 1470.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "27e1b1a4-ad0b-49d9-812f-fedf43b8aabe", + "knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa", + "bl-87c93185-b2bf-40af-ae23-3c830c007abf", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "077d064f-3489-4c05-9aca-3782f96b51db", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "75e036d3-c170-4e3f-acc2-e456a6850ee2", + "a1000001-0000-0000-0000-000000000001" + ], + "n_returned": 10, + "latency_ms": 1513.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "mem-82158b02-a180-435d-84f0-0b7ce37511b4", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "e4f27651-52c5-43fd-aff3-61d31685b3cd", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-833dbbcd-2400-4594-bb35-93b023049ac0", + "a1000001-0000-0000-0000-000000000009", + "mem-759e78ca-5394-4244-aa39-1c1468bc5f3e", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd" + ], + "n_returned": 10, + "latency_ms": 1226.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "bl-3f57bc69-7285-4f4a-a861-2de52efca058", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-f230b362-b201-4402-9833-4160c89ab3d4", + "bl-0d8c5dfa-e163-4fef-a58b-56b0d076c5a8", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "mem-b99efff0-00e6-40c8-9c5b-730330eef33b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "tag-childhood" + ], + "n_returned": 10, + "latency_ms": 1342.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "3499d5da-0e9c-4de4-9bc4-8941b14e0b1f", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c" + ], + "n_returned": 10, + "latency_ms": 1433.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.45454545454545453, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 1104.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.3076923076923077, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "tag-trailer-park-paladins", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "project-trailer-park-paladins", + "kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c" + ], + "n_returned": 10, + "latency_ms": 1183.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.45454545454545453, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "bl-2515d870-e35e-443b-ba20-5150bbc73fed", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "bl-bea7473c-c687-414c-9c0b-00c509a616c1", + "bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "mem-ab34c2f7-3243-424b-affa-25555f6cf9cc" + ], + "n_returned": 10, + "latency_ms": 1199.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "tag-hope", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e" + ], + "n_returned": 10, + "latency_ms": 1202.9, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.18181818181818182, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "knw-729fc901-8335-44c4-9f3a-b150b4aa0915", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "knw-473f3f24-20f6-4f39-8589-3709538eb6ac", + "?Z?=DEPTH: continue + p=act[cur] + for oid,w in adj.get(cur,()): + n=N.get(oid) + if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue + na=p*w*DECAY*float(n.get('salience') or 0.0) + if na=lim: break + if si=lim: break + if aiSEED_MIN] + if guard: S=[x for x in S if PRINT.match(x or '')] + seeds=[x for x in L[:3] if x in N] + seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and (not guard or PRINT.match(eids[j] or ''))] + A=assoc(seeds,s) if seeds else [] + if guard: A=[x for x in A if PRINT.match(x or '')] + res[qid]=inter3(L,S,A); legs[qid]=(L,S,A) + return res,legs +def score(res,label,verbose=False): + det={} + for qid,q in gold.items(): + out=res[qid][:5] + if q['category']=='nonsense': ok=(len(res[qid])==0) + elif q['category']=='superseded': + must=q.get('must_outrank') or {}; ok=False + for good,bad in (must.items() if isinstance(must,dict) else []): + ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)