diff --git a/tools/retrieval-eval/comparison-hybrid-vs-assoc.json b/tools/retrieval-eval/comparison-hybrid-vs-assoc.json new file mode 100644 index 0000000..210e59e --- /dev/null +++ b/tools/retrieval-eval/comparison-hybrid-vs-assoc.json @@ -0,0 +1,143 @@ +{ + "baseline": "hybrid-semantic", + "candidate": "assoc-leg", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q27", + "q28", + "q29", + "q31" + ], + "broken_by_candidate": [], + "discordant": 4, + "net_queries": 4, + "mcnemar_exact_p": 0.125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1228.5, + "latency_ms_p95": 1681.8, + "latency_ms_max": 1718.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-assoc-leg-r2.json b/tools/retrieval-eval/results-assoc-leg-r2.json new file mode 100644 index 0000000..3c134a6 --- /dev/null +++ b/tools/retrieval-eval/results-assoc-leg-r2.json @@ -0,0 +1,956 @@ +{ + "label": "assoc-leg-r2", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc", + "soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json", + "limit": 10, + "port": 7894, + "wall_clock_s": 51.8, + "child_pid": 87150, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1223.5, + "latency_ms_p95": 1676.1, + "latency_ms_max": 1740.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 306.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "project-engram", + "project-engram-lang", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-89a2", + "bl-13babd0c-582e-4e28-a9e4-a77e65925e5d", + "870ede67-3454-4e00-9988-46cb13a8a4e2", + "bl-3e433255-3710-49fc-a093-c25e71de2ccb", + "mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48" + ], + "n_returned": 10, + "latency_ms": 331.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 292.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 309.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_" + ], + "n_returned": 3, + "latency_ms": 331.0, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 295.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "????7???Ջ3", + "kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "ctx-63e3", + "?ǚ?7??????", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b" + ], + "n_returned": 10, + "latency_ms": 613.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6" + ], + "n_returned": 10, + "latency_ms": 538.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.375, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "? ?}&?#??X\b", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee" + ], + "n_returned": 10, + "latency_ms": 553.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.4444444444444444, + "recall@10": 0.4444444444444444, + "precision@5": 0.8, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "ԍ????X????", + "project-harmonic-framework", + "?ǚ?7??????", + "project-harmonic-framework_com", + "knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a", + "??????X??2c", + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "????7???Ջ3", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356" + ], + "n_returned": 10, + "latency_ms": 527.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.1111111111111111, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "? ?}&?#??X\b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35", + "mem-b8ecd23e-77ce-42f7-984c-f51453fec16d" + ], + "n_returned": 10, + "latency_ms": 546.0, + "error": null, + "hit@5": 1.0, + "recall@5": 0.5, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 9, + "latency_ms": 892.4, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2857142857142857, + "recall@10": 0.5, + "precision@5": 0.8, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "?", + "mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f", + "?", + "mem-7cd90611-88a3-423d-a38a-0db2812952fa", + "?", + "bl-07375bf9-a169-42cd-adb3-7d32b25982f0", + "?" + ], + "n_returned": 10, + "latency_ms": 754.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "????7???Ջ3", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c" + ], + "n_returned": 10, + "latency_ms": 1644.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef", + "kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1740.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "7?e?7???\f3?", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "?of?7???", + "? ?}&?#??X\b", + "????7???Ջ3", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e" + ], + "n_returned": 10, + "latency_ms": 1393.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491", + "7?e?7???\f3?", + "??o?'?B???k", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "? ?}&?#??X\b", + "?of?7???", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1118.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "7?e?7???\f3?", + "ctx-cc7f", + "?of?7???", + "ctx-4a41", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "a1000001-0000-0000-0000-000000000001", + "knw-729fc901-8335-44c4-9f3a-b150b4aa0915", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "ctx-175f" + ], + "n_returned": 9, + "latency_ms": 1440.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4" + ], + "n_returned": 10, + "latency_ms": 1316.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "? ?}&?#??X\b", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "? ?}&?#??X\b", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "? ?}&?#??X\b", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "7?e?7???\f3?" + ], + "n_returned": 10, + "latency_ms": 1620.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "? ?}&?#??X\b", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "? ?}&?#??X\b", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "ԍ????X????", + "??????X??2c", + "dR????X?-?S" + ], + "n_returned": 10, + "latency_ms": 1393.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "7?e?7???\f3?", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "?of?7???", + "dR????X?-?S", + "dz????Xƹ?i", + "ԍ????X????", + "??????X??2c", + "kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491" + ], + "n_returned": 10, + "latency_ms": 1676.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "????7???Ջ3", + "a1000001-0000-0000-0000-000000000001", + "?ǚ?7??????", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "7?e?7???\f3?", + "mem-ade9440f-f161-4c18-9b35-1976257e6ebb", + "?of?7???", + "ea95f600-8dfd-4c7e-b077-a93dc3cd3623" + ], + "n_returned": 10, + "latency_ms": 1534.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "mem-bbb126a1-b297-42bb-86be-796871829c94", + "mem-45022957-2d78-48aa-a714-16d6eca52e0f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "7?e?7???\f3?", + "? ?}&?#??X\b", + "'?T?a\"B~-?8" + ], + "n_returned": 10, + "latency_ms": 1567.2, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "?ǚ?7??????", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "7?e?7???\f3?", + "ctx-bb74", + "??S?7???", + "knw-f671966c-3387-4848-abca-b5deec122e00" + ], + "n_returned": 10, + "latency_ms": 1280.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "? ?}&?#??X\b", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "7?e?7???\f3?", + "mem-7b74cac0-905f-4c35-9688-fbcce105a177" + ], + "n_returned": 10, + "latency_ms": 1401.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc" + ], + "n_returned": 10, + "latency_ms": 1512.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.36363636363636365, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "art-e0bdf5d8-d163-491f-b649-453fee8b721d", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 1165.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.3076923076923077, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "? ?}&?#??X\b", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40" + ], + "n_returned": 10, + "latency_ms": 1230.6, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.36363636363636365, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "ctx-e5427d7d", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "ctx-bb74", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6", + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98" + ], + "n_returned": 10, + "latency_ms": 1223.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b" + ], + "n_returned": 9, + "latency_ms": 1235.8, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.18181818181818182, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "? ?}&?#??X\b", + "4f698ae6-c40e-464e-9798-50350991a188", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 1460.4, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.2727272727272727, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 750.0, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 531.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [ + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "?V?", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "?m?\\}Q??6??", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "?m?\\}Q??6??", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd" + ], + "n_returned": 10, + "latency_ms": 769.1, + "error": null, + "clean": false, + "false_positives": 10 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "7?e?7???\f3?", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "? ?}&?#??X\b", + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "? ?}&?#??X\b", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "?of?7???", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "? ?}&?#??X\b", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff" + ], + "n_returned": 10, + "latency_ms": 1327.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 10 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "? ?}&?#??X\b", + "12082f7e-e320-438b-bd65-083d8259748f", + "? ?}&?#??X\b", + "527ecb25-2587-47eb-8269-73be2431abd4", + "? ?}&?#??X\b", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "? ?}&?#??X\b", + "4f698ae6-c40e-464e-9798-50350991a188", + "? ?}&?#??X\b", + "be3b6036-6eca-44a7-8fdf-37b23edfdfd1" + ], + "n_returned": 10, + "latency_ms": 1687.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666, + "outranks": true, + "rank_correct": 6, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "7?e?7???\f3?", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "? ?}&?#??X\b", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "%???2??jH??", + "art-8a0870d5-a716-4672-8094-f7463af1265b", + "???Ͼd??W\b?", + "bl-556438af-57b2-4bd8-a747-9f868aaee290" + ], + "n_returned": 10, + "latency_ms": 1040.6, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 4 + } + ] +} \ No newline at end of file diff --git a/tools/retrieval-eval/results-assoc-leg.json b/tools/retrieval-eval/results-assoc-leg.json new file mode 100644 index 0000000..5aec308 --- /dev/null +++ b/tools/retrieval-eval/results-assoc-leg.json @@ -0,0 +1,956 @@ +{ + "label": "assoc-leg", + "soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc", + "soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5", + "corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json", + "corpus_nodes": 78768, + "corpus_edges": 14214, + "gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json", + "limit": 10, + "port": 7893, + "wall_clock_s": 52.4, + "child_pid": 87099, + "child_confirmed_dead": true, + "aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1228.5, + "latency_ms_p95": 1681.8, + "latency_ms_max": 1718.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "rows": [ + { + "id": "q01", + "category": "exact_rare", + "query": "unjailbreakable", + "returned": [ + "mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226" + ], + "n_returned": 1, + "latency_ms": 306.3, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q02", + "category": "exact_rare", + "query": "engram-migrate", + "returned": [ + "mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5", + "mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257", + "project-engram", + "project-engram-lang", + "mem-60778715-758c-4677-933d-fc39b8f94152", + "ctx-89a2", + "bl-13babd0c-582e-4e28-a9e4-a77e65925e5d", + "870ede67-3454-4e00-9988-46cb13a8a4e2", + "bl-3e433255-3710-49fc-a093-c25e71de2ccb", + "mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48" + ], + "n_returned": 10, + "latency_ms": 360.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q03", + "category": "exact_rare", + "query": "cartabandonedevent", + "returned": [ + "mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091" + ], + "n_returned": 1, + "latency_ms": 324.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q04", + "category": "exact_rare", + "query": "pre-apprenticeship", + "returned": [ + "mem-89c02aae-d3ca-43f9-9e5d-eb369896276c" + ], + "n_returned": 1, + "latency_ms": 313.7, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q05", + "category": "exact_rare", + "query": "inferencenodemanager", + "returned": [ + "mem-73969486-143f-4431-b5e6-6845d1cc9848", + "bl-c1765767-3e27-449a-8c94-10411d1eb7c0", + "project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_" + ], + "n_returned": 3, + "latency_ms": 331.5, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q06", + "category": "exact_rare", + "query": "clear-eyed", + "returned": [ + "knw-c72597c5-c23d-4c08-8e9e-996dadf26a99" + ], + "n_returned": 1, + "latency_ms": 303.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q07", + "category": "phrase", + "query": "patterns not returns", + "returned": [ + "mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "????7???Ջ3", + "kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "ctx-63e3", + "?ǚ?7??????", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b" + ], + "n_returned": 10, + "latency_ms": 588.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 1.0 + }, + { + "id": "q08", + "category": "phrase", + "query": "thirty moves", + "returned": [ + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6" + ], + "n_returned": 10, + "latency_ms": 533.3, + "error": null, + "hit@5": 1.0, + "recall@5": 0.1875, + "recall@10": 0.375, + "precision@5": 0.6, + "mrr@10": 1.0 + }, + { + "id": "q09", + "category": "phrase", + "query": "Grandma Lucas", + "returned": [ + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "? ?}&?#??X\b", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee" + ], + "n_returned": 10, + "latency_ms": 542.4, + "error": null, + "hit@5": 1.0, + "recall@5": 0.4444444444444444, + "recall@10": 0.4444444444444444, + "precision@5": 0.8, + "mrr@10": 1.0 + }, + { + "id": "q10", + "category": "phrase", + "query": "Directed Harmonic", + "returned": [ + "ԍ????X????", + "project-harmonic-framework", + "?ǚ?7??????", + "project-harmonic-framework_com", + "knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a", + "??????X??2c", + "bl-dcee1887-34c4-4ffa-9119-1e291685ba08", + "????7???Ջ3", + "mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356" + ], + "n_returned": 10, + "latency_ms": 517.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.1111111111111111, + "precision@5": 0.0, + "mrr@10": 0.1111111111111111 + }, + { + "id": "q11", + "category": "phrase", + "query": "Sarah Bishop", + "returned": [ + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "? ?}&?#??X\b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "766de879-f9d0-4a07-b6df-b43ee13763d8", + "mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35", + "mem-b8ecd23e-77ce-42f7-984c-f51453fec16d" + ], + "n_returned": 10, + "latency_ms": 547.1, + "error": null, + "hit@5": 1.0, + "recall@5": 0.5, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q12", + "category": "phrase", + "query": "Directed Autonomous Runtime Modification", + "returned": [ + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff", + "mem-e6327f52-2bda-4ce7-9471-2fffd1e172de", + "bl-145a0985-2382-400f-a7c5-c335c5e30a72", + "mem-82b93b21-a865-410f-9ec1-fc54121d9bb5", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "7?e?7???\f3?", + "?of?7???", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 9, + "latency_ms": 903.4, + "error": null, + "hit@5": 1.0, + "recall@5": 0.2857142857142857, + "recall@10": 0.5, + "precision@5": 0.8, + "mrr@10": 1.0 + }, + { + "id": "q13", + "category": "phrase", + "query": "zero-knowledge encrypted backup", + "returned": [ + "7774a16c-1027-4e3b-a21e-67f1f95a4acd", + "8f3abb0d-77ed-4af3-9f4d-ba62cd198886", + "deda48cd-5e1a-46cb-bd43-8016afdb3a8a", + "?", + "mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f", + "?", + "mem-7cd90611-88a3-423d-a38a-0db2812952fa", + "?", + "bl-07375bf9-a169-42cd-adb3-7d32b25982f0", + "?" + ], + "n_returned": 10, + "latency_ms": 774.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.3333333333333333 + }, + { + "id": "q14", + "category": "paraphrase", + "query": "the elderly relative who passed while he stayed away", + "returned": [ + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "????7???Ջ3", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c" + ], + "n_returned": 10, + "latency_ms": 1641.0, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q15", + "category": "paraphrase", + "query": "a soldier sidelined by illness who refused to quit", + "returned": [ + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9", + "art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef", + "kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1718.6, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q16", + "category": "paraphrase", + "query": "choosing an uncomfortable fact over a pleasant fiction", + "returned": [ + "7?e?7???\f3?", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "?of?7???", + "? ?}&?#??X\b", + "????7???Ջ3", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "kn-d97920d0-1649-4223-9508-c0bb621e7fc0", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e" + ], + "n_returned": 10, + "latency_ms": 1409.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q17", + "category": "paraphrase", + "query": "a tight payload beats a bloated one", + "returned": [ + "kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491", + "7?e?7???\f3?", + "??o?'?B???k", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-e8423822-eacf-4029-aa7b-10d4d28d621e", + "? ?}&?#??X\b", + "?of?7???", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "? ?}&?#??X\b" + ], + "n_returned": 10, + "latency_ms": 1135.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q18", + "category": "paraphrase", + "query": "if you are able and nobody is coming the job is yours", + "returned": [ + "7?e?7???\f3?", + "ctx-cc7f", + "?of?7???", + "ctx-4a41", + "imp-dce1da0f-8776-4a9e-972b-33411a7ca138", + "a1000001-0000-0000-0000-000000000001", + "knw-729fc901-8335-44c4-9f3a-b150b4aa0915", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "ctx-175f" + ], + "n_returned": 9, + "latency_ms": 1446.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q19", + "category": "paraphrase", + "query": "learning is the wealth creditors cannot seize", + "returned": [ + "knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a", + "knw-ed33e669-0790-44cb-a036-958d605c6fea", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "knw-0087493b-25cd-45b0-bf46-c078c5b49718", + "knw-7902acca-604e-409b-8faf-ad85424211d0", + "knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a", + "kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4" + ], + "n_returned": 10, + "latency_ms": 1296.7, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q20", + "category": "paraphrase", + "query": "reliability proven by track record not assertion", + "returned": [ + "? ?}&?#??X\b", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "? ?}&?#??X\b", + "knw-f671966c-3387-4848-abca-b5deec122e00", + "? ?}&?#??X\b", + "a708dd6e-fe73-4f2f-a21e-89daa0985487", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "7?e?7???\f3?" + ], + "n_returned": 10, + "latency_ms": 1621.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q21", + "category": "paraphrase", + "query": "boundaries that enable instead of confine", + "returned": [ + "? ?}&?#??X\b", + "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "? ?}&?#??X\b", + "project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_", + "? ?}&?#??X\b", + "? ?}&?#??X\b", + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "ԍ????X????", + "??????X??2c", + "dR????X?-?S" + ], + "n_returned": 10, + "latency_ms": 1411.8, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5 + }, + { + "id": "q22", + "category": "paraphrase", + "query": "what shifts tells you where to cut a system apart", + "returned": [ + "7?e?7???\f3?", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "kn-f8974b26-78a6-4aad-b893-19a73b20013d", + "kn-9397c74b-35f3-4428-b4b0-5123353bbcd1", + "?of?7???", + "dR????X?-?S", + "dz????Xƹ?i", + "ԍ????X????", + "??????X??2c", + "kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491" + ], + "n_returned": 10, + "latency_ms": 1681.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q23", + "category": "paraphrase", + "query": "a mind that compounds instead of resetting each day", + "returned": [ + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8", + "????7???Ջ3", + "a1000001-0000-0000-0000-000000000001", + "?ǚ?7??????", + "mem-024598a9-ed2e-4eeb-b1e1-5410856ff132", + "7?e?7???\f3?", + "mem-ade9440f-f161-4c18-9b35-1976257e6ebb", + "?of?7???", + "ea95f600-8dfd-4c7e-b077-a93dc3cd3623" + ], + "n_returned": 10, + "latency_ms": 1536.8, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q24", + "category": "paraphrase", + "query": "loved for the unedited self and not the polished exterior", + "returned": [ + "mem-bbb126a1-b297-42bb-86be-796871829c94", + "mem-45022957-2d78-48aa-a714-16d6eca52e0f", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "knw-35940684-abc4-42f0-b942-818f66b1f69a", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "art-0f0277a1-4a8e-4645-95dd-fa379976f31c", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "7?e?7???\f3?", + "? ?}&?#??X\b", + "'?T?a\"B~-?8" + ], + "n_returned": 10, + "latency_ms": 1567.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q25", + "category": "paraphrase", + "query": "cheerfulness you arrive at instead of assuming", + "returned": [ + "?ǚ?7??????", + "knw-08559f5c-2306-4220-a146-398c74f1643c", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b", + "7?e?7???\f3?", + "ctx-bb74", + "??S?7???", + "knw-f671966c-3387-4848-abca-b5deec122e00" + ], + "n_returned": 10, + "latency_ms": 1278.3, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q26", + "category": "paraphrase", + "query": "a childhood offering no solid foundation to inherit", + "returned": [ + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "? ?}&?#??X\b", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "7?e?7???\f3?", + "mem-7b74cac0-905f-4c35-9688-fbcce105a177" + ], + "n_returned": 10, + "latency_ms": 1401.1, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q27", + "category": "associative", + "query": "Grandma Lucas stroke February 2006 goodbye window", + "returned": [ + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-e0423482-cfa5-4796-8689-8495c93b66bc" + ], + "n_returned": 10, + "latency_ms": 1509.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.36363636363636365, + "precision@5": 0.2, + "mrr@10": 0.2 + }, + { + "id": "q28", + "category": "associative", + "query": "Marines hernia sepsis medical ward", + "returned": [ + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "54608b69-78b6-4239-b60f-b8206cfecacc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "art-e0bdf5d8-d163-491f-b649-453fee8b721d", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 1156.7, + "error": null, + "hit@5": 1.0, + "recall@5": 0.07692307692307693, + "recall@10": 0.3076923076923077, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q29", + "category": "associative", + "query": "Sarah Bishop Dyer trailer performance", + "returned": [ + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "? ?}&?#??X\b", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "? ?}&?#??X\b", + "015644f5-8194-4af0-800d-dd4a0cd71396", + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "kn-5de5a9ac-fd15-45ab-bf18-77566781cf40" + ], + "n_returned": 10, + "latency_ms": 1232.2, + "error": null, + "hit@5": 1.0, + "recall@5": 0.09090909090909091, + "recall@10": 0.36363636363636365, + "precision@5": 0.2, + "mrr@10": 0.25 + }, + { + "id": "q30", + "category": "associative", + "query": "Swarm Architecture containment lateral worker", + "returned": [ + "art-ee615cdb-e599-423d-9a4d-977859390ed3", + "bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43", + "ctx-e5427d7d", + "kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9", + "ctx-bb74", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "kn-6f248a50-355b-47bb-aec8-e0e646a9b077", + "kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6", + "bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae", + "bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98" + ], + "n_returned": 10, + "latency_ms": 1228.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0 + }, + { + "id": "q31", + "category": "associative", + "query": "hope won inside the narrative preface", + "returned": [ + "kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "kn-5b606390-a52d-4ca2-8e0e-eba141d13440", + "knw-4aebd815-4eaf-49d7-954b-03595f3d48be", + "kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "knw-9e74ee95-ba7d-49b1-9262-977eae9729d1", + "knw-d357b6bb-ad8a-4791-b516-426aea45fa5b" + ], + "n_returned": 9, + "latency_ms": 1251.6, + "error": null, + "hit@5": 1.0, + "recall@5": 0.18181818181818182, + "recall@10": 0.18181818181818182, + "precision@5": 0.4, + "mrr@10": 0.5 + }, + { + "id": "q32", + "category": "associative", + "query": "man of the house six years old expectation", + "returned": [ + "? ?}&?#??X\b", + "4f698ae6-c40e-464e-9798-50350991a188", + "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "kn-6061318f-046b-4935-907d-8eafdce14930", + "? ?}&?#??X\b", + "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "? ?}&?#??X\b", + "kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "? ?}&?#??X\b", + "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71" + ], + "n_returned": 10, + "latency_ms": 1459.9, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.2727272727272727, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666 + }, + { + "id": "q33", + "category": "nonsense", + "query": "zqxjvw plimforth grebulon", + "returned": [], + "n_returned": 0, + "latency_ms": 746.6, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q34", + "category": "nonsense", + "query": "flarnbistle quommetry", + "returned": [], + "n_returned": 0, + "latency_ms": 519.1, + "error": null, + "clean": true, + "false_positives": 0 + }, + { + "id": "q35", + "category": "nonsense", + "query": "xxqzzt vurblenacht throom", + "returned": [ + "art-4a99aa1a-489b-4b43-958b-25217adb1aad", + "?V?", + "knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca", + "knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4", + "?m?\\}Q??6??", + "8cbb60c5-4999-4ec1-8682-2592aedc4249", + "art-79042b8b-6192-440f-90b0-60708f7e6325", + "?m?\\}Q??6??", + "knw-723551f5-1950-42a3-8b89-b6a06913cef0", + "art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd" + ], + "n_returned": 10, + "latency_ms": 764.0, + "error": null, + "clean": false, + "false_positives": 10 + }, + { + "id": "q36", + "category": "superseded", + "query": "is the self-improvement architecture called DARMA or DHARMA", + "returned": [ + "7?e?7???\f3?", + "mem-80d7416b-20e9-48a0-b176-b215527e2f56", + "? ?}&?#??X\b", + "mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da", + "? ?}&?#??X\b", + "kn-b7e98d63-8b83-4911-b4d0-990602a7f575", + "?of?7???", + "knw-e047bb42-dc5b-4383-9e88-e508dc03abe3", + "? ?}&?#??X\b", + "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff" + ], + "n_returned": 10, + "latency_ms": 1346.9, + "error": null, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "precision@5": 0.2, + "mrr@10": 0.5, + "outranks": true, + "rank_correct": 2, + "rank_stale": 10 + }, + { + "id": "q37", + "category": "superseded", + "query": "how many provisional patents does Will actually have", + "returned": [ + "? ?}&?#??X\b", + "12082f7e-e320-438b-bd65-083d8259748f", + "? ?}&?#??X\b", + "527ecb25-2587-47eb-8269-73be2431abd4", + "? ?}&?#??X\b", + "3cf706a1-3825-45d8-b0a9-06cae6cdf5b8", + "? ?}&?#??X\b", + "4f698ae6-c40e-464e-9798-50350991a188", + "? ?}&?#??X\b", + "be3b6036-6eca-44a7-8fdf-37b23edfdfd1" + ], + "n_returned": 10, + "latency_ms": 1687.1, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 1.0, + "precision@5": 0.0, + "mrr@10": 0.16666666666666666, + "outranks": true, + "rank_correct": 6, + "rank_stale": null + }, + { + "id": "q38", + "category": "superseded", + "query": "is MCP still the live integration layer", + "returned": [ + "kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356", + "bl-7328cbe3-0200-43c2-88e7-0a164e15fca4", + "7?e?7???\f3?", + "mem-101e81b4-8097-4749-8d8d-7bb66de34517", + "? ?}&?#??X\b", + "4509ed62-9fb2-48b8-9038-ac569fca9604", + "%???2??jH??", + "art-8a0870d5-a716-4672-8094-f7463af1265b", + "???Ͼd??W\b?", + "bl-556438af-57b2-4bd8-a747-9f868aaee290" + ], + "n_returned": 10, + "latency_ms": 1030.5, + "error": null, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "precision@5": 0.0, + "mrr@10": 0.0, + "outranks": false, + "rank_correct": null, + "rank_stale": 4 + } + ] +} \ No newline at end of file diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 815021e..3fe3fad 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -7463,6 +7463,176 @@ static int64_t engram_interleave(const EngramRankEntry* L, int64_t nL, return no; } +/* ── Associative leg (claim 10 typed relations + claim 1 activation) ──────── + * + * WHY THIS EXISTS. engram_search_json has two legs, and neither can reach a + * node that shares no token with the query and no embedding neighbourhood + * with it. The route the design reserves for that case is the graph: a node + * is reachable because it is STRUCTURALLY associated with something the query + * did hit. Nothing on the recall path consults an edge today. + * + * WHY NOT engram_activate. Wiring recall wholesale to activation was measured + * (PR #135) and lost 57 points of phrase accuracy. The failure was one of + * RANK, not reach: activation seeds on every token-matching node, so a 2-hop + * associate at strength ~0.06 never outranks thousands of 1-hop neighbours of + * strong lexical seeds. So this is a separate, small, ranked list that is + * merged BESIDE the other two, exactly as the semantic leg is. + * + * TYPED RELATIONS (06-claims.md claim 10). Expansion follows only relations + * that assert a structural claim about meaning. The corpus is 4,915 `tagged` + * and 3,767 `triggers-safety` edges against 475 `identity` and 11 `contains`; + * walking the bulk relations turns any seed into a flood (measured: 1,387 + * candidates from one seed) while the structural ones stay in the low tens. + * This is the first code on the read path to branch on a relation type at all. + * + * DIRECTION. Edges are walked in BOTH directions. Claim 23 requires the store + * to keep reverse edge records keyed by target id, and the adjacency index + * already materialises them (adj_to). It has to be both: every value node in + * this corpus has exactly ONE inbound edge (hub -> value) and no outbound + * structural edge at all, so a forward-only walk from a value node reaches + * nothing. Note this is an extension of the traversal as literally specified + * (05-detailed-description l.66 says "all outgoing edges"); the reverse index + * is designed and stored, but the description does not say the walk reads it. + */ +#define ENGRAM_ASSOC_SEEDS 3 /* top-N lexical hits form the context */ +#define ENGRAM_ASSOC_DEPTH 2 /* seed -> hub -> sibling */ +#define ENGRAM_ASSOC_FIRE 0.02 /* same firing threshold as engram_activate */ +#define ENGRAM_ASSOC_MAX 64 /* cap on candidates carried forward */ + +static int eg_rel_is_structural(const char* r) { + if (!r || !*r) return 0; + static const char* ok[] = { + "identity", "contains", "superseded_by", "references", "embodies", + "demonstrated_by", "canonical-self", "depends_on", "currently_holds", + "activates", NULL + }; + for (int i = 0; ok[i]; i++) if (strcmp(r, ok[i]) == 0) return 1; + return 0; +} + +/* Nodes that are index artefacts rather than recallable content. Same + * exclusion eg_embed_eligible() already applies when deciding what deserves an + * embedding, reused here so the associative leg cannot surface — or relay + * through — a Tag. Relaying through them is what makes a graph walk explode: + * tag-tier_note alone has 187 members. */ +static int eg_assoc_excluded(const EngramNode* n) { + if (!n->node_type) return 0; + return strcmp(n->node_type, "Tag") == 0 + || strcmp(n->node_type, "InternalStateEvent") == 0; +} + +/* Breadth-first structural expansion from `seeds`, then ORDER BY query + * similarity. Activation decides REACHABILITY (the conjunctive prune of + * claim 1: parent strength x edge weight x target salience, cut at the firing + * threshold); cosine decides ORDER within what was reached. Ranking the + * neighbourhood by activation alone does not work and the reason is + * structural: every identity edge in this corpus carries weight 0.5 and every + * value node salience 0.7, so the activation product degenerates into a + * function of hop count and ranks the relay hub above all of its own + * children. Composing the two is mine, not Will's — the description ranks the + * activation result set by strength (l.78). */ +static int64_t engram_assoc_leg(EngramStore* g, + const EngramRankEntry* L, int64_t nL, + const float* qv, int32_t qdim, + EngramSemEntry* out, int64_t out_cap) { + if (!g || nL <= 0 || !qv || qdim <= 0 || out_cap <= 0) return 0; + if (g->adj_dirty || !g->adj_from || !g->adj_to) engram_adj_rebuild(g); + if (!g->adj_from || !g->adj_to) return 0; + + double* act = calloc((size_t)g->node_count, sizeof(double)); + char* seen = calloc((size_t)g->node_count, sizeof(char)); + int64_t* q = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t)); + int64_t* hop = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t)); + if (!act || !seen || !q || !hop) { free(act); free(seen); free(q); free(hop); return 0; } + int64_t qcap = ENGRAM_ASSOC_MAX * 4, qh = 0, qt = 0; + + int64_t nseed = nL < ENGRAM_ASSOC_SEEDS ? nL : ENGRAM_ASSOC_SEEDS; + for (int64_t s = 0; s < nseed; s++) { + int64_t idx = L[s].idx; + if (idx < 0 || idx >= g->node_count) continue; + act[idx] = 1.0; seen[idx] = 2; /* 2 = seed: never a result */ + if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; } + } + + const double SPREAD_DECAY = 0.7; + while (qh < qt) { + int64_t cur = q[qh]; int64_t h = hop[qh]; qh++; + if (h >= ENGRAM_ASSOC_DEPTH) continue; + double parent = act[cur]; + int from_len = g->adj_from_len[cur]; + int to_len = g->adj_to_len[cur]; + for (int scan = 0; scan < from_len + to_len; scan++) { + int64_t ei = (scan < from_len) ? g->adj_from[cur][scan] + : g->adj_to[cur][scan - from_len]; + EngramEdge* e = &g->edges[ei]; + if (!eg_rel_is_structural(e->relation)) continue; + int64_t oi = (scan < from_len) ? engram_idmap_get(g, e->to_id) + : engram_idmap_get(g, e->from_id); + if (oi < 0 || oi >= g->node_count) continue; + EngramNode* on = &g->nodes[oi]; + if (eg_assoc_excluded(on)) continue; + double na = parent * e->weight * SPREAD_DECAY * on->salience; + if (na < ENGRAM_ASSOC_FIRE) continue; + if (seen[oi] && na <= act[oi]) continue; + act[oi] = na; + if (!seen[oi]) seen[oi] = 1; + if (qt < qcap) { q[qt] = oi; hop[qt] = h + 1; qt++; } + } + } + + int64_t n = 0; + for (int64_t i = 0; i < g->node_count && n < out_cap; i++) { + if (seen[i] != 1) continue; /* skip unreached and seeds */ + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + EngramNode* nd = &g->nodes[i]; + if (!nd->emb || nd->emb_dim != qdim) continue; + double c = eg_cosine(nd->emb, qv, qdim); + if (c <= 0.0) continue; + out[n].idx = i; out[n].sem = c; n++; + } + qsort(out, (size_t)n, sizeof(EngramSemEntry), engram_sem_cmp); + free(act); free(seen); free(q); free(hop); + return n; +} + +/* Three-leg merge: lexical, semantic, associative — strict rotation, + * L1, S1, A1, L2, S2, A2, ... deduplicated, capped at lim. + * + * The cost is explicit and worse than the two-leg case: a lexical hit at rank + * r lands at output position 3r-2 when both other legs are non-empty. That is + * survivable here only because the structural-relation filter leaves the + * associative list EMPTY for most queries — a Memory node whose only edges are + * `tagged` and `related` expands to nothing, so its ranking is untouched. */ +static int64_t engram_interleave3(const EngramRankEntry* L, int64_t nL, + const EngramSemEntry* S, int64_t nS, + const EngramSemEntry* A, int64_t nA, + int64_t lim, int64_t* out) { + int64_t no = 0, li = 0, si = 0, ai = 0; + while (no < lim && (li < nL || si < nS || ai < nA)) { + if (li < nL) { + int dup = 0; + for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; } + if (!dup) out[no++] = L[li].idx; + li++; + } + if (no >= lim) break; + if (si < nS) { + int dup = 0; + for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; } + if (!dup) out[no++] = S[si].idx; + si++; + } + if (no >= lim) break; + if (ai < nA) { + int dup = 0; + for (int64_t k = 0; k < no; k++) if (out[k] == A[ai].idx) { dup = 1; break; } + if (!dup) out[no++] = A[ai].idx; + ai++; + } + } + return no; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -9372,9 +9542,19 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { } qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp); + /* Claim-10 associative leg: expand the top lexical hits along + * structural relations only, order the reached set by query + * similarity. Empty whenever the seeds have no structural + * edges, which is the common case and is what keeps the + * lexical ordering safe. */ + EngramSemEntry* assoc = qv ? malloc((size_t)ENGRAM_ASSOC_MAX * sizeof(EngramSemEntry)) : NULL; + int64_t nassoc = assoc + ? engram_assoc_leg(g, hits, nhits, qv, qdim, assoc, ENGRAM_ASSOC_MAX) + : 0; int64_t* order = malloc((size_t)lim * sizeof(int64_t)); if (order) { - int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order); + int64_t no = engram_interleave3(hits, nhits, sem, nsem, + assoc, nassoc, lim, order); for (int64_t k = 0; k < no; k++) { if (!first) jb_putc(&b, ','); engram_emit_node_json(&b, &g->nodes[order[k]], 0); @@ -9382,6 +9562,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { } free(order); } + free(assoc); free(hits); free(sem); free(qv);