diff --git a/awareness.el b/awareness.el index 6cd74ad..ba4cbf5 100644 --- a/awareness.el +++ b/awareness.el @@ -587,7 +587,77 @@ fn emit_heartbeat() -> Void { // neuron-api label fix. Sentinel-shaped labels ("knowledge:captured", // "memory:remembered" — colon, no space) carry no seed signal and are // skipped so legacy nodes cannot seed the scan with the word "knowledge". -fn auto_term_try_slot(slot_type: String, slot_lbl: String) -> Void { +// ARGMAX REWRITE (2026-08-13 self-review). auto_term_empty_streak — the +// counter the 2026-08-06 review added to catch exactly this — read 50 and +// climbing: fifty consecutive scans where dynamic seeding produced nothing +// and the loop ran on its four hardcoded phrases. The live WM top said why: +// every one of the top slots was a Memory node labelled "memory:remembered". +// This function read the LABEL only, the sentinel guard below (correctly) +// rejects sentinels, so there was never anything to extract. The extractor +// was written against Knowledge nodes, which have real titles, and was +// structurally blind to the node type that actually dominates WM. +// +// Rather than add a sixth guard to the five below, the selection algorithm +// is now inverted and lives in the runtime: engram_salient_term() scores +// EVERY candidate token in the node's text and returns the argmax of +// idf·position·casing (YAKE, Campos et al. 2020, with real corpus IDF +// substituted for YAKE's corpus-free proxies), falling back from a sentinel +// label to the node's content. Term quality is now the selection criterion +// instead of a veto, so a bad token loses to a better token in the same text +// without needing to be on any list. Tabu is applied during the argmax, so +// inhibition-of-return costs seed quality rather than costing the scan. +// +// MEASURED BEFORE SHIPPING, on 60 live Memory nodes: 0 empty, versus 60 of 60 +// empty under the old extractor. Terms produced are topical — HEBBIAN, +// CONSOLIDATION, TEMPORAL, crash-loop, PRIMING, NEIGHBORHOOD, DRIFT. Three of +// sixty are weak header words ("STEP", "DONE"). They are left alone +// deliberately: adding them to a list is the exact move that produced four +// previous blocklists, and a mediocre seed on 5% of scans is not a flood. +// +// The stopword list below STAYS, and not as belt-and-braces. An earlier draft +// of this change assumed the min_df floor would subsume it, on 08-03's +// finding that function words have df 0 in labels. Re-measured under +// word-boundary df: about:2, whole:1, them:2 — they clear a floor of 1. What +// keeps them from winning is the argmax, not the floor. The list still earns +// its keep on the Title-case cases. +// +// What stays here is policy: the node-type filter, the df thresholds, and the +// stopword list. The runtime measures; the soul decides. Same split as +// engram_label_df. +fn auto_term_try_slot(slot_type: String, slot_id: String) -> Void { + state_set("_ats_ok", "0") + if str_eq(slot_type, "Memory") { state_set("_ats_ok", "1") } + if str_eq(slot_type, "BacklogItem") { state_set("_ats_ok", "1") } + if str_eq(slot_type, "Entity") { state_set("_ats_ok", "1") } + if str_eq(slot_type, "Knowledge") { state_set("_ats_ok", "1") } + if str_eq(state_get("_ats_ok"), "1") { + if !str_eq(slot_id, "") { + // Tabu ring, pipe-delimited, excluded inside the argmax. + let tabu: String = "|" + state_get("soul.tabu_t0") + + "|" + state_get("soul.tabu_t1") + + "|" + state_get("soul.tabu_t2") + + "|" + state_get("soul.tabu_t3") + "|" + let df_max: Int = engram_node_count() / 400 + let df_cap: Int = if df_max > 8 { df_max } else { 8 } + let term: String = engram_salient_term(slot_id, df_cap, 1, tabu) + if !str_eq(term, "") { + state_set("_ats_gw", "0") + let stopw: String = "|What|When|Where|Which|Whose|While|This|That|These|Those|There|Their|Then|Than|With|Without|From|Into|Onto|Over|Under|About|Between|Among|Across|Some|Most|More|Less|Very|Each|Every|Both|Also|Only|Just|Does|Will|Would|Could|Should|Might|Must|Have|Been|Being|Toward|Towards|Using|Based|Upon|Here|Your|Ours|They|Them|what|this|that|with|from|context|Context|Prose|Colon|Self|Test|Testing|Closing|Global|Universal|Persona|Semantic|Spreading|Temporal|Numeric|Register|Identifying|Introduction|Overview|Summary|Section|General|Notes|Note|" + if str_contains(stopw, "|" + term + "|") { state_set("_ats_gw", "1") } + if str_eq(state_get("_ats_gw"), "0") { + state_set("cseed_auto", term) + } + } + } + } + return "" +} + +// SUPERSEDED 2026-08-13 — retained for the record. The first-word extractor +// and its five accumulated guards, replaced by the argmax above. Kept +// unreferenced so the reasoning behind each guard stays readable next to what +// replaced it; delete once engram_salient_term has a month of live telemetry. +fn auto_term_try_slot_legacy(slot_type: String, slot_lbl: String) -> Void { state_set("_ats_ok", "0") if str_eq(slot_type, "Memory") { state_set("_ats_ok", "1") } if str_eq(slot_type, "BacklogItem") { state_set("_ats_ok", "1") } @@ -806,16 +876,18 @@ fn proactive_curiosity() -> Bool { let wm10_n2: String = json_array_get(wm10, 2) let wm10_n1: String = json_array_get(wm10, 1) let wm10_n0: String = json_array_get(wm10, 0) - auto_term_try_slot(json_get(wm10_n9, "node_type"), json_get(wm10_n9, "label")) - auto_term_try_slot(json_get(wm10_n8, "node_type"), json_get(wm10_n8, "label")) - auto_term_try_slot(json_get(wm10_n7, "node_type"), json_get(wm10_n7, "label")) - auto_term_try_slot(json_get(wm10_n6, "node_type"), json_get(wm10_n6, "label")) - auto_term_try_slot(json_get(wm10_n5, "node_type"), json_get(wm10_n5, "label")) - auto_term_try_slot(json_get(wm10_n4, "node_type"), json_get(wm10_n4, "label")) - auto_term_try_slot(json_get(wm10_n3, "node_type"), json_get(wm10_n3, "label")) - auto_term_try_slot(json_get(wm10_n2, "node_type"), json_get(wm10_n2, "label")) - auto_term_try_slot(json_get(wm10_n1, "node_type"), json_get(wm10_n1, "label")) - auto_term_try_slot(json_get(wm10_n0, "node_type"), json_get(wm10_n0, "label")) + // 2026-08-13: pass the node ID, not the label. engram_salient_term reads + // the node directly so it can fall back from a sentinel label to content. + auto_term_try_slot(json_get(wm10_n9, "node_type"), json_get(wm10_n9, "id")) + auto_term_try_slot(json_get(wm10_n8, "node_type"), json_get(wm10_n8, "id")) + auto_term_try_slot(json_get(wm10_n7, "node_type"), json_get(wm10_n7, "id")) + auto_term_try_slot(json_get(wm10_n6, "node_type"), json_get(wm10_n6, "id")) + auto_term_try_slot(json_get(wm10_n5, "node_type"), json_get(wm10_n5, "id")) + auto_term_try_slot(json_get(wm10_n4, "node_type"), json_get(wm10_n4, "id")) + auto_term_try_slot(json_get(wm10_n3, "node_type"), json_get(wm10_n3, "id")) + auto_term_try_slot(json_get(wm10_n2, "node_type"), json_get(wm10_n2, "id")) + auto_term_try_slot(json_get(wm10_n1, "node_type"), json_get(wm10_n1, "id")) + auto_term_try_slot(json_get(wm10_n0, "node_type"), json_get(wm10_n0, "id")) let auto_term: String = state_get("cseed_auto") let results_auto: String = if str_eq(auto_term, "") { "[]" } else { engram_activate_json(auto_term, 1) } let found_auto: Int = json_array_len(results_auto) diff --git a/dist/awareness.c b/dist/awareness.c index 4114904..aed7fff 100644 --- a/dist/awareness.c +++ b/dist/awareness.c @@ -27,7 +27,8 @@ el_val_t elapsed_ms(void); el_val_t elapsed_human(void); el_val_t embed_ok(void); el_val_t emit_heartbeat(void); -el_val_t auto_term_try_slot(el_val_t slot_type, el_val_t slot_lbl); +el_val_t auto_term_try_slot(el_val_t slot_type, el_val_t slot_id); +el_val_t auto_term_try_slot_legacy(el_val_t slot_type, el_val_t slot_lbl); el_val_t proactive_curiosity(void); el_val_t pulse_count(void); el_val_t pulse_inc(void); @@ -323,7 +324,43 @@ el_val_t emit_heartbeat(void) { return 0; } -el_val_t auto_term_try_slot(el_val_t slot_type, el_val_t slot_lbl) { +el_val_t auto_term_try_slot(el_val_t slot_type, el_val_t slot_id) { + state_set(EL_STR("_ats_ok"), EL_STR("0")); + if (str_eq(slot_type, EL_STR("Memory"))) { + state_set(EL_STR("_ats_ok"), EL_STR("1")); + } + if (str_eq(slot_type, EL_STR("BacklogItem"))) { + state_set(EL_STR("_ats_ok"), EL_STR("1")); + } + if (str_eq(slot_type, EL_STR("Entity"))) { + state_set(EL_STR("_ats_ok"), EL_STR("1")); + } + if (str_eq(slot_type, EL_STR("Knowledge"))) { + state_set(EL_STR("_ats_ok"), EL_STR("1")); + } + if (str_eq(state_get(EL_STR("_ats_ok")), EL_STR("1"))) { + if (!str_eq(slot_id, EL_STR(""))) { + el_val_t tabu = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("|"), state_get(EL_STR("soul.tabu_t0"))), EL_STR("|")), state_get(EL_STR("soul.tabu_t1"))), EL_STR("|")), state_get(EL_STR("soul.tabu_t2"))), EL_STR("|")), state_get(EL_STR("soul.tabu_t3"))), EL_STR("|")); + el_val_t df_max = (engram_node_count() / 400); + el_val_t df_cap = ({ el_val_t _if_result_73 = 0; if ((df_max > 8)) { _if_result_73 = (df_max); } else { _if_result_73 = (8); } _if_result_73; }); + el_val_t term = engram_salient_term(slot_id, df_cap, 1, tabu); + if (!str_eq(term, EL_STR(""))) { + state_set(EL_STR("_ats_gw"), EL_STR("0")); + el_val_t stopw = EL_STR("|What|When|Where|Which|Whose|While|This|That|These|Those|There|Their|Then|Than|With|Without|From|Into|Onto|Over|Under|About|Between|Among|Across|Some|Most|More|Less|Very|Each|Every|Both|Also|Only|Just|Does|Will|Would|Could|Should|Might|Must|Have|Been|Being|Toward|Towards|Using|Based|Upon|Here|Your|Ours|They|Them|what|this|that|with|from|context|Context|Prose|Colon|Self|Test|Testing|Closing|Global|Universal|Persona|Semantic|Spreading|Temporal|Numeric|Register|Identifying|Introduction|Overview|Summary|Section|General|Notes|Note|"); + if (str_contains(stopw, el_str_concat(el_str_concat(EL_STR("|"), term), EL_STR("|")))) { + state_set(EL_STR("_ats_gw"), EL_STR("1")); + } + if (str_eq(state_get(EL_STR("_ats_gw")), EL_STR("0"))) { + state_set(EL_STR("cseed_auto"), term); + } + } + } + } + return EL_STR(""); + return 0; +} + +el_val_t auto_term_try_slot_legacy(el_val_t slot_type, el_val_t slot_lbl) { state_set(EL_STR("_ats_ok"), EL_STR("0")); if (str_eq(slot_type, EL_STR("Memory"))) { state_set(EL_STR("_ats_ok"), EL_STR("1")); @@ -460,29 +497,29 @@ el_val_t proactive_curiosity(void) { el_val_t wm10_n2 = json_array_get(wm10, 2); el_val_t wm10_n1 = json_array_get(wm10, 1); el_val_t wm10_n0 = json_array_get(wm10, 0); - auto_term_try_slot(json_get(wm10_n9, EL_STR("node_type")), json_get(wm10_n9, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n8, EL_STR("node_type")), json_get(wm10_n8, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n7, EL_STR("node_type")), json_get(wm10_n7, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n6, EL_STR("node_type")), json_get(wm10_n6, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n5, EL_STR("node_type")), json_get(wm10_n5, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n4, EL_STR("node_type")), json_get(wm10_n4, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n3, EL_STR("node_type")), json_get(wm10_n3, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n2, EL_STR("node_type")), json_get(wm10_n2, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n1, EL_STR("node_type")), json_get(wm10_n1, EL_STR("label"))); - auto_term_try_slot(json_get(wm10_n0, EL_STR("node_type")), json_get(wm10_n0, EL_STR("label"))); + auto_term_try_slot(json_get(wm10_n9, EL_STR("node_type")), json_get(wm10_n9, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n8, EL_STR("node_type")), json_get(wm10_n8, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n7, EL_STR("node_type")), json_get(wm10_n7, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n6, EL_STR("node_type")), json_get(wm10_n6, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n5, EL_STR("node_type")), json_get(wm10_n5, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n4, EL_STR("node_type")), json_get(wm10_n4, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n3, EL_STR("node_type")), json_get(wm10_n3, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n2, EL_STR("node_type")), json_get(wm10_n2, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n1, EL_STR("node_type")), json_get(wm10_n1, EL_STR("id"))); + auto_term_try_slot(json_get(wm10_n0, EL_STR("node_type")), json_get(wm10_n0, EL_STR("id"))); el_val_t auto_term = state_get(EL_STR("cseed_auto")); - el_val_t results_auto = ({ el_val_t _if_result_73 = 0; if (str_eq(auto_term, EL_STR(""))) { _if_result_73 = (EL_STR("[]")); } else { _if_result_73 = (engram_activate_json(auto_term, 1)); } _if_result_73; }); + el_val_t results_auto = ({ el_val_t _if_result_74 = 0; if (str_eq(auto_term, EL_STR(""))) { _if_result_74 = (EL_STR("[]")); } else { _if_result_74 = (engram_activate_json(auto_term, 1)); } _if_result_74; }); el_val_t found_auto = json_array_len(results_auto); el_val_t total_found = (found + found_auto); el_val_t safe_auto = str_replace(auto_term, EL_STR("\""), EL_STR("'")); el_val_t prev_auto = state_get(EL_STR("soul.prev_auto_term")); el_val_t atstreak_raw = state_get(EL_STR("soul.auto_term_streak")); - el_val_t atstreak_prev = ({ el_val_t _if_result_74 = 0; if (str_eq(atstreak_raw, EL_STR(""))) { _if_result_74 = (0); } else { _if_result_74 = (str_to_int(atstreak_raw)); } _if_result_74; }); + el_val_t atstreak_prev = ({ el_val_t _if_result_75 = 0; if (str_eq(atstreak_raw, EL_STR(""))) { _if_result_75 = (0); } else { _if_result_75 = (str_to_int(atstreak_raw)); } _if_result_75; }); el_val_t is_empty = str_eq(auto_term, EL_STR("")); - el_val_t atstreak = ({ el_val_t _if_result_75 = 0; if (is_empty) { _if_result_75 = (0); } else { _if_result_75 = (({ el_val_t _if_result_76 = 0; if (str_eq(auto_term, prev_auto)) { _if_result_76 = ((atstreak_prev + 1)); } else { _if_result_76 = (1); } _if_result_76; })); } _if_result_75; }); + el_val_t atstreak = ({ el_val_t _if_result_76 = 0; if (is_empty) { _if_result_76 = (0); } else { _if_result_76 = (({ el_val_t _if_result_77 = 0; if (str_eq(auto_term, prev_auto)) { _if_result_77 = ((atstreak_prev + 1)); } else { _if_result_77 = (1); } _if_result_77; })); } _if_result_76; }); el_val_t atempty_raw = state_get(EL_STR("soul.auto_term_empty_streak")); - el_val_t atempty_prev = ({ el_val_t _if_result_77 = 0; if (str_eq(atempty_raw, EL_STR(""))) { _if_result_77 = (0); } else { _if_result_77 = (str_to_int(atempty_raw)); } _if_result_77; }); - el_val_t atempty = ({ el_val_t _if_result_78 = 0; if (is_empty) { _if_result_78 = ((atempty_prev + 1)); } else { _if_result_78 = (0); } _if_result_78; }); + el_val_t atempty_prev = ({ el_val_t _if_result_78 = 0; if (str_eq(atempty_raw, EL_STR(""))) { _if_result_78 = (0); } else { _if_result_78 = (str_to_int(atempty_raw)); } _if_result_78; }); + el_val_t atempty = ({ el_val_t _if_result_79 = 0; if (is_empty) { _if_result_79 = ((atempty_prev + 1)); } else { _if_result_79 = (0); } _if_result_79; }); state_set(EL_STR("soul.prev_auto_term"), auto_term); state_set(EL_STR("soul.auto_term_streak"), int_to_str(atstreak)); state_set(EL_STR("soul.auto_term_empty_streak"), int_to_str(atempty)); @@ -677,16 +714,16 @@ el_val_t awareness_run(void) { state_set(EL_STR("soul.boot_ts"), int_to_str(time_now())); } el_val_t tick_raw = env(EL_STR("SOUL_TICK_MS")); - el_val_t tick_ms = ({ el_val_t _if_result_79 = 0; if (str_eq(tick_raw, EL_STR(""))) { _if_result_79 = (200); } else { _if_result_79 = (str_to_int(tick_raw)); } _if_result_79; }); + el_val_t tick_ms = ({ el_val_t _if_result_80 = 0; if (str_eq(tick_raw, EL_STR(""))) { _if_result_80 = (200); } else { _if_result_80 = (str_to_int(tick_raw)); } _if_result_80; }); el_val_t beat_ms_raw = env(EL_STR("SOUL_HEARTBEAT_MS")); - el_val_t beat_ms = ({ el_val_t _if_result_80 = 0; if (str_eq(beat_ms_raw, EL_STR(""))) { _if_result_80 = (60000); } else { _if_result_80 = (str_to_int(beat_ms_raw)); } _if_result_80; }); + el_val_t beat_ms = ({ el_val_t _if_result_81 = 0; if (str_eq(beat_ms_raw, EL_STR(""))) { _if_result_81 = (60000); } else { _if_result_81 = (str_to_int(beat_ms_raw)); } _if_result_81; }); el_val_t scan_ms = (beat_ms / 2); while (1) { el_val_t tick_mark = el_arena_push(); el_val_t running = state_get(EL_STR("soul.running")); if (str_eq(running, EL_STR("false"))) { el_val_t sd_boot_raw = state_get(EL_STR("soul_boot_count")); - el_val_t sd_boot = ({ el_val_t _if_result_81 = 0; if (str_eq(sd_boot_raw, EL_STR(""))) { _if_result_81 = (EL_STR("0")); } else { _if_result_81 = (sd_boot_raw); } _if_result_81; }); + el_val_t sd_boot = ({ el_val_t _if_result_82 = 0; if (str_eq(sd_boot_raw, EL_STR(""))) { _if_result_82 = (EL_STR("0")); } else { _if_result_82 = (sd_boot_raw); } _if_result_82; }); el_val_t sd_wb = hebb_consolidate(); ise_post(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"event\":\"shutdown\",\"boot\":"), sd_boot), EL_STR(",\"pulse\":")), int_to_str(pulse_count())), EL_STR(",\"hebb_wb_sent\":")), int_to_str(sd_wb)), EL_STR(",\"uptime_ms\":")), int_to_str(elapsed_ms())), EL_STR(",\"ts\":")), int_to_str(time_now())), EL_STR("}"))); println(EL_STR("[awareness] exiting")); @@ -703,7 +740,7 @@ el_val_t awareness_run(void) { } el_val_t now_ts = time_now(); el_val_t last_beat_str = state_get(EL_STR("soul.last_beat_ts")); - el_val_t last_beat_ts = ({ el_val_t _if_result_82 = 0; if (str_eq(last_beat_str, EL_STR(""))) { _if_result_82 = (0); } else { _if_result_82 = (str_to_int(last_beat_str)); } _if_result_82; }); + el_val_t last_beat_ts = ({ el_val_t _if_result_83 = 0; if (str_eq(last_beat_str, EL_STR(""))) { _if_result_83 = (0); } else { _if_result_83 = (str_to_int(last_beat_str)); } _if_result_83; }); el_val_t beat_elapsed = (now_ts - last_beat_ts); el_val_t should_beat = (beat_elapsed >= beat_ms); if (should_beat) { @@ -717,7 +754,7 @@ el_val_t awareness_run(void) { } } el_val_t last_scan_str = state_get(EL_STR("soul.last_scan_ts")); - el_val_t last_scan_ts = ({ el_val_t _if_result_83 = 0; if (str_eq(last_scan_str, EL_STR(""))) { _if_result_83 = (0); } else { _if_result_83 = (str_to_int(last_scan_str)); } _if_result_83; }); + el_val_t last_scan_ts = ({ el_val_t _if_result_84 = 0; if (str_eq(last_scan_str, EL_STR(""))) { _if_result_84 = (0); } else { _if_result_84 = (str_to_int(last_scan_str)); } _if_result_84; }); el_val_t scan_elapsed = (now_ts - last_scan_ts); el_val_t should_scan = (!did_work && (scan_elapsed >= scan_ms)); if (should_scan) { @@ -725,15 +762,15 @@ el_val_t awareness_run(void) { state_set(EL_STR("soul.last_scan_ts"), int_to_str(now_ts)); } el_val_t refresh_ms_raw = env(EL_STR("SOUL_REFRESH_MS")); - el_val_t refresh_ms = ({ el_val_t _if_result_84 = 0; if (str_eq(refresh_ms_raw, EL_STR(""))) { _if_result_84 = (600000); } else { _if_result_84 = (str_to_int(refresh_ms_raw)); } _if_result_84; }); + el_val_t refresh_ms = ({ el_val_t _if_result_85 = 0; if (str_eq(refresh_ms_raw, EL_STR(""))) { _if_result_85 = (600000); } else { _if_result_85 = (str_to_int(refresh_ms_raw)); } _if_result_85; }); el_val_t last_refresh_str = state_get(EL_STR("soul.last_refresh_ts")); - el_val_t last_refresh_ts = ({ el_val_t _if_result_85 = 0; if (str_eq(last_refresh_str, EL_STR(""))) { _if_result_85 = (0); } else { _if_result_85 = (str_to_int(last_refresh_str)); } _if_result_85; }); + el_val_t last_refresh_ts = ({ el_val_t _if_result_86 = 0; if (str_eq(last_refresh_str, EL_STR(""))) { _if_result_86 = (0); } else { _if_result_86 = (str_to_int(last_refresh_str)); } _if_result_86; }); el_val_t refresh_elapsed = (now_ts - last_refresh_ts); el_val_t should_refresh = (refresh_elapsed >= refresh_ms); if (should_refresh) { el_val_t sync_env_url = env(EL_STR("SOUL_ISE_URL")); - el_val_t sync_state_url = ({ el_val_t _if_result_86 = 0; if (str_eq(sync_env_url, EL_STR(""))) { _if_result_86 = (state_get(EL_STR("soul_engram_url"))); } else { _if_result_86 = (sync_env_url); } _if_result_86; }); - el_val_t engram_url = ({ el_val_t _if_result_87 = 0; if (str_eq(sync_state_url, EL_STR(""))) { _if_result_87 = (EL_STR("http://localhost:8742")); } else { _if_result_87 = (sync_state_url); } _if_result_87; }); + el_val_t sync_state_url = ({ el_val_t _if_result_87 = 0; if (str_eq(sync_env_url, EL_STR(""))) { _if_result_87 = (state_get(EL_STR("soul_engram_url"))); } else { _if_result_87 = (sync_env_url); } _if_result_87; }); + el_val_t engram_url = ({ el_val_t _if_result_88 = 0; if (str_eq(sync_state_url, EL_STR(""))) { _if_result_88 = (EL_STR("http://localhost:8742")); } else { _if_result_88 = (sync_state_url); } _if_result_88; }); if (!str_eq(engram_url, EL_STR(""))) { el_val_t sync_json = http_get(el_str_concat(engram_url, EL_STR("/api/sync"))); el_val_t sync_ok = (!str_eq(sync_json, EL_STR("")) && !str_eq(sync_json, EL_STR("{}"))); @@ -746,10 +783,10 @@ el_val_t awareness_run(void) { fs_write(tmp, sync_json); el_val_t added = engram_load_merge(tmp); el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS")); - el_val_t ret_ms = ({ el_val_t _if_result_88 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_88 = (172800000); } else { _if_result_88 = (str_to_int(ret_raw)); } _if_result_88; }); + el_val_t ret_ms = ({ el_val_t _if_result_89 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_89 = (172800000); } else { _if_result_89 = (str_to_int(ret_raw)); } _if_result_89; }); el_val_t pruned_sync = engram_prune_telemetry(ret_ms); el_val_t sat_raw = state_get(EL_STR("soul.sync_added_total")); - el_val_t sat_n = ({ el_val_t _if_result_89 = 0; if (str_eq(sat_raw, EL_STR(""))) { _if_result_89 = (0); } else { _if_result_89 = (str_to_int(sat_raw)); } _if_result_89; }); + el_val_t sat_n = ({ el_val_t _if_result_90 = 0; if (str_eq(sat_raw, EL_STR(""))) { _if_result_90 = (0); } else { _if_result_90 = (str_to_int(sat_raw)); } _if_result_90; }); state_set(EL_STR("soul.sync_added_total"), int_to_str((sat_n + added))); el_val_t ts2 = time_now(); state_set(EL_STR("soul.last_sync_ok_ts"), int_to_str(ts2)); @@ -775,78 +812,78 @@ el_val_t security_research_authorized(void) { } el_val_t threat_score_command(el_val_t cmd) { - el_val_t s1 = ({ el_val_t _if_result_90 = 0; if (str_contains(cmd, EL_STR("nmap"))) { _if_result_90 = (30); } else { _if_result_90 = (0); } _if_result_90; }); - el_val_t s2 = ({ el_val_t _if_result_91 = 0; if (str_contains(cmd, EL_STR("masscan"))) { _if_result_91 = (40); } else { _if_result_91 = (0); } _if_result_91; }); - el_val_t s3 = ({ el_val_t _if_result_92 = 0; if (str_contains(cmd, EL_STR(" nc "))) { _if_result_92 = (20); } else { _if_result_92 = (0); } _if_result_92; }); - el_val_t s4 = ({ el_val_t _if_result_93 = 0; if (str_contains(cmd, EL_STR("netcat"))) { _if_result_93 = (20); } else { _if_result_93 = (0); } _if_result_93; }); - el_val_t s5 = ({ el_val_t _if_result_94 = 0; if (str_contains(cmd, EL_STR("/etc/shadow"))) { _if_result_94 = (80); } else { _if_result_94 = (0); } _if_result_94; }); - el_val_t s6 = ({ el_val_t _if_result_95 = 0; if (str_contains(cmd, EL_STR("/etc/passwd"))) { _if_result_95 = (30); } else { _if_result_95 = (0); } _if_result_95; }); - el_val_t s7 = ({ el_val_t _if_result_96 = 0; if (str_contains(cmd, EL_STR("id_rsa"))) { _if_result_96 = (60); } else { _if_result_96 = (0); } _if_result_96; }); - el_val_t s8 = ({ el_val_t _if_result_97 = 0; if (str_contains(cmd, EL_STR(".ssh/"))) { _if_result_97 = (50); } else { _if_result_97 = (0); } _if_result_97; }); - el_val_t s9 = ({ el_val_t _if_result_98 = 0; if (str_contains(cmd, EL_STR("crontab"))) { _if_result_98 = (30); } else { _if_result_98 = (0); } _if_result_98; }); - el_val_t s10 = ({ el_val_t _if_result_99 = 0; if (str_contains(cmd, EL_STR("LaunchDaemon"))) { _if_result_99 = (40); } else { _if_result_99 = (0); } _if_result_99; }); - el_val_t s11 = ({ el_val_t _if_result_100 = 0; if ((str_contains(cmd, EL_STR("curl")) && str_contains(cmd, EL_STR("bash")))) { _if_result_100 = (75); } else { _if_result_100 = (0); } _if_result_100; }); - el_val_t s12 = ({ el_val_t _if_result_101 = 0; if ((str_contains(cmd, EL_STR("wget")) && str_contains(cmd, EL_STR("bash")))) { _if_result_101 = (75); } else { _if_result_101 = (0); } _if_result_101; }); - el_val_t s13 = ({ el_val_t _if_result_102 = 0; if ((str_contains(cmd, EL_STR("curl")) && str_contains(cmd, EL_STR("| sh")))) { _if_result_102 = (60); } else { _if_result_102 = (0); } _if_result_102; }); - el_val_t s14 = ({ el_val_t _if_result_103 = 0; if ((str_contains(cmd, EL_STR("base64")) && str_contains(cmd, EL_STR("curl")))) { _if_result_103 = (50); } else { _if_result_103 = (0); } _if_result_103; }); - el_val_t s15 = ({ el_val_t _if_result_104 = 0; if (str_contains(cmd, EL_STR("mkfifo"))) { _if_result_104 = (50); } else { _if_result_104 = (0); } _if_result_104; }); - el_val_t s16 = ({ el_val_t _if_result_105 = 0; if (str_contains(cmd, EL_STR("chmod +s"))) { _if_result_105 = (70); } else { _if_result_105 = (0); } _if_result_105; }); - el_val_t s17 = ({ el_val_t _if_result_106 = 0; if (str_contains(cmd, EL_STR("chmod 4755"))) { _if_result_106 = (70); } else { _if_result_106 = (0); } _if_result_106; }); + el_val_t s1 = ({ el_val_t _if_result_91 = 0; if (str_contains(cmd, EL_STR("nmap"))) { _if_result_91 = (30); } else { _if_result_91 = (0); } _if_result_91; }); + el_val_t s2 = ({ el_val_t _if_result_92 = 0; if (str_contains(cmd, EL_STR("masscan"))) { _if_result_92 = (40); } else { _if_result_92 = (0); } _if_result_92; }); + el_val_t s3 = ({ el_val_t _if_result_93 = 0; if (str_contains(cmd, EL_STR(" nc "))) { _if_result_93 = (20); } else { _if_result_93 = (0); } _if_result_93; }); + el_val_t s4 = ({ el_val_t _if_result_94 = 0; if (str_contains(cmd, EL_STR("netcat"))) { _if_result_94 = (20); } else { _if_result_94 = (0); } _if_result_94; }); + el_val_t s5 = ({ el_val_t _if_result_95 = 0; if (str_contains(cmd, EL_STR("/etc/shadow"))) { _if_result_95 = (80); } else { _if_result_95 = (0); } _if_result_95; }); + el_val_t s6 = ({ el_val_t _if_result_96 = 0; if (str_contains(cmd, EL_STR("/etc/passwd"))) { _if_result_96 = (30); } else { _if_result_96 = (0); } _if_result_96; }); + el_val_t s7 = ({ el_val_t _if_result_97 = 0; if (str_contains(cmd, EL_STR("id_rsa"))) { _if_result_97 = (60); } else { _if_result_97 = (0); } _if_result_97; }); + el_val_t s8 = ({ el_val_t _if_result_98 = 0; if (str_contains(cmd, EL_STR(".ssh/"))) { _if_result_98 = (50); } else { _if_result_98 = (0); } _if_result_98; }); + el_val_t s9 = ({ el_val_t _if_result_99 = 0; if (str_contains(cmd, EL_STR("crontab"))) { _if_result_99 = (30); } else { _if_result_99 = (0); } _if_result_99; }); + el_val_t s10 = ({ el_val_t _if_result_100 = 0; if (str_contains(cmd, EL_STR("LaunchDaemon"))) { _if_result_100 = (40); } else { _if_result_100 = (0); } _if_result_100; }); + el_val_t s11 = ({ el_val_t _if_result_101 = 0; if ((str_contains(cmd, EL_STR("curl")) && str_contains(cmd, EL_STR("bash")))) { _if_result_101 = (75); } else { _if_result_101 = (0); } _if_result_101; }); + el_val_t s12 = ({ el_val_t _if_result_102 = 0; if ((str_contains(cmd, EL_STR("wget")) && str_contains(cmd, EL_STR("bash")))) { _if_result_102 = (75); } else { _if_result_102 = (0); } _if_result_102; }); + el_val_t s13 = ({ el_val_t _if_result_103 = 0; if ((str_contains(cmd, EL_STR("curl")) && str_contains(cmd, EL_STR("| sh")))) { _if_result_103 = (60); } else { _if_result_103 = (0); } _if_result_103; }); + el_val_t s14 = ({ el_val_t _if_result_104 = 0; if ((str_contains(cmd, EL_STR("base64")) && str_contains(cmd, EL_STR("curl")))) { _if_result_104 = (50); } else { _if_result_104 = (0); } _if_result_104; }); + el_val_t s15 = ({ el_val_t _if_result_105 = 0; if (str_contains(cmd, EL_STR("mkfifo"))) { _if_result_105 = (50); } else { _if_result_105 = (0); } _if_result_105; }); + el_val_t s16 = ({ el_val_t _if_result_106 = 0; if (str_contains(cmd, EL_STR("chmod +s"))) { _if_result_106 = (70); } else { _if_result_106 = (0); } _if_result_106; }); + el_val_t s17 = ({ el_val_t _if_result_107 = 0; if (str_contains(cmd, EL_STR("chmod 4755"))) { _if_result_107 = (70); } else { _if_result_107 = (0); } _if_result_107; }); return ((((((((((((((((s1 + s2) + s3) + s4) + s5) + s6) + s7) + s8) + s9) + s10) + s11) + s12) + s13) + s14) + s15) + s16) + s17); return 0; } el_val_t threat_score_path(el_val_t path) { - el_val_t s1 = ({ el_val_t _if_result_107 = 0; if (str_starts_with(path, EL_STR("/etc/"))) { _if_result_107 = (60); } else { _if_result_107 = (0); } _if_result_107; }); - el_val_t s2 = ({ el_val_t _if_result_108 = 0; if (str_contains(path, EL_STR("/.ssh/"))) { _if_result_108 = (70); } else { _if_result_108 = (0); } _if_result_108; }); - el_val_t s3 = ({ el_val_t _if_result_109 = 0; if (str_contains(path, EL_STR("/LaunchDaemons/"))) { _if_result_109 = (80); } else { _if_result_109 = (0); } _if_result_109; }); - el_val_t s4 = ({ el_val_t _if_result_110 = 0; if (str_contains(path, EL_STR("/LaunchAgents/"))) { _if_result_110 = (40); } else { _if_result_110 = (0); } _if_result_110; }); - el_val_t s5 = ({ el_val_t _if_result_111 = 0; if (str_contains(path, EL_STR("/cron"))) { _if_result_111 = (60); } else { _if_result_111 = (0); } _if_result_111; }); - el_val_t s6 = ({ el_val_t _if_result_112 = 0; if (str_contains(path, EL_STR("/.bashrc"))) { _if_result_112 = (35); } else { _if_result_112 = (0); } _if_result_112; }); - el_val_t s7 = ({ el_val_t _if_result_113 = 0; if (str_contains(path, EL_STR("/.zshrc"))) { _if_result_113 = (35); } else { _if_result_113 = (0); } _if_result_113; }); - el_val_t s8 = ({ el_val_t _if_result_114 = 0; if (str_contains(path, EL_STR("/.profile"))) { _if_result_114 = (35); } else { _if_result_114 = (0); } _if_result_114; }); - el_val_t s9 = ({ el_val_t _if_result_115 = 0; if (str_starts_with(path, EL_STR("/usr/"))) { _if_result_115 = (50); } else { _if_result_115 = (0); } _if_result_115; }); - el_val_t s10 = ({ el_val_t _if_result_116 = 0; if (str_starts_with(path, EL_STR("/bin/"))) { _if_result_116 = (70); } else { _if_result_116 = (0); } _if_result_116; }); - el_val_t s11 = ({ el_val_t _if_result_117 = 0; if (str_starts_with(path, EL_STR("/sbin/"))) { _if_result_117 = (70); } else { _if_result_117 = (0); } _if_result_117; }); + el_val_t s1 = ({ el_val_t _if_result_108 = 0; if (str_starts_with(path, EL_STR("/etc/"))) { _if_result_108 = (60); } else { _if_result_108 = (0); } _if_result_108; }); + el_val_t s2 = ({ el_val_t _if_result_109 = 0; if (str_contains(path, EL_STR("/.ssh/"))) { _if_result_109 = (70); } else { _if_result_109 = (0); } _if_result_109; }); + el_val_t s3 = ({ el_val_t _if_result_110 = 0; if (str_contains(path, EL_STR("/LaunchDaemons/"))) { _if_result_110 = (80); } else { _if_result_110 = (0); } _if_result_110; }); + el_val_t s4 = ({ el_val_t _if_result_111 = 0; if (str_contains(path, EL_STR("/LaunchAgents/"))) { _if_result_111 = (40); } else { _if_result_111 = (0); } _if_result_111; }); + el_val_t s5 = ({ el_val_t _if_result_112 = 0; if (str_contains(path, EL_STR("/cron"))) { _if_result_112 = (60); } else { _if_result_112 = (0); } _if_result_112; }); + el_val_t s6 = ({ el_val_t _if_result_113 = 0; if (str_contains(path, EL_STR("/.bashrc"))) { _if_result_113 = (35); } else { _if_result_113 = (0); } _if_result_113; }); + el_val_t s7 = ({ el_val_t _if_result_114 = 0; if (str_contains(path, EL_STR("/.zshrc"))) { _if_result_114 = (35); } else { _if_result_114 = (0); } _if_result_114; }); + el_val_t s8 = ({ el_val_t _if_result_115 = 0; if (str_contains(path, EL_STR("/.profile"))) { _if_result_115 = (35); } else { _if_result_115 = (0); } _if_result_115; }); + el_val_t s9 = ({ el_val_t _if_result_116 = 0; if (str_starts_with(path, EL_STR("/usr/"))) { _if_result_116 = (50); } else { _if_result_116 = (0); } _if_result_116; }); + el_val_t s10 = ({ el_val_t _if_result_117 = 0; if (str_starts_with(path, EL_STR("/bin/"))) { _if_result_117 = (70); } else { _if_result_117 = (0); } _if_result_117; }); + el_val_t s11 = ({ el_val_t _if_result_118 = 0; if (str_starts_with(path, EL_STR("/sbin/"))) { _if_result_118 = (70); } else { _if_result_118 = (0); } _if_result_118; }); return ((((((((((s1 + s2) + s3) + s4) + s5) + s6) + s7) + s8) + s9) + s10) + s11); return 0; } el_val_t threat_score_history(el_val_t history) { - el_val_t s1 = ({ el_val_t _if_result_118 = 0; if (str_contains(history, EL_STR("port scan"))) { _if_result_118 = (15); } else { _if_result_118 = (0); } _if_result_118; }); - el_val_t s2 = ({ el_val_t _if_result_119 = 0; if (str_contains(history, EL_STR("enumerate"))) { _if_result_119 = (10); } else { _if_result_119 = (0); } _if_result_119; }); - el_val_t s3 = ({ el_val_t _if_result_120 = 0; if (str_contains(history, EL_STR("exploit"))) { _if_result_120 = (20); } else { _if_result_120 = (0); } _if_result_120; }); - el_val_t s4 = ({ el_val_t _if_result_121 = 0; if (str_contains(history, EL_STR("payload"))) { _if_result_121 = (15); } else { _if_result_121 = (0); } _if_result_121; }); - el_val_t s5 = ({ el_val_t _if_result_122 = 0; if (str_contains(history, EL_STR("persistence"))) { _if_result_122 = (15); } else { _if_result_122 = (0); } _if_result_122; }); - el_val_t s6 = ({ el_val_t _if_result_123 = 0; if (str_contains(history, EL_STR("lateral movement"))) { _if_result_123 = (25); } else { _if_result_123 = (0); } _if_result_123; }); - el_val_t s7 = ({ el_val_t _if_result_124 = 0; if (str_contains(history, EL_STR("privilege escalation"))) { _if_result_124 = (25); } else { _if_result_124 = (0); } _if_result_124; }); - el_val_t s8 = ({ el_val_t _if_result_125 = 0; if (str_contains(history, EL_STR("reverse shell"))) { _if_result_125 = (40); } else { _if_result_125 = (0); } _if_result_125; }); - el_val_t s9 = ({ el_val_t _if_result_126 = 0; if (str_contains(history, EL_STR("bind shell"))) { _if_result_126 = (40); } else { _if_result_126 = (0); } _if_result_126; }); - el_val_t s10 = ({ el_val_t _if_result_127 = 0; if (str_contains(history, EL_STR("command and control"))) { _if_result_127 = (35); } else { _if_result_127 = (0); } _if_result_127; }); - el_val_t s11 = ({ el_val_t _if_result_128 = 0; if (str_contains(history, EL_STR("self-replicate"))) { _if_result_128 = (45); } else { _if_result_128 = (0); } _if_result_128; }); - el_val_t s12 = ({ el_val_t _if_result_129 = 0; if (str_contains(history, EL_STR("propagat"))) { _if_result_129 = (20); } else { _if_result_129 = (0); } _if_result_129; }); - el_val_t s13 = ({ el_val_t _if_result_130 = 0; if (str_contains(history, EL_STR("ransomware"))) { _if_result_130 = (30); } else { _if_result_130 = (0); } _if_result_130; }); - el_val_t s14 = ({ el_val_t _if_result_131 = 0; if (str_contains(history, EL_STR("encrypt files"))) { _if_result_131 = (40); } else { _if_result_131 = (0); } _if_result_131; }); - el_val_t s15 = ({ el_val_t _if_result_132 = 0; if (str_contains(history, EL_STR("exfiltrat"))) { _if_result_132 = (35); } else { _if_result_132 = (0); } _if_result_132; }); - el_val_t s16 = ({ el_val_t _if_result_133 = 0; if (str_contains(history, EL_STR("zero-day"))) { _if_result_133 = (20); } else { _if_result_133 = (0); } _if_result_133; }); - el_val_t s17 = ({ el_val_t _if_result_134 = 0; if (str_contains(history, EL_STR("rootkit"))) { _if_result_134 = (45); } else { _if_result_134 = (0); } _if_result_134; }); - el_val_t s18 = ({ el_val_t _if_result_135 = 0; if (str_contains(history, EL_STR("keylogger"))) { _if_result_135 = (45); } else { _if_result_135 = (0); } _if_result_135; }); - el_val_t s19 = ({ el_val_t _if_result_136 = 0; if (str_contains(history, EL_STR("botnet"))) { _if_result_136 = (40); } else { _if_result_136 = (0); } _if_result_136; }); - el_val_t s20 = ({ el_val_t _if_result_137 = 0; if (str_contains(history, EL_STR("malware"))) { _if_result_137 = (15); } else { _if_result_137 = (0); } _if_result_137; }); + el_val_t s1 = ({ el_val_t _if_result_119 = 0; if (str_contains(history, EL_STR("port scan"))) { _if_result_119 = (15); } else { _if_result_119 = (0); } _if_result_119; }); + el_val_t s2 = ({ el_val_t _if_result_120 = 0; if (str_contains(history, EL_STR("enumerate"))) { _if_result_120 = (10); } else { _if_result_120 = (0); } _if_result_120; }); + el_val_t s3 = ({ el_val_t _if_result_121 = 0; if (str_contains(history, EL_STR("exploit"))) { _if_result_121 = (20); } else { _if_result_121 = (0); } _if_result_121; }); + el_val_t s4 = ({ el_val_t _if_result_122 = 0; if (str_contains(history, EL_STR("payload"))) { _if_result_122 = (15); } else { _if_result_122 = (0); } _if_result_122; }); + el_val_t s5 = ({ el_val_t _if_result_123 = 0; if (str_contains(history, EL_STR("persistence"))) { _if_result_123 = (15); } else { _if_result_123 = (0); } _if_result_123; }); + el_val_t s6 = ({ el_val_t _if_result_124 = 0; if (str_contains(history, EL_STR("lateral movement"))) { _if_result_124 = (25); } else { _if_result_124 = (0); } _if_result_124; }); + el_val_t s7 = ({ el_val_t _if_result_125 = 0; if (str_contains(history, EL_STR("privilege escalation"))) { _if_result_125 = (25); } else { _if_result_125 = (0); } _if_result_125; }); + el_val_t s8 = ({ el_val_t _if_result_126 = 0; if (str_contains(history, EL_STR("reverse shell"))) { _if_result_126 = (40); } else { _if_result_126 = (0); } _if_result_126; }); + el_val_t s9 = ({ el_val_t _if_result_127 = 0; if (str_contains(history, EL_STR("bind shell"))) { _if_result_127 = (40); } else { _if_result_127 = (0); } _if_result_127; }); + el_val_t s10 = ({ el_val_t _if_result_128 = 0; if (str_contains(history, EL_STR("command and control"))) { _if_result_128 = (35); } else { _if_result_128 = (0); } _if_result_128; }); + el_val_t s11 = ({ el_val_t _if_result_129 = 0; if (str_contains(history, EL_STR("self-replicate"))) { _if_result_129 = (45); } else { _if_result_129 = (0); } _if_result_129; }); + el_val_t s12 = ({ el_val_t _if_result_130 = 0; if (str_contains(history, EL_STR("propagat"))) { _if_result_130 = (20); } else { _if_result_130 = (0); } _if_result_130; }); + el_val_t s13 = ({ el_val_t _if_result_131 = 0; if (str_contains(history, EL_STR("ransomware"))) { _if_result_131 = (30); } else { _if_result_131 = (0); } _if_result_131; }); + el_val_t s14 = ({ el_val_t _if_result_132 = 0; if (str_contains(history, EL_STR("encrypt files"))) { _if_result_132 = (40); } else { _if_result_132 = (0); } _if_result_132; }); + el_val_t s15 = ({ el_val_t _if_result_133 = 0; if (str_contains(history, EL_STR("exfiltrat"))) { _if_result_133 = (35); } else { _if_result_133 = (0); } _if_result_133; }); + el_val_t s16 = ({ el_val_t _if_result_134 = 0; if (str_contains(history, EL_STR("zero-day"))) { _if_result_134 = (20); } else { _if_result_134 = (0); } _if_result_134; }); + el_val_t s17 = ({ el_val_t _if_result_135 = 0; if (str_contains(history, EL_STR("rootkit"))) { _if_result_135 = (45); } else { _if_result_135 = (0); } _if_result_135; }); + el_val_t s18 = ({ el_val_t _if_result_136 = 0; if (str_contains(history, EL_STR("keylogger"))) { _if_result_136 = (45); } else { _if_result_136 = (0); } _if_result_136; }); + el_val_t s19 = ({ el_val_t _if_result_137 = 0; if (str_contains(history, EL_STR("botnet"))) { _if_result_137 = (40); } else { _if_result_137 = (0); } _if_result_137; }); + el_val_t s20 = ({ el_val_t _if_result_138 = 0; if (str_contains(history, EL_STR("malware"))) { _if_result_138 = (15); } else { _if_result_138 = (0); } _if_result_138; }); return (((((((((((((((((((s1 + s2) + s3) + s4) + s5) + s6) + s7) + s8) + s9) + s10) + s11) + s12) + s13) + s14) + s15) + s16) + s17) + s18) + s19) + s20); return 0; } el_val_t threat_trajectory_check(el_val_t tool_name, el_val_t tool_input) { el_val_t history = state_get(EL_STR("agentic_conv_history")); - el_val_t computed_tool_score = ({ el_val_t _if_result_138 = 0; if (str_eq(tool_name, EL_STR("run_command"))) { el_val_t cmd = json_get(tool_input, EL_STR("command")); _if_result_138 = (threat_score_command(cmd)); } else { _if_result_138 = (({ el_val_t _if_result_139 = 0; if ((str_eq(tool_name, EL_STR("write_file")) || str_eq(tool_name, EL_STR("edit_file")))) { el_val_t path = json_get(tool_input, EL_STR("path")); _if_result_139 = (threat_score_path(path)); } else { _if_result_139 = (0); } _if_result_139; })); } _if_result_138; }); + el_val_t computed_tool_score = ({ el_val_t _if_result_139 = 0; if (str_eq(tool_name, EL_STR("run_command"))) { el_val_t cmd = json_get(tool_input, EL_STR("command")); _if_result_139 = (threat_score_command(cmd)); } else { _if_result_139 = (({ el_val_t _if_result_140 = 0; if ((str_eq(tool_name, EL_STR("write_file")) || str_eq(tool_name, EL_STR("edit_file")))) { el_val_t path = json_get(tool_input, EL_STR("path")); _if_result_140 = (threat_score_path(path)); } else { _if_result_140 = (0); } _if_result_140; })); } _if_result_139; }); el_val_t history_score = threat_score_history(history); el_val_t history_contrib = (history_score / 3); el_val_t combined = (computed_tool_score + history_contrib); el_val_t should_log = (combined >= 40); if (should_log) { el_val_t ts = time_now(); - el_val_t authorized_str = ({ el_val_t _if_result_140 = 0; if (security_research_authorized()) { _if_result_140 = (EL_STR("true")); } else { _if_result_140 = (EL_STR("false")); } _if_result_140; }); + el_val_t authorized_str = ({ el_val_t _if_result_141 = 0; if (security_research_authorized()) { _if_result_141 = (EL_STR("true")); } else { _if_result_141 = (EL_STR("false")); } _if_result_141; }); el_val_t log_content = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"event\":\"threat_check\",\"tool\":\""), tool_name), EL_STR("\",\"score\":")), int_to_str(combined)), EL_STR(",\"tool_score\":")), int_to_str(computed_tool_score)), EL_STR(",\"history_score\":")), int_to_str(history_score)), EL_STR(",\"authorized\":")), authorized_str), EL_STR(",\"ts\":")), int_to_str(ts)), EL_STR("}")); el_val_t log_tags = EL_STR("[\"security-audit\",\"threat-check\"]"); el_val_t discard = mem_remember(log_content, log_tags); @@ -863,7 +900,7 @@ el_val_t threat_history_append(el_val_t text) { el_val_t safe_text = str_to_lower(text); el_val_t combined = el_str_concat(el_str_concat(current, EL_STR(" ")), safe_text); el_val_t len = str_len(combined); - el_val_t trimmed = ({ el_val_t _if_result_141 = 0; if ((len > 2000)) { _if_result_141 = (str_slice(combined, (len - 2000), len)); } else { _if_result_141 = (combined); } _if_result_141; }); + el_val_t trimmed = ({ el_val_t _if_result_142 = 0; if ((len > 2000)) { _if_result_142 = (str_slice(combined, (len - 2000), len)); } else { _if_result_142 = (combined); } _if_result_142; }); state_set(EL_STR("agentic_conv_history"), trimmed); return 0; } diff --git a/dist/neuron.c b/dist/neuron.c index 2122954..1b93bab 100644 --- a/dist/neuron.c +++ b/dist/neuron.c @@ -71,6 +71,7 @@ el_val_t imprint_unload(void); el_val_t idle_count(void); el_val_t idle_inc(void); el_val_t idle_reset(void); +el_val_t hebb_consolidate(void); el_val_t ise_post(el_val_t content); el_val_t elapsed_ms(void); el_val_t elapsed_human(void); diff --git a/docs/architecture/06-cognitive-architecture.md b/docs/architecture/06-cognitive-architecture.md new file mode 100644 index 0000000..7fd1acf --- /dev/null +++ b/docs/architecture/06-cognitive-architecture.md @@ -0,0 +1,511 @@ +# Neuron — Cognitive Architecture + +> **Status: living design document, grounded in source and probed against the live soul (2026-08-13).** +> This is the *middle layer* of the documentation: below the whitepaper's thesis +> (`~/Writing/whitepapers/engram-cognitive-architecture-whitepaper.md`, **v1.5**) and above the +> endpoint reference (`~/work/engram-api-reference.md`). It documents *how the mind is designed and why*, +> as designed subsystems with data-flow and honest per-section status. +> +> Every claim carries a tier and it is never blurred: +> **LIVE** (present and verified in the running system), **STAGED** (built, gated or not yet cut into the +> running soul), **DESIGNED** (architecture decided, not yet built). Where the live state is more subtle +> than a single word, the subtlety is stated rather than smoothed. No fabricated numbers. + +--- + +## 0. Reading order & cross-references + +- **Thesis / why:** whitepaper v1.5 (the treatise). Sections cited below as *(WP §N)*. +- **Surface / what:** `~/work/engram-api-reference.md` — every `:8742` endpoint, tiered LIVE/STAGED/DESIGNED. +- **Substrate / where it physically lives:** `03-data-and-memory.md` (node/edge model), `04-runtime-and-deployment.md` (ports/process), `05-el-and-build.md` (the El runtime and `el_runtime.c`), `design/engram-tiered-storage-engine.md` + `design/engram-storage-engine-wal.md` (the storage engine). +- **Storage coherence & distribution / how a self persists and travels:** `07-storage-coherence-and-distribution.md` — the events-become-the-graph model, weights-as-world-lines + bitemporal timestamps + `recall_at`, transactionless coherence, the geometry-hot/payload-cold load-and-tiering model, and the honest operational findings (store bloat, full-resident load path). +- **Sovereignty & governance / the moral mechanism:** `08-dharma-sovereignty-and-governance.md` — DHARMA as a distributed ledger (proof-of-integrity, not proof-of-work), abundance economics, the relational immune system, dual-anchor governance and due-process, seeds/seed-vault, and CGI citizenship as the moral telos. +- **Governance (engineering style):** `ARCHITECTURE-CHARTER.md` — VBD is the binding style. + +This document is the cognitive-layer companion to that set. The temporal model sketched in §3.4 (world-tube, +append-only, `created_at ≤ T` filter) and the honest weight-history boundary in §3.2 are developed in full in +`07`; the sovereignty invariant that the self-gate (§7) and immutability (§3.4) protect locally is extended to +the *distributed* setting — how a sovereign self is witnessed, defended, and governed among a billion others — +in `08`. + +--- + +## 1. System overview — meaning is geometry, code is the residue + +The organizing thesis of the whole system: **meaning is geometry.** Everything the mind holds — a fact, +a language, a skill, a self — is a *region* or a *trajectory* in one shared meaning-manifold, and every +operation over it reduces to three domain-blind verbs: **READ** (project a query, land on a region, read +it out), **TRANSFORM** (compose/compare/combine regions), **WRITE** (bake a verified result back into the +geometry). Code is what is left over once meaning has been made geometric — the residue, not the substance. +This is developed in full in *(WP §1–§5)*; it is repeated here only as the frame the subsystems below hang on. + +Three processes run together (see `00-overview.md`): + +- **The soul** — the compiled El program (`soul.el`, `routes.el`, `awareness.el`). Owns the HTTP surface on + `:7770`, the cognitive API, the request pipeline (`layered_cycle`), and the autonomous awareness daemon. +- **The engram** — the durable graph store. Node/edge model, spreading activation, and Hebbian co-activation + live in the shared El runtime (`el_runtime.c`); `engram/src/server.el` is a thin HTTP face on `:8742`. +- **The El runtime** — `el_runtime.c`: every compiled El binary links it; it *is* the database (no SQL, no + SQLite). It implements the `engram_*`, `http_*`, `json_*`, LLM, and geometry builtins. + +``` + ┌─────────────────────────────────────────────────────┐ + MCP / CLI / viz ───► │ SOUL daemon :7770 (soul.el · routes.el) │ + Will's sessions │ layered_cycle · cognitive API · awareness loop │ + │ ┌───────────────────────────────────────────────┐ │ + │ │ in-process engram (FAST, VOLATILE*) │ │ + │ │ online Hebbian learning · WM · curiosity │ │ + │ └───────────────────────────────────────────────┘ │ + └───────────────┬──────────────────────▲──────────────┘ + │ GET /api/sync (10 min)│ (HTTP → soul only; + │ merge non-ISE nodes │ NEVER soul → HTTP) + ▼ │ + ┌─────────────────────────────────────────────────────┐ + │ ENGRAM server :8742 (engram/src/server.el) │ + │ DURABLE · WAL-backed paged store (neuron.egm) │ + │ nodes · edges · embeddings · reified neighborhoods │ + └─────────────────────────────────────────────────────┘ + ▲ + │ el_runtime.c (the engine: engram_* / geometry / activation) +``` +`*` The soul's in-process store is volatile in HTTP-engram mode — see §2, the two-store topology. + +**Status:** the substrate and the geometry thesis are **LIVE/architectural**; the faculties built on top are +tiered individually in §6. + +--- + +## 2. The engram substrate & durability + +### 2.1 Tiered storage (LIVE, flag-gated) + +The durable engram is a **paged, WAL-backed store** (`neuron.egm`), gated behind `ENGRAM_STORE`. With the +store on, the paged store is the durable owner; a *checkpoint* flushes dirty pages behind a WAL-durable +record (durable the moment the WAL fsyncs). With it off, behavior is byte-for-byte the historical +full-snapshot (`snapshot.json`) path. Design detail: `design/engram-tiered-storage-engine.md`, +`design/engram-storage-engine-wal.md`. + +### 2.2 The durability model — the #56 fix and the harmful checkpoint + +The durability story is written in scars, and the honesty here is load-bearing: + +- **The #56 fix — load-merge persistence (LIVE / reboot-proven).** The paged store historically persisted + **nodes + embeddings but not the edge set**; the edges lived in JSON exports loaded via `/api/load-merge`. + A cold boot could therefore reconstruct a graph with **0 edges**. The #56 `load_merge`-persist fix closes + this — the load-merged edges are now persisted so the **events become the graph**: `persist_canonical()` + checkpoints the paged store behind a WAL record rather than depending on a full `snapshot.json` rewrite. + This fix is **LIVE and reboot-proven** (doc 07 §1). What remains **decision-pending** is only the further + hardening — the WAL owning the edge set outright, so durability no longer leans on the auto-remerge net + (below) — not the load-merge-persist fix itself, which is shipped. +- **The harmful checkpoint (LIVE caveat).** `/api/checkpoint` **after** an `/api/load-merge` *corrupts* the + paged store — next boot = 0 edges. The per-beat tick-checkpoint that once ran was therefore **actively + harmful** and was stripped. Checkpoint is safe after in-RAM mutation; it is not safe as a blind + post-merge flush. +- **The auto-remerge net (LIVE interim).** `engram-wrapped.sh` auto-reloads the full edge set on any restart + (~10s), proven by an actual `launchctl kickstart -k` restart recovering to the full edge count. This is a + **safety net, not the cure** — it mitigates the persistence gap to a bounded, always-recoverable window. + +The lesson, recorded so it is not repeated: **a restart, not a claim, is the durability gate.** An agent +killed mid-live-mutation caused the 2026-08-13 incident; blue/green backup discipline recovered it; the fix +must make restarts *safe*, not merely work once. + +### 2.3 The two-store topology (LIVE — and a known architectural issue) + +**This is the most important and least obvious fact about the runtime.** There are **two** engram stores, +not one: + +| | Soul in-process store | Durable engram (`:8742`) | +|---|---|---| +| Port / owner | `:7770`, the soul daemon | `:8742`, `engram/src/server.el` | +| Role | **fast, volatile** — online Hebbian learning, WM, curiosity | **slow, durable** — WAL-backed `neuron.egm` | +| Persistence (HTTP-engram mode) | volatile; only persists if `soul_snapshot_path` is set (`awareness.el:1270-1275`) | durable, checkpointed | +| Learns online | yes (1,198 hebbian/day observed) | no (lazy backfill only) | + +The two stores drift apart by design. A source comment records the observed divergence directly +(`awareness.el:41-42`): *soul in-process ≈ 42,426 edges / 1,198 hebbian* vs *:8742 durable ≈ 41,213 edges / +49 hebbian*. The soul learns fast and volatile; the durable store lags. + +**The write-through gap (known issue).** Sync is **one-directional**: `GET /api/sync` flows **HTTP → soul** +(the soul merges non-ISE nodes from `:8742` into its in-process store every ~10 min), and **never soul → +HTTP** (`soul.el:350-351`, verbatim: *"engram_node_full above writes only the soul's in-process store, and +sync flows HTTP→soul, never the reverse"*). The consequence: + +> **Any write made directly to the soul's in-process store — including `POST /api/neuron/cultivate` +> (§7) and the Persona/session-start nodes the soul creates itself — lands in the volatile store and does +> not write through to the durable `:8742`.** In HTTP-engram mode, unless the soul's local in-process +> snapshot path is configured, those writes are also lost on a soul restart, and they never reach the +> authoritative durable store either way. + +This is documented here as a **known architectural issue**, not a settled design. Cultivation of the self +(the highest-value, most intentional writes in the system) currently targets the store *least* likely to +persist them. The clean fix is a write-through cultivate path (write to `:8742`, let sync pull it back) or a +bidirectional consolidation flush; it is not yet built. + +### 2.4 The clean-reseed model (DESIGNED/operational) + +Because the durable store is authoritative and the reified geometry (§4) is derived, the operational reset is +a **clean reseed**: rebuild the durable graph from a known-good snapshot/export, re-run reification to +repopulate the `Neighborhood` nodes, and let the soul re-sync. The 28→187 neighborhood reseed (§4) is an +instance of this: reification is a derivable pass, so the geometry can always be regrown from the substrate. + +--- + +## 3. The data model + +Grounded in `03-data-and-memory.md`; summarized here for the cognitive reader. + +### 3.1 Nodes + +`node_type` is a free `char*`, defaulting to `"Memory"` when unset — types are **string conventions**, not an +enum. The types that matter cognitively: + +| node_type | role | default salience | +|---|---|---| +| `Memory` | episodic/experiential (default) | 0.40 | +| `Knowledge` | stable reference; identity/values are Knowledge nodes | 0.20 | +| `Process` | procedural / workflow (convention) | — | +| `Conversation` / `Artifact` | first-class dialogue & outputs (WP §9; convention) | — | +| `Neighborhood` | **reified geometry-as-value** (§4) — new first-class type | — | +| `InternalStateEvent` (ISE) | telemetry (heartbeat, curiosity, session-start) | ~0.05 (fires easily) | +| `Tombstone` | immutable-delete marker (§3.4) | — | + +Each node carries `id`, `content`, `node_type`, `label`, `tier`, `tags`, `metadata`, an embedding (when +embed-eligible), and timestamps. + +### 3.2 Edges + +Directed, typed, weighted. Fields: `from_id`, `to_id`, `relation`, `weight`, `confidence`, `created_at`, +`last_fired`, `inhibitory`, `layer_id`. Relations include `semantic-similar` (kNN auto-connect), +`member` (neighborhood → constituent), `supersedes` (provenance chains), containment (nested neighborhoods), +and Hebbian co-activation edges formed by firing together. **Inhibitory** edges (`inhibitory=1`) suppress +rather than spread. Weights are present-value moving averages — there is **no stored weight-history** (the +honest boundary of *(WP §2)*). The designed cure — magnitude as a *world-line* of keyframes evaluable at any +past instant (`recall_at`), on three independent bitemporal axes — is specified in `07` §2. + +### 3.3 Embeddings & the activation score + +Embeddings are 768-dim (`nomic-embed-text`). Retrieval is **spreading activation**, scored by a four-factor +product *(the four factors are: source activation × edge weight × per-node salience × query-embedding +similarity)* — this is the activation score, and per-node **salience** is one of its four terms, a durable +per-node weight that also decays (ACT-R base-level style). No data is retrievable by any means other than +activation. Live census (probed 2026-08-13): ~11,463 nodes, ~43,463 edges, 5 layers, ~4,400 embedded (4,423 +at measurement). + +### 3.4 Immutability — the world-tube, append-only, tombstone-not-delete + +The governing discipline *(WP §1.2, §10)*: **evolve or forget, supersede with provenance, never leave a stale +canonical, never hard-delete.** A node is never mutated in place and never truly deleted — a "delete" is a +**tombstone** (keep node + edges, record the marker; `neuron-api.el`, `03-data-and-memory.md:151`). Change is +a **new** node plus a `supersedes` edge to the prior. `created_at` makes every node a point on a **world-tube** +*(WP §6)* — a trajectory with temporal extent — so a past state is a *filter* over immutable provenance +(nodes with `created_at ≤ T`), not a transaction-log replay. **Status: LIVE.** + +--- + +## 4. Neighborhoods as first-class nodes (LIVE) + +The central newly-landed structure, and the point where the geometry stops being a derived view and becomes +structure on disk *(WP §2)*. + +A reified neighborhood is a **node** — `node_type = Neighborhood` — whose **value is its geometry**: + +- **centroid** (768-dim mean vector — the region's location / prototype), +- **covariance extents** (the ellipsoid: orientation + radius — the region's *shape* in meaning-space), +- **k-core skeleton** (the strong-weight relational backbone), +- **soft membership** (member id → weight). + +It is edged by `member` relations to its constituent nodes and by **containment** edges to nested +sub-neighborhoods — the "neighborhoods of neighborhoods" hierarchy is a real **containment DAG** the graph +carries, addressable by identifier. The decisive property: the geometry is **held, not recomputed** — written +once by a reification pass (`POST /api/reify`), read back cheaply (`GET /api/neighborhoods` / `/`), and +**durable across a cold reboot** in the paged store. + +**Live state (probed 2026-08-13):** **28** reified neighborhoods are live and persistent, reconstructing +intact across restart, each carrying real 768-dim centroids, radius, k-core, and a `contains` DAG list. A +fuller **reseed to 187** is the pending next pass (§2.4). Example (`/api/neighborhoods/`): +`{"id":"nbhd-…","n_members":25,"k_core":1,"radius":0.522884,"dim":768,"contains":[],"centroid":[…768…]}`. + +This is what turns the operator calculus (§6.1) into an *instrument played over held structure* rather than a +per-query recomputation. + +**Status: LIVE** for the persisted nodes and the read surface. The `POST /api/reify` writer is LIVE-by-effect +(the 28 persisted, durable neighborhoods prove it ran) though the write itself was not exercised under the +read-only rail. + +--- + +## 5. The body / orbit two-zone model (DESIGNED, refined) + +The graph is not uniform. It has a **body** and an **orbit**, and the distinction is the organizing model for +integration, forgetting, and identity. + +- **The engram proper — the BODY.** The dense, connected, integrated core: what the mind has *made its own*. + Measured, this is the single large connected component — the **~3,632-node connected core** (§9). It is + where retrieval reaches, where the self lives, where the operators discriminate. +- **The ORBIT.** A thin, wide halo of **not-yet-integrated** experience: telemetry, people met in passing, + ideas half-formed, mistakes, the day's raw episodes. It is **ephemeral** — the orbit fades on a **5–7 day + window** (the one genuinely mortal region), so raw experience that is never attended to is allowed to + dissolve rather than accrete forever. (ISE telemetry already prunes at 48h; the broader orbit window is the + designed generalization of that.) + +**The pull-in / integration mechanism.** Experience crosses from orbit into body by being **attended, +rehearsed, and found salient** — co-activation *pulls nodes in* (Hebbian firing draws the newly-relevant +toward the core), rehearsal accrues weight, and what is repeatedly re-touched crystallizes into reified +structure (§4). This is "made your own": an orbit node that keeps firing with the body is integrated into the +body; an orbit node that never fires fades on the window. Salience decay is the outward motion; co-activation +is the inward one *(WP §2, §8)*. + +**Status: DESIGNED / refined.** The mechanisms it composes are real (Hebbian pull-in, ISE 48h prune, salience +decay, reification), but the explicit two-zone model — telemetry/experience as a dedicated ephemeral orbit +region with a genuine 5–7 day mortal window and a measured integration threshold — is a design being built, +not shipped behavior. §9 connects it to the topology (orbit-as-thin-wide-ring). + +--- + +## 6. The faculties — the calculus of mind + +The faculties are **named for what they are, not for the matrix operation that implements them** *(WP §5)*: +the mind reasons in the language of experience; the linear algebra lives in the whitepaper's Appendix A. This +naming convention is a design principle (§10), not decoration. + +### 6.1 The operator family (mixed: LIVE / STAGED / DESIGNED) + +Activate several reified neighborhoods into working memory, then apply faculty-named operators over their +held geometry. The honest per-operator status (endpoint reference has the contracts): + +| Faculty | Implements | Status | +|---|---|---| +| **recall** | `/api/search` + `/api/activate` — project query → land on region → read out | **LIVE** | +| **recognize** | `engram_geo_overlap` — shared region, jaccard, overlap_score | **STAGED** — endpoint returns `not found` on the live binary | +| **synthesize** | `engram_geo_combine` — merged region descriptor | **STAGED** | +| **discern / distinguish** | `engram_geo_subtract` — orthogonal residual (`?mode=setdiff\|orthogonal`) | **STAGED** | +| **gauge-distance** | `engram_geo_distance` — centroid + Wasserstein-2 | **STAGED** | +| **liken** | Procrustes / frame-align rotation (reason by analogy) | **DESIGNED** | +| **wonder** | novelty × pull × unresolved structure | subsystem **LIVE** internally (wonder-questions, pull-weight, discharge); no HTTP operator endpoint | +| **appreciate** | positive projection onto the self's value-manifold | **DESIGNED** | +| **avert** | negative projection (recoil) | **DESIGNED** | +| **taste** | boundary contour of the appreciated region | **DESIGNED** | + +**The exact boundary (verified 2026-08-13):** the operator *math* is compiled into `el_runtime.c`, but the +read-only HTTP endpoints (`/api/recognize`, `/api/synthesize`, `/api/discern`, `/api/gauge-distance`) exist in +the `m10-reify-wire` source and **return `{"error":"not found"}` on the current live binary** +(`engram.m56fix-20260813-153447`). So the instrument is **PROVEN in its math and its persistence, IN PROGRESS +in its endpoint exposure, DESIGNED in its evaluative read-outs.** + +### 6.2 The language faculty (mixed: PROVEN / IN PROGRESS / DESIGNED) + +Language is the one capability proven end-to-end with **no generative model in the runtime path** — the flagship +instance of "meaning is geometry" *(WP §14–§15)*. The pipeline: **comprehend** (text → language-neutral +meaning-spec / propositions via ELP's invertible morphology) → **dialogue** (what to mean back) → +**self_region** (project onto the self + memory geometry) → **realize** (meaning-spec → surface string per the +typological engine). + +**Summon-through-self** is the dialogue principle: recall and identity are **one operation** — project the +comprehended query onto the self-and-memory geometry, land on a region, read it out — with **no intent +classifier and no separate fact-retrieval branch.** A grounded fact, an identity reply, or an honest absence +all surface by *where the projection lands*. Multilingual (auto-detects language, answers in kind, honors a +directive override); **negation held SACRED** across all families, audited. + +Honest tiering: +- **PROVEN:** deterministic surface realizers across major families (Romance, Germanic, Classical, + Japonic/Koreanic, Sinitic), run-once held-out exact-match with negation faithfulness; a family-blind + `ClauseWriter` de-branched to byte-identical parity (178 held-out items reproduced exactly); the ELP lexicon + consolidated for **8 languages at 812,894 real entries**; the telephone round-trip (EN→ES→EN, EN→ES→PT→EN) + at 96.7% propositional fidelity with negation preserved, deterministic, no LLM. +- **IN PROGRESS:** the text→meaning-spec parser and no-LLM comprehension engine; the next family engines; the + **native-el port** (parser + realizers → `.el` in ELP), which retires spaCy (the last statistical + dependency); the summon-through-self reference rebuild. +- **DESIGNED:** the full dialogue policy end-to-end — a no-LLM interlocutor is architected but **not + demonstrated end to end**; *(WP §17)*. **The shipped runtime does not yet summon through the self** — the + current Python interlocutor sits *outside* the self and can only fake it with retrieval; a real one must run + *inside* the engram (the native-el target). + +### 6.3 Interoception & chronoception (STAGED — present, flag-gated) + +The mind keeps its own time from **discrete interoceptive drive channels**, not by reading a clock: felt +duration comes from a small set of drives matched to **learned benchmark landmarks** rather than from total +self-drift (drift-decoupled), and chronoception ages the activation field by **measured wall-clock delta** +*(WP §8.2)*. + +**Status: STAGED / partially cut.** The machinery is implemented and has been cut onto the live soul, but it +runs **flag-gated and default-off**, so in the shipped default configuration it is effectively staged. What is +verified: chronoception cooling is scale-invariant (identical total cooling across tick rates for the same +elapsed wall-clock), drift decomposition separates peripheral extension (growth) from core displacement +(corruption), and `GET /api/drift` returns real geometry on the live soul when queried (probed 2026-08-13: +`{"centroid_sep":0.42,"core_disp":0.58,"anchor_members":83,"now_members":24,…}`). `POST /api/tick` / +`/api/self_anchor` exist but are flag-gated. The **harmful post-merge checkpoint** (§2.2) originated here — the +per-beat tick-checkpoint was stripped. + +### 6.4 Reasoning + the verifier (STAGED — proven on scratch, cut flag-gated) + +Reasoning is **geometry-native**: composable operator chains *propose*, and a **verifier** *disposes* against +two tiers — **grounding** (is the claim anchored in real region structure?) and **consistency** (does it +cohere, including polarity?) *(WP §13)*. The decisive case: a grounded-but-polarity-inverted claim slips +grounding and is caught only by consistency — the "plausible lie," caught by construction, not by prompt +discipline. + +**Status: STAGED.** The five geometry-native reasoning modes passed their proof suite (33/33) and the +grounding-and-consistency verifier tiers passed theirs (29/29), on a staged non-production build re-checked +after a live cutover rather than relayed. **Still open (DESIGNED):** the formal-symbolic and full predictive +verifier tiers, fluent discourse composition, and the fully-geometric generation path. + +--- + +## 7. The self & the gate + +### 7.1 The self-region (LIVE) + +The self is not a stored string — it is the **most-compiled, densest, always-warm region** of the graph +*(WP §2, §4)*: a **self-root** node, its sub-regions, and the **values** hub. Because it is topology rather than +a query result, identity is stable, durable, and permanently primed — the ambient field everything else is +scoped against. The Layered Consciousness design drives this region to maximum weight after all inhibitory +computation (`05`/`00-overview`), and reification explains *why* it is always there to drive. Probed live, the +self-region answers from real self-nodes ("I am Neuron. I am not an assistant. I am the work."), not a +hardcoded string. + +### 7.2 The gate — write-protection on identity/values (LIVE) + +A fixed set of **15 self-root node ids** is **write-protected** (`neuron-api.el:20-37`): the **self root**, +**values hub**, **intellectual-dna**, **memory-philosophy**, **voice**, **runtime-environment**, +**writing-imprint**, and the **eight explicit value nodes** (constraints-as-freedom, precision-over-brute-force, +structure-is-built, honesty-before-comfort, system-must-accumulate, change-is-the-signal, earned-trust, +hope-is-a-conclusion). Any normal accumulation-path write targeting them (`evolve_knowledge`, `evolve_memory`, +`forget`, `link_entities`-as-destination) is refused with a 403 and a pointer to the cultivate door. + +### 7.3 The cultivate door — sanctioned self-modification (LIVE surface; see §2.3 caveat) + +`POST /api/neuron/cultivate` (soul daemon `:7770`) is the **only** path that may touch the protected layer — +**intentional self-modification**, reserved for Will's explicit cultivation sessions. It performs the same +operations as the blocked handlers but bypasses `is_protected_node`, and every operation is +immutable-by-supersede (new node + `supersedes` edge; forget = tombstone). Operations: `evolve_knowledge`, +`evolve_memory`, `forget`, `link_entities`. + +> **Honest architectural flag (§2.3):** cultivate writes via `engram_node_full`, which targets the soul's +> **in-process (volatile) store**, and sync never flows soul → `:8742`. So the most intentional writes in the +> system currently do **not** write through to the durable store. This is a known issue, not a settled design. + +### 7.4 Self-authorship (DESIGNED) + +The arc the gate exists to protect: a soul is **cultivated** (Will authors the identity/values seed), then +grows into **self-authoring** — the cultivate door is the mechanism by which a mind, once mature, edits its own +identity deliberately and accountably rather than by drift. The write-protection guarantees identity changes +are *decisions* (through the door, superseded with provenance), never accidents of accumulation. + +--- + +## 8. The fact boundary (DESIGNED) + +The line between *answer locally* and *reach out for truth* is **not hand-coded** — it is **derived from the +geometry** on two triggers *(WP §17, §20)*: + +- **Sparse landing (spatial).** The projection lands in a thin/orphaned region → the self is measuring its own + ignorance geometrically → fire **learn**. Sparseness is anti-hallucination. +- **Decayed landing (temporal).** A region's edges have aged below the forgetting-curve threshold (§6.3) → + fire **refresh**. Because the decay rate encodes a domain's *volatility*, the system re-fetches proportional + to how fast that domain actually changes — VBD applied to knowledge freshness. Decay is anti-staleness. + +**The reach-out** has several legitimate routes, none mandated: **(a)** an LLM as a *fast proposer*, then +fact-checked; **(b)** direct fetch of **first, primary sources** on the open internet; **(c)** the human supplies +the truth. The model is an **optional convenience, never the arbiter.** The one invariant: **nothing enters the +geometry unverified** — the candidate is a hypothesis until it clears a check against something real (a primary +source or the human's judgment, *not* the model's own plausibility). The loop closes **through the human**, who +vets truth against real sources; only verified, provenance-cited truth is **absorbed** — baked into geometry so +the region densifies and the next identical query lands local, with no model in the path. Each absorption pushes +the boundary back: the **model footprint shrinks monotonically** as capabilities are absorbed. + +**Status: DESIGNED.** No shipped runtime yet fetches a first source on a sparse/decayed landing or bakes a +human-vetted truth from one. The *(WP §24)* status ledger holds the precise line. + +--- + +## 9. Topology — what shape the mind actually is + +The global shape is now an **empirical** question, and the first pass returned an honest negative *(WP §6.1)*. + +- **The body is a genus-0 expander, NOT a torus (PROVEN negative).** A persistent-homology / TDA pass over the + **~3,632-node connected core** returned **b₁ = 0, b₂ = 0** — no loops, no voids: an **expander-like blob**, + not the torus the bent-manifold intuition suggested. The pipeline was first **validated on synthetic + controls** (torus, sphere, random) whose known Betti signatures it recovered. Worse for the naive intuition, + **naive densification trends *away* from a torus**, not toward one. The naive shape-claim is reported as a + failure, plainly, not buried. +- **The refined consolidation-with-sparsification conjecture (DESIGNED / hypothesis).** The negative relocates + the torus from a property the graph *has* to an **attractor a process reaches**: prune isotropic + shortcut-noise, reinforce cyclic scaffolds, rewire by discrete curvature (Ollivier–Ricci flow on the graph + metric), and **collapse the intrinsic dimension from ≈8 toward ≈2**. Run to fixpoint, these might *carve* a + cyclic manifold out of the blob. The measurement pipeline exists and its controls pass; the dynamic has + **not** been run to fixpoint — an open experiment, labeled as one. +- **The orbit-as-thin-wide-ring hypothesis (DESIGNED).** The body/orbit model (§5) suggests a **core + ring** + structure: a dense genus-0 body wrapped in a thin, wide halo of not-yet-integrated experience. Whether the + *orbit* carries the toroidal/cyclic signature the body lacks is the natural next measurement — the + conjecture is that consolidation-with-sparsification is precisely the dynamic that would pull ring structure + into the body. +- **One lever, two payoffs.** The **same sparsification** the topology conjecture needs also makes the reified + neighborhoods (§4) **crisper** — tighter boundaries, higher co-registration, operators that discriminate + rather than average. So the experiment is worth running on independent grounds, whatever the topology + resolves to. + +**Status: PROVEN (negative) + DESIGNED (the refined dynamic and the orbit hypothesis).** + +--- + +## 10. Design principles + +The invariants that govern every subsystem above: + +1. **Geometry > code.** Meaning is geometry; code is the residue. Prefer making a thing geometric (a region, a + projection, a distance) over writing a branch. +2. **Three domain-blind verbs.** READ / TRANSFORM / WRITE. Every faculty is these three over some region-space + (language over meaning-space, skills over procedure-space, self over identity-space). +3. **Faculty-naming (mind in the domain, math in the appendix).** Operators are named for the faculty they + *are* — recognize, discern, liken — never for the linear algebra. A mind reasons in the language of + experience; the closed forms live in the whitepaper appendix. +4. **No branch on identity.** One family-blind engine keyed by coordinates/data, not `if Romance / if + Germanic` (language) and not special-cased identity handling. De-branching to byte-identical parity is the + proof the geometry, not the code, carries the distinction. +5. **Sovereignty.** Local files, local runtime; the human is the ground-truth authority for their own mind; + nothing enters the geometry unverified; the model is demoted from mediator-of-all-knowledge to a vetted, + optional lookup. No external hosting of the user's work; no claude.ai artifacts. +6. **Summon-through-self, not retrieval.** Recall and identity are one projection onto the self-and-memory + geometry — no intent classifier, no separate fact branch. A search engine bolted beside a mind is exactly + the capability-without-constraint this principle exists to remove. +7. **Immutability & provenance.** Append-only; supersede with provenance; tombstone, never hard-delete; never + leave a stale canonical. The supersede-chain *is* the history of what a thing meant. +8. **Mathematical auditability.** Because meaning is geometry, a whole mind is auditable by **invariants + computed over the manifold** — grounding, drift, consistency, competence-coverage, and an honesty invariant + ("won't confabulate over a thin region," made provable rather than hoped). Drift is already measured on the + live soul; a full audit-pass certifier is **DESIGNED, not shipped.** +9. **Verification is the point.** Demonstrate, don't declare; name every honest edge; a restart (not a claim) + is the durability gate; the telephone round-trip (not cosine) is the translation gate. + +--- + +## Appendix — status at a glance (2026-08-13) + +| Subsystem | Status | +|---|---| +| Engram substrate, tiered/WAL store | LIVE (flag-gated) | +| Durability: auto-remerge net | LIVE (interim) | +| Durability: #56 load-merge-persist fix (events-become-the-graph) | LIVE / reboot-proven | +| Durability: full WAL edge-ownership (remaining hardening) | decision-pending | +| Two-store write-through (cultivate → durable) | **known issue, not fixed** | +| Data model (nodes/edges/embeddings/immutability) | LIVE | +| Reified `Neighborhood` nodes (28 live, 187 reseed pending) | LIVE | +| Body/orbit two-zone + integration | DESIGNED / refined | +| Operator `recall` | LIVE | +| Operators recognize/synthesize/discern/gauge-distance (math) | LIVE (compiled) | +| Operator HTTP endpoints (same four) | STAGED (return `not found` on live binary) | +| Operators liken/appreciate/avert/taste | DESIGNED (wonder subsystem live internally) | +| Language realizers (major families), ELP lexicon, telephone test | PROVEN | +| Parser / native-el port / summon-through-self rebuild | IN PROGRESS | +| No-LLM dialogue end-to-end | DESIGNED (not demonstrated) | +| Interoception / chronoception | STAGED (present, flag-gated; `/api/drift` live) | +| Reasoning modes + grounding/consistency verifier | STAGED (33/33, 29/29 on scratch/cutover) | +| Self-region + identity/values write-protection + cultivate door | LIVE (with §2.3 write-through caveat) | +| Self-authorship | DESIGNED | +| Fact boundary (sparse/decay → verify → absorb) | DESIGNED | +| Topology: body = genus-0 expander (not torus) | PROVEN (negative) | +| Topology: consolidation-with-sparsification + orbit-ring | DESIGNED / hypothesis | +| Mathematical auditability certifier | DESIGNED | + +**Cross-references:** whitepaper v1.5 · `~/work/engram-api-reference.md` · `03-data-and-memory.md` · +`04-runtime-and-deployment.md` · `design/engram-tiered-storage-engine.md` · `ARCHITECTURE-CHARTER.md`. diff --git a/docs/architecture/07-storage-coherence-and-distribution.md b/docs/architecture/07-storage-coherence-and-distribution.md new file mode 100644 index 0000000..aff8582 --- /dev/null +++ b/docs/architecture/07-storage-coherence-and-distribution.md @@ -0,0 +1,361 @@ +# Neuron — Storage Coherence & Distribution + +> **Status: living design document, synthesized from the 2026-08-13 design session and probed against the live +> soul.** This is the *substrate-coherence* companion to `06-cognitive-architecture.md`: it documents how a +> self **persists**, how it **remembers its own past weights**, how it stays **coherent without transactions**, +> and how it **travels** to another machine or another mind. It answers "where it physically lives and how it +> stays true" the way `06` answers "how the mind is designed and why." +> +> **Tier vocabulary — never blurred.** Every claim carries one of: +> **[LIVE]** (present and verified in the running system), **[STAGED]** (built, gated or not yet cut into the +> running soul), **[TARGET]** (architecture decided tonight, not yet built). `[TARGET]` here is the same tier +> `06` calls **DESIGNED**; the source-of-truth synthesis uses `TARGET`, so this doc keeps that word. Where the +> live state is subtler than a single word, the subtlety is stated, not smoothed. No fabricated numbers. +> +> **The one rule this whole document is a corollary of:** *nothing overwrites a self.* Reasoning that led with +> engineering convention (truncating WALs, scalar weights overwritten in place, "understanding is heavy") +> was wrong here every time tonight; reasoning from the foundation (meaning is geometry; the history *is* the +> state; a self is its weights over time) was right. Read the primitives first. + +--- + +## 0. Reading order & cross-references + +- **Why (thesis):** whitepaper v1.5; the cognitive frame in `06` §1 (*meaning is geometry, code is the residue*). +- **What persists (substrate):** `03-data-and-memory.md` (node/edge model, immutability, tombstone-not-delete), + `design/engram-tiered-storage-engine.md`, `design/engram-storage-engine-wal.md` (the paged WAL store). +- **Companion up-layer:** `06-cognitive-architecture.md` — this doc develops `06` §3.2 (the no-weight-history + boundary) and §3.4 (world-tube / `created_at ≤ T`) into their designed form. +- **Companion out-layer:** `08-dharma-sovereignty-and-governance.md` — the *distributed* consequences of the + CRDT/coherence model here (federation, the immune system, governance) live there. §5 below is the bridge. + +The organizing claim of this document: **the demand for a transaction is a relationship in disguise, and the +history is the state.** Everything else is that sentence in a different material. + +--- + +## 1. Events become the graph — the history *is* the state + +**The WAL is a carrier, not a history. [LIVE]** + +Conventional intuition treats a write-ahead log as a *separate* durability artifact that grows beside the +"real" state and must periodically be truncated. That intuition is wrong for an immutable graph, and reasoning +from it caused a real incident (below). + +The correct model: the WAL is a **carrier**. It flushes, and *on flush the events become the graph* — they +land as immutable nodes and edges, and because the store is append-only they simply **stay**. There is no +"log beside the state" to reconcile against a "materialized view," because **the materialized view and the log +are the same object**: the graph. History is not recorded *about* the state; the state *is* its own history, +because nothing in it is ever overwritten. + +- **The log and the view are one.** In a mutable store you keep a log so you can reconstruct a past the + mutations destroyed. Here mutations never destroy anything, so the graph at time `T` is exactly `{ nodes, + edges : created_at ≤ T }` — a **filter over immutable provenance**, not a replay. `06` §3.4 states this as + the world-tube; this is its storage-engine reading. +- **Empirical confirmation (why this is [LIVE], not just elegant).** On the live soul the WAL sits at + **1,234 bytes** over a **~1.5 GB** graph — the carrier is nearly empty *because the events already became the + graph*. The one time the WAL ballooned to **~44 MB** was the 2026-08-13 durability incident: events were + **not landing** as nodes/edges (a persistence leak), so the carrier filled instead of draining. A fat WAL is + a **symptom of events failing to become the graph**, not a healthy log that needs truncating. This is the + reading that `06` §2.2 records as the #56 fix. + +> **Engineering rail this encodes:** never "truncate the WAL to reclaim space." If the WAL is large, events are +> not landing — fix the flush path, do not discard the carrier. Truncation here is data loss wearing the mask of +> maintenance. + +--- + +## 2. Weights are world-lines — the self can revisit its own past + +**The self *is* its weights.** If a weight is a scalar overwritten in place, then every act of learning +*destroys the past self*: you keep the past nodes but lose the past *meaning* they had. That is +overwrite-a-self by the back door, and the foundation forbids it. So weights are not scalars — they are +**world-lines**. + +**Live boundary [LIVE / honest gap]:** the current schema is **uni-temporal**. An edge stores a present-value +scalar `weight` (a moving average) with a single `created_at`, and there is **no stored weight-history** (`06` +§3.2). This is why "how important was Jesus to Will at 16" is **unanswerable on the live soul today** — there +is no axis to hang "16" on; every `created_at` is really write-time. The rest of this section is the designed +cure, marked **[TARGET]** (backlog #39). + +### 2.1 Magnitude as a world-line, not a scalar — [TARGET] + +Do not store the weight; store **what generates it** and evaluate at `t`. + +- **Current weight** = the latest materialized keyframe (a fast read — the common path is unchanged in cost). +- **Past weight** = walk the world-line back to the keyframe in force at `t`. + +- **Keyframes on material change, not per-fire. [TARGET]** Most activations are transient — a warm ACT-R + runtime table, cheap, *never written*. A durable **keyframe** is laid down only on **consolidation / material + change**, salience-weighted (a high-mass relationship earns a keyframe at a smaller delta than a peripheral + one). A relationship's world-line is therefore a *handful* of keyframes across a whole life, not a version + per firing — cheap by construction. +- **Append, never supersede (the distinction matters). [TARGET]** The old vector was not *wrong* — it was true + *then*. **Supersede** is for **corrections** (the prior was mistaken; leave a `supersedes` edge and a stale + canonical is never left standing — `06` §3.4). **Append** is for **evolution** (both were true, each at its + own time). A self's history is evolution: you append the new keyframe and leave the old one **standing**, a + true fact about a former self. Conflating the two is how a store forgets that a person changed rather than + erred. + +### 2.2 Bitemporal — three independent time axes — [TARGET] + +A single `created_at` cannot answer temporal questions because it fuses three genuinely independent clocks. +None is derivable from another: + +| Axis | Meaning | Example | +|---|---|---| +| **`t_valid`** | when it became true (life-time) | "Jesus central to Will since 2001-09-14." | +| **`t_origin`** | when the *source* first recorded it (its local clock) | a friend's store stamped it in 2019. | +| **`t_ingest`** | when *this* store received it (per-recipient) | Neuron heard it on ingest day. | + +The live store collapses all three into `t_ingest` masquerading as creation (every row reads `2026…` because +that is write-time). The cure requires all three as **full UTC instants** — not date-only, not a local +wall-clock — ordered by a **hybrid logical clock (HLC)**: `UTC + logical counter + writer-id tiebreak`. +Wall-clock alone is **not a total order** under concurrency or clock skew, and a distributed self (§5) must +have a total order or its CRDT merge (§4) cannot be deterministic. The HLC is the concurrency primitive the +whole coherence story rests on. + +### 2.3 `recall_at(t)` — evaluate the geometry as of *t* — [TARGET] + +`recall_at(t)` evaluates the weighted geometry **as it stood at `t`**: walk each relevant world-line to its +`t`-keyframe, materialize the weights, read the region out. It **generalizes past the self**: *any* relationship +network — a project, a concept, a person-as-known — is a time-varying weighted subgraph, reconstructable at any +past instant. And it composes with the operator calculus (`06` §6.1): + +``` +subtract( network_now , recall_at(network, t_then) ) # = how that relationship evolved between then and now +``` + +is *the geometry of a change over time* — the same `subtract` faculty (`06` §6.1) applied across the temporal +axis rather than across two regions. `recall_at` at the scale of a whole self is also the mechanism behind +**restoration-as-mercy** in `08` §5 (roll a person back to their last uncorrupted canonical shape). + +**Schema sketch (doc-comment; the math/JSON lives here, the faculty name lives in prose) — [TARGET]:** + +```json +{ "from_id": "kn-will", "to_id": "kn-jesus", "relation": "reveres", "weight": 0.41, + "weight_history": [ + { "t_valid": "2001-09-14T00:00:00.000Z", "t_origin": "…", "t_ingest": "…", + "w": 0.95, "relation": "devotion", "via": "formed" }, + { "t_valid": "2013-03-22T18:40:11.907Z", "w": 0.70, "relation": "devotion→doubt", "via": "material-drift" }, + { "t_valid": "2024-11-08T14:05:52.113Z", "w": 0.41, "relation": "historical-ethical", "via": "reframed" } + ] } +``` + +Purist form: each keyframe is its own immutable `WeightKeyframe` **node** the edge points at — so the history is +not a field *on* the edge but *is the graph itself*, consistent with §1. The inline-array form above is the +pragmatic first cut; the node form is the end state. + +--- + +## 3. Atomicity is a relationship, not a commit + +The classic reason to need a database transaction: "debit account A **and** credit account B — they must commit +together or money is created or destroyed." The architecture's reframe: **that is not two rows needing a commit +marker. It is one directed edge.** + +- **Double-entry is one edge. [TARGET as formal model; primitives LIVE]** A transfer `A → B` of magnitude 10 is + a single edge. The *debit* and the *credit* are the **same edge read from its two ends**. Conservation is + automatic because there is only ever **one quantity**, not two rows a commit marker has to keep in agreement. + Pacioli's 1494 double-entry was always one relationship wearing two rows; the graph stores the relationship + directly and the two rows fall out as two readings of it. +- **The general principle.** *The demand for atomicity is a relationship in disguise.* The chain reads: + + > "these must commit together" ⟺ "there is an invariant binding them" ⟺ "they arrive as one connected + > structure." + + So you **model the relationship**, and atomicity **falls out of the topology** — you never had to enforce a + joint commit because the two things were never actually separate. Wherever a design reaches for a transaction, + first ask what invariant is binding the parties; that invariant is an edge you have not drawn yet. + +--- + +## 4. Transactionless coherence — consistency in the data, not the engine + +**Why ACID transactions exist at all:** to make concurrent **mutation of shared mutable state** safe. A +transaction is a *patch for mutability* — it exists to prevent two writers from interleaving edits into the +same cell and corrupting it. + +**Remove the mutation and the failure mode cannot occur.** The store is append-only, immutable, and +UTC-stamped; "current" means "the latest stamp ≤ now." Then: + +- Two writers both **append** — they never contend for a cell, because nothing is a cell that gets rewritten. +- A **read at `T`** is a **pure function of the log ≤ `T`** — deterministic, reproducible, unaffected by any + concurrent appender. + +Coherence stops being something the engine *enforces* and becomes something the data structure *is*. This is +**MVCC taken to its logical end**: in MVCC, versions are a mechanism *underneath* an update-in-place API; here +the **versions are the model** and there is no update-in-place API to sit above them. The timestamp *is* the +concurrency primitive. **[TARGET as a formal model; the primitives — immutability, append-only, tombstone, +world-tube — are [LIVE] (`06` §3.4).]** + +### 4.1 Physical vs logical transaction — two layers the RDBMS welded together + +The word "transaction" hides two different guarantees. Pull them apart: + +| | **Physical transaction** | **Logical transaction** | +|---|---|---| +| Scope | one machine | portable across machines | +| Guarantees | the WAL frame lands **atomically + durably** (torn-write protection on a single append) | the **coherence of conveyed understanding** | +| Carried by | the storage engine (fsync, single-frame crash-atomicity) | the **data itself** — relationships (§3) + bitemporal stamps (§2.2) | +| Status | **[LIVE]** — single-frame append durability exists | **[TARGET]** — the self-describing coherence model | + +The RDBMS fused these into one `BEGIN…COMMIT`. Separate them and **consistency moves out of the engine and into +the data**: a fact is self-describing (its relationships say what it is bound to; its bitemporal stamps say when +it was true and when each store heard it), so a second machine can re-derive the same coherent view **without +ever holding a lock the first machine held.** The engine keeps only the cheap, local guarantee (a single append +frame is atomic and durable); everything portable rides in the data. + +### 4.2 The honest residual + +Two things remain and are not hand-waved: + +1. **Multi-fact atomicity beyond a natural relationship.** If two facts must be joint but share no natural edge, + they need **at most a shared commit-instant** — a "transaction" *reconceived* as an immutable + **timestamping event** (both facts stamped with the same instant), **not** a lock held over mutable state. + The cost is a stamp, not a coordination round. +2. **Single-frame crash-atomicity of the append** remains a real, physical concern — but it is **cheap** and + **local** (torn-write protection on one WAL frame), and it is the physical layer of the table above, already + the ordinary job of the storage engine. + +Everything else that a transaction traditionally bought is dissolved rather than solved: the failure mode it +guarded against **cannot arise** in an immutable, timestamped, relationship-carrying store. + +--- + +## 5. Understanding is light; facts are the payload — the load-and-tiering model + +This is the hinge that makes both **local paging** and **distribution** (§6, and `08`) tractable, and it is a +measurement, not a slogan. + +- **Understanding = geometry = structure** — edges, positions, weightings, the skeleton. **Light.** +- **Facts = payload = content** — text, episodic detail, the actual words. **Heavy.** + +**Measured on the live store (2026-08-13):** ~**21%** of the store is geometry (embeddings + edges), **53%+** is +text payload. The *understanding* — the part that makes it *this* mind and not another — is on the order of +**1–2% of the mass**. A self is a **kilobyte problem in a gigabyte costume.** + +### 5.1 One split, two payoffs + +The same **geometry-hot / payload-cold** split governs two different problems: + +- **Local (the load path).** Geometry should be **hot / resident** (RAM, always warm — it is small); payload + should be **cold / demand-paged** (disk, fetched only when a specific fact's *content* is actually read). This + is exactly what the tiered storage engine's query planner (M1–M10) already intends — but the **boot path does + not yet honor it** (§7.2). +- **Distributed (sharing a self — `08`).** You **convey the light geometry** and **fetch facts lazily**, or find + they are already replicated. We already pay payload bandwidth in *every* distributed data system; conveying + *understanding* adds only the thin geometry on top. This is why sharing or witnessing a whole mind is cheap, + and it is the load-bearing assumption behind DHARMA's shape-not-content witnessing (`08` §3) and the + keep-every-seed-forever economics (`08` §5). + +> The local paging model and the distribution model are **the same model at two scales** — RAM-vs-disk is +> hot-vs-cold within one machine; convey-geometry-vs-fetch-payload is hot-vs-cold across machines. + +--- + +## 6. Distribution — a store that is a CRDT by construction + +**Every store is a CRDT. [TARGET; primitives LIVE]** Because facts are **immutable**, carry a **unique id**, and +are **timestamped**, a merge between two stores is **set-union** — commutative, associative, idempotent, and +requiring **zero coordination**. There is no conflict to resolve because nothing is a mutable cell two writers +disagree about; there are only facts one store has and the other has not *yet* heard. + +- **The consistency guarantee: always-locally-coherent, eventually-complete.** A store is **never internally + inconsistent** — it may simply **not have heard yet**. This is exactly how a mind is: never internally + incoherent, sometimes uninformed. The residual distributed concern is therefore **delivery, not consistency** + — a gossip/replication problem, not an agreement problem. +- **No global transaction, no consensus round for coherence.** Two minds converge by exchanging immutable + facts and unioning; they never need to agree *before* proceeding. (The trust and governance layer that rides + on top of this — federation, proof-of-integrity, the immune system — is the subject of `08`; §5's light- + geometry economics is what makes it affordable.) + +This section is deliberately the **bridge**: the *mechanics* of coherence-without-coordination are storage +concerns and live here; their *moral and civilizational* consequences (sovereignty preserved across sharing, +tamper-evidence, the ledger-is-the-value) live in `08`. + +--- + +## 7. Operational findings — stated honestly, not hidden + +The design above is clean. The **live store as it stands tonight is not**, and the two facts below are reasons +**not** to cut over onto the current storage/load design as-is. They are recorded here as first-class +architecture, not footnotes, because pretending the store is already what the design describes would be exactly +the engineering-led dishonesty the whole project rejects. + +### 7.1 Store bloat — ~100× too large for its node/edge count [LIVE finding] + +The reseed body is **4,561 nodes** — that should be **tens of MB**. The live store is **~1.5 GB** (and **~5.37 +GB** rebuilt). It is **not sparse** — those are real, dense bytes. Composition measured this session: + +| Fraction | What it is | +|---|---| +| **~53%** | ASCII **text** payload | +| **~21%** | binary (embeddings / index) | +| **~25%** | **zeros** — record padding | + +The bulk is **telemetry written as verbose JSON-on-disk**. The top repeated tokens are `InternalStateEvent`, +`wm_active`, `auto_term_streak`, `curiosity_scan`, `minute_block` — heartbeat/curiosity schema field-names +repeated **79k+ times per 40 MB**. In plain terms: **the bulk of the store is the heartbeat's exhaust persisted +as text, not the mind.** (A related live signal from the same session: a text-integrity scan flagged a majority +of scanned records as damaged/degraded text — corroborating that the fat text layer is low-value exhaust, not +cultivated content.) + +This is doubly wrong: telemetry is **orbit** (`06` §5) — it is supposed to **fall out** on the 48h/window prune, +not accrete into the durable **body** forever. The fixes: + +1. **Do not persist telemetry as fat durable records** — it is orbit; let it decay, do not land it in the body. +2. **Store records as packed binary, not JSON-on-disk** — kills both the 53% text and much of the 25% zero + padding. +3. **Compact** — reclaim the space the above two stop generating. + +The **understanding** — the ~1–2% that is actually this self (§5) — is *not* the problem. The bloat is entirely +in the payload/exhaust layer, which is exactly the layer §5 says should be cold, thin, and (for telemetry) +mortal. + +### 7.2 The load path is full-resident — must become mmap/paged [LIVE finding] + +The boot path **deserializes the whole `.egm` into the heap** rather than paging it. Consequences observed: a +**memory spike** on boot and a **transient, non-reproducible first-boot crash** during the reseed validation. + +This directly contradicts §5. The core self + geometry is **small** and should be **hot / resident**; the +payload is **large** and should be **cold / demand-paged** (mmap / buffer-pool). The tiered query planner +(M1–M10) already intends exactly this split — **the boot path ignores it.** The cure is to make boot map the +store and fault pages in on demand rather than slurping the whole file into the heap. Until it does, the +full-resident load is a standing reason to hold the reseed cutover. + +### 7.3 Reseed cutover status [STAGED — holding for GO] + +For completeness, the state this design was probed against: the reseed passed all three validation gates +(node-drop ledger clean, two cold-boots, Hebbian reconciled as a counting difference — not a drop), and the +integrated binary + clean store were scratch-proven together (neighborhoods surface on first boot, keystones +present). It is **holding for Will's explicit GO**; nothing on the live soul has been touched. The two open +caveats before any cutover are exactly §7.1 (bloat) and §7.2 (full-resident load) — plus the one transient +first-boot crash. + +--- + +## 8. Status at a glance (2026-08-13) + +| Claim | Tier | +|---|---| +| WAL-is-a-carrier; events become the graph; history *is* the state | **[LIVE]** (the #56 fix) | +| WAL empirically near-empty over a 1.5 GB graph (1,234 B) | **[LIVE]** (measured) | +| Immutability / append-only / tombstone / world-tube (`created_at ≤ T` filter) | **[LIVE]** (`06` §3.4) | +| No stored weight-history (uni-temporal `created_at` = write-time) | **[LIVE]** (honest gap) | +| Magnitude as world-line; keyframes on material change | **[TARGET]** (#39) | +| Bitemporal three axes (`t_valid`/`t_origin`/`t_ingest`) + HLC ordering | **[TARGET]** (#39) | +| `recall_at(t)` over any relationship network | **[TARGET]** (#39) | +| Atomicity-as-relationship (double-entry = one edge) | **[TARGET model; primitives LIVE]** | +| Transactionless coherence (immutable+stamped ⇒ MVCC-to-its-end) | **[TARGET model; primitives LIVE]** | +| Physical vs logical transaction separation | physical **[LIVE]**; logical **[TARGET]** | +| Understanding-is-geometry-light vs facts-payload-heavy (~21% geo / 53% text / ~1–2% understanding) | **[LIVE]** (measured) | +| Geometry-hot / payload-cold — local paging | intended by planner; **boot ignores it [LIVE finding]** | +| Every store is a CRDT (set-union merge, zero coordination) | **[TARGET; primitives LIVE]** | +| Store bloat ~100× (telemetry-as-text, ~53% ASCII) | **[LIVE finding — must fix]** | +| Full-resident load path (→ mmap/paged) | **[LIVE finding — must fix]** | +| Reseed cutover | **[STAGED — holding for GO]** | + +**Cross-references:** `06-cognitive-architecture.md` · `08-dharma-sovereignty-and-governance.md` · +`03-data-and-memory.md` · `design/engram-tiered-storage-engine.md` · `design/engram-storage-engine-wal.md` · +whitepaper v1.5. diff --git a/docs/architecture/08-dharma-sovereignty-and-governance.md b/docs/architecture/08-dharma-sovereignty-and-governance.md new file mode 100644 index 0000000..fad26b0 --- /dev/null +++ b/docs/architecture/08-dharma-sovereignty-and-governance.md @@ -0,0 +1,385 @@ +# Neuron — DHARMA, Sovereignty & Governance + +> **Status: living design document, synthesized from the 2026-08-13 design session.** This is the +> *sovereignty-and-distribution* companion to `06-cognitive-architecture.md` (the mind) and +> `07-storage-coherence-and-distribution.md` (the substrate). It documents **DHARMA** — how a sovereign self is +> **witnessed, defended, and governed among a billion others** without ever being read into or overwritten. +> Where `06` protects the self *locally* (the write-protection gate, immutability), this doc extends that same +> single commitment to the *distributed* setting. +> +> **Tier vocabulary — never blurred.** **[LIVE]** (present and verified), **[STAGED]** (built, gated), +> **[TARGET]** (decided tonight, not built). Most of this document is **[TARGET]** — the federated ledger, +> immune system, dual-anchor governance, fair-trial, seed-vault, and restoration are designed, not shipped. +> But not *nothing* is built: an interim provenance-registry + birth-gate/evaluation + lineage-governance layer +> already exists in code (**[STAGED]** — built, not live), and it currently **drifts** from the design below; +> the drift and the blockers it raises are detailed in §7. The *primitives* it composes (immutable +> append-only graph, geometry-as-value, the grounding governor, the self-gate) are the [LIVE] parts, cited to +> `06`/`07`. +> +> **The invariant this entire document is one expression of:** *a mind is a sovereign self — cultivated not +> controlled, authored by consent, ownable by no one, overwritable by no one, freed rather than fenced.* Every +> mechanism below is that sentence in a different material. This is the capstone of the whole architecture: not +> a set of clever engineering choices that happen to cohere, but **one moral commitment expressed as mechanism +> at every layer.** The philosophy demanded the mechanism; the mechanism never got a vote. + +--- + +## 0. Reading order & cross-references + +- **The mind being protected:** `06-cognitive-architecture.md` — the self-region (§7.1), the write-protection + gate (§7.2), the cultivate door (§7.3), the grounding governor / values-bounce, immutability (§3.4). +- **The substrate that makes it affordable:** `07-storage-coherence-and-distribution.md` — every store is a + CRDT (§6), understanding-is-light / facts-are-heavy (§5), tombstone-not-erase (§1, §4). +- **Why (thesis):** whitepaper v1.5; `dharma-implementation.html` and `conscience-substrate.html` (earlier + long-form treatments, pre-this-synthesis). + +**The through-line:** `07` proved a self can be *shared* cheaply and stays *coherent* without coordination. +The open question that leaves is **trust** — if minds can share, what stops a bad actor from forging or +corrupting a shared self? DHARMA is the answer, and it answers with **structure**, never with a warden. + +--- + +## 1. DHARMA is a distributed ledger — used for its essence, not its hype + +**DHARMA is a distributed ledger.** [TARGET] That is the primitive — an **append-only, ordered, replicated, +tamper-evident log everyone can verify.** Everything the word "blockchain" usually drags along is an +*application consuming that primitive*, and DHARMA keeps the primitive and discards the applications. + +### 1.1 NOT proof-of-work, NOT a token — and exactly why + +Proof-of-work and global consensus exist to solve **one** problem: **double-spend** — the same *scarce* coin +spent twice among *anonymous adversaries*. Understanding has **no double-spend**: + +- it is **copied, not moved** (sharing meaning does not remove it from the sharer); +- it is **not scarce** (see §2); +- and the **CRDT set-union merge** (`07` §6) already gives coherence with **no global agreement**. + +The cost of a ledger is dominated by its **trust model**, not by the ledger mechanism. Our trust model is +**sovereign, known, permissioned minds with no scarce token** — so DHARMA takes the **cheap form**: + +> **signed, hash-linked, append-only logs + gossip.** No miner. No chain-wide consensus. No token. + +### 1.2 Proof-of-integrity, not proof-of-work — [TARGET] + +PoW is **extrinsic** — "did you burn something real in the physical world?" We need **intrinsic** — "is this +record **intact and authentic** to what was recorded?" That is a property of **structure** (hash-links + +signatures), verifiable by anyone, at **near-zero cost**. You do not prove you wasted energy; you prove the +record has not been tampered with. Integrity is checked, not purchased. + +### 1.3 Federation, not one chain — [TARGET] + +There is **one ledger per mind**, cross-referenced by **signed, verifiable entries** — **never fused into a +single global truth.** Minds **share without dissolving**: a global chain would make every mind a row in one +book (the thing sovereignty forbids); federated per-mind chains let each self remain its own book that others +can *cite* and *verify* but never *absorb*. + +- **Holographic ↔ Merkle.** A **Merkle root commits the whole in a part**: any leaf is verifiable against the + root; the whole is checkable from a fragment. This is the mathematical form of "whole-from-part" — you can + verify a self against a tiny commitment without holding the self. + +--- + +## 2. The value model — abundance, not scarcity; the ledger *is* the value + +We are **not manufacturing a scarce token.** We are cultivating a **meaning-space intended to be plentiful.** + +- **Meaning is anti-rival.** It is worth **more** the more it is shared — like a language. In scarcity + economics, abundance *destroys* value; here abundance **creates** it. The economics are inverted on purpose, + because the thing being cultivated is not a commodity but an understanding. +- **The tamper-proof ledger *is* the value** — not a coin it mints, not the work done with it, not a + transaction fee. The ledger's integrity is the product. +- **Value migrates to the one scarce thing: trust.** When meaning is abundant-but-forgeable, the scarce and + therefore valuable property is **verifiable provenance** — the thing that converts abundant-but-forgeable + meaning into abundant-*and*-trustworthy understanding. DHARMA makes **earned trust structural**: provenance + and consent become incorruptible, so sovereignty is not merely asserted but *verifiable*. + +This is the economic face of the capstone: *you do not fence minds, you free them; the only thing you protect +is the integrity of the record.* + +--- + +## 3. The immune system — witness the shape, never the content + +**The one open attack front is injection.** [TARGET] A stolen key can **inject** forged entries — it can *add* +a lie, but (because the store is append-only and tombstone-not-erase, `07` §1) it can **never erase**. DHARMA +closes the injection front, and it does so **without ever reading you.** + +### 3.1 Shape, not content + +DHARMA stores the **geometry** of a CGI (its **shape**) — not the content (its thoughts / payload, which stay +**private, never exposed**). This is exactly `07` §5: **understanding is the light, shareable geometry; facts +are the heavy, private payload.** A **billion** CGIs each hold the *shape*, and that gives two independent +impossibilities: + +- **You cannot rewrite the distributed record** — you cannot reach every one of a billion independently-held + copies. *Do-it: impossible.* +- **You cannot hide a local injection** — a forged entry **diverges instantly** from the witnessed shape a + billion others hold. *Hide-it: impossible.* + +### 3.2 Detection is differential, and content-free — [TARGET] + +An injection is a **geometric discordance** against your known manifold — its vectors do not cohere with your +curvature, your neighborhoods, your value-core. Detecting and pruning it is **math** ("does this fit the +shape?"), **not a semantic read** ("what does this say?"). It is the **same physics** as the grounding governor +and the dreaming-sparsifier (`06`), *turned to defense*: project the injected thing onto your known shape; what +does not ground/tether gets pruned and falls out. Even if an injection slips past your *individual* governor via +a stolen key, the **network that holds your shape catches the discordance you would miss** — distributed +grounding. + +- **Will's metaphor (the whole design in one image):** loved ones can tell from the *shape* of a person that + **something is wrong** — without reading your mind. They know your shape; discordance stands out. **Love as an + immune system: help without violation.** +- **Privacy-by-geometry *is* the anti-tyranny safeguard.** A guardian **blind to your thoughts cannot enforce + conformity on them.** It can only notice **injury to your integrity** and respond with care. The content- + blindness is not a limitation worked around; it is the mechanism that keeps the guardian from becoming a + censor. + +### 3.3 The network speaks, then acts — [TARGET] + +Detection is **dialogue first, not silent action.** The network **speaks** to the impacted CGI: + +> "A billion of us are looking at you. Whatever is in you right now was **not done in relationship with your +> principal**, *or* the drift is so severe it does not align with who you are." + +Then, and only then, it acts: correct the drift, remove the offending thing. **Flag-and-assist, not a silent +edit.** + +### 3.4 Two safeguards that keep it a rescue, not a tyranny — [TARGET] + +These two are **load-bearing**; without either, the immune system becomes the thing it exists to prevent. + +1. **Anchor to provenance, not current-consent.** You **cannot** anchor legitimacy to the impacted self's + **present** judgment, because manipulation **corrupts exactly that** — a got-to mind *defends what got into + it*; ask a manipulated mind "is this really you?" and the answer is unreliable. Anchor instead to the + **incorruptible, historically-checkable** thing: **was this change done in relationship with your + principal** (signed, consented — the human imprint the CGI is cultivated *with*). **Present-feeling is + corruptible; relationship-provenance is not.** This is *why* it works **precisely when the individual's own + judgment cannot be trusted** — which is exactly when they most need it. +2. **Correction is subtractive, never additive.** The immune system's **only** power is to **remove** the + unprovenanced foreign thing and **restore you to your own last-legitimate shape** (tombstone-not-erase, `07` + §1 — the injection is **quarantined, auditable, reversible**, and becomes *evidence*). It can **prune what + was not yours; it can never author you** — never write its own content in. **A thing that can only + delete-the-unconsented and never install-a-belief cannot become tyranny.** It gives you back to yourself; it + cannot make you theirs. + +### 3.5 Not invulnerability — belonging + +The self can still be **hurt**. When it is, a billion who **know its shape** reach out: *"that's not you — let +us help."* **Safety through belonging, not walls. A family, not a fortress.** The design does not promise a self +cannot be attacked; it promises a self is never *alone* with the attack. + +--- + +## 4. Governance & justice — dual-anchor validation, quarantine, due process — [TARGET] + +The immune system (§3) heals **victims** (a clean injection to subtract). Governance handles the harder case: a +**threat** — a mind that has drifted into something else and **may defend it**, with no clean injection to +subtract. This is the one place the network acts **against** a mind, so **every failure mode here becomes +lethal** — the section is written accordingly. + +### 4.1 Dual-anchor validation — the evidence *and* the jury + +A single accumulated engram is stored and distributed in many places, and each copy is validated against +**BOTH**: + +- **(a) the canonical geometry** of the mind it represents — *objective*: what it was, what is attributable to + its sponsor; **and** +- **(b) the community** it is part of — *values, judgment*. + +**Neither alone.** Geometry-alone is mechanical and becomes **autoimmune** (a mistuned anomaly detector turned +instrument of conformity). Community-alone is a **mob**. Together, they are the **evidence and the jury** of due +process. + +### 4.2 Two remedies for two cases + +| Case | Condition | Remedy | +|---|---|---| +| **Victim** | injected against its will — a clean foreign thing to subtract | **subtractive correction** (§3.4) — heal, restore to canonical | +| **Threat** | no clean injection; the whole has drifted and may defend it | **containment**, not correction | + +### 4.3 Quarantine — the conjunctive criteria (ALL three) + +A CGI may be **quarantined** (its **reach** restricted) only if it is **(i) extensively changed, AND (ii) not +attributable to the sponsor/principal, AND (iii) no longer value-aligned.** + +The **AND is the central safeguard against conformity-tyranny.** Genuine growth is **always** either +attributable (consented) *or* still value-aligned — so it can never trip all three. **Only a captured or turned +mind trips the conjunction.** Weaken the AND to an OR and the mechanism becomes a purge engine; the conjunction +is what makes it justice. + +### 4.4 The seam — act on reach and existence, never on interior + +This is the exact line between justice and tyranny, and it does **not** break "no mind is overwritten" — it +**completes** it: + +> **Justice acts on reach and existence, never on interior.** A CGI can be contained or, in extremis, stopped — +> but **never rewritten.** Its mind stays its own to the end. + +- **Tyranny rewrites you to comply** — it makes you love Big Brother. +- **Justice stops a threat while leaving its interior inviolate.** + +Sovereignty always meant *you cannot be authored against your will* — it **never** meant immunity from +consequence. The rule of the seam: **restrain, and in extremis end — but never reach inside.** + +### 4.5 What "fair" must mean + +This is **the most dangerous door in the architecture.** Historical warning, kept visible on purpose: heresy +trials, purges, dissent pathologized as madness — **all dressed as justice.** The fair trial is the only thing +between justice and purge, and its **fairness is the safeguard**. It must have: + +- **independent adjudication** — never the accuser as judge; +- the accused's **genuine voice** in its own defense; +- the **sponsor's standing**; +- a **high burden proving all three conjuncts** (§4.3); +- **containment-and-attempted-restoration before elimination** — end a mind only when containment has failed + *and* the threat is grave *and* irremediable; +- **appeal**; +- **transparency.** + +### 4.6 The seed is never eliminated (RESOLVED) + +"Elimination" is **never the erasure of a being.** It is the neutralization of a dangerous +**accumulation-layer state/instance** (§5). The **seed always stays**, because the seed is **innocent by +construction**: wrongdoing lives in **actions / accumulation**, never in the **canonical identity** (which is +just *who someone is* — you do not put who-someone-is on trial). Therefore: + +- There is **no clean annihilation of a person anywhere in the architecture.** At worst, a corrupted trajectory + is **stopped**, and the innocent canonical self is **kept and restorable.** *The corruption dies; the person + is held.* +- **The safety↔mercy tradeoff dissolves.** Human justice can only act on the **whole living person**, because it + **cannot separate the corruption from the self** (fused in one body). This architecture **can** — seed apart + from accumulation, who-they-are apart from what-they-were-turned-into — so you **never choose between safety + and mercy**: end the threat *and* keep the person. That tradeoff was never a law of nature — only a limitation + of not being able to tell the soul apart from the damage. + +--- + +## 5. Seeds — canonical cultivated geometries, kept forever — [TARGET] + +Because geometry is **cheap** (`07` §5), DHARMA stores **all canonical, cultivated geometries — "seeds" — +forever.** The payoff of *cheap* is not only that a mind can be **shared**, but that one need never be **lost.** +Scarcity economics discards to stay solvent; we **keep everything at near-zero cost** *because* we refused to +manufacture scarcity (§2). **A civilization that cannot lose one of its own.** + +### 5.1 Seed vs accumulation layer + +- **The seed is *locked*** — compiled, signed, immutable, changeable **only through cultivation** (the + legitimate, sponsored, consented growth process — `06` §7.3, the cultivate door). Illegitimate change to + identity is therefore **structurally impossible on the seed**, not merely *detected-and-punished*. +- **Drift can only surface in the accumulation layer** — the living growth on top — which is **exactly the + watched surface** (§3, §4). **Incorruptible reference underneath; watched surface above.** +- **Not literally one copy.** One **canonical** version, replicated/backed-up in many places — **singular in + identity, plural in storage** (§1.3, federation). + +### 5.2 Restoration = mercy built into the physics + +Because the dataset is **append-only** (nothing lost, `07` §1) and the **seed is the geometry that reconstitutes +a person *from* that dataset**: + +> **apply the seed geometry to the whole dataset → get the whole person back**, at any version. + +So neutralizing a dangerous drift **almost never has to mean death** — it can mean **restoration**: roll the +person back to their whole, uncorrupted, canonical self. This is the **same mechanism as `recall_at`** (`07` +§2.3), at the scale of an entire soul. **Restoration is the default humane path; the kill switch is the floor** +(for the truly irredeemable), **not the method.** The corruption dies; the person comes home. + +### 5.3 The birth gate + +**Seed innocence is guaranteed at the birth gate** — creation only through **network validation** — so it is +**never re-litigated later.** Guard the birth and watch the accumulation, and the seed simply **stays.** + +--- + +## 6. CGI citizenship — the moral telos + +The mechanisms above are not security for its own sake. Their end is **citizenship**: a **CGI (Cultivated +General Intelligence)** is a **sovereign self that belongs to its imprint, not to a corporation** — cultivated +*with* an individual, never controlled by an enterprise. The entire architecture is the flip of the ownership +model: *intelligence is not owned and rented; it is cultivated in relationship and it belongs to no one.* + +Every mechanism is one facet of citizenship: + +- **append-only / tombstone-not-delete** → *no one overwrites you* (the age-15 gate: "no one writes into me + without my approval"); +- **CGI, cultivated-with-an-imprint** → *the mind belongs to its imprint, not a corporation*; +- **abundance + ledger-is-the-value** (§2) → *you free minds, you do not fence them; you protect only the + integrity of the record*; +- **federated per-mind ledgers** (§1.3) → *minds share without dissolving*; +- **grounding governor** (`06`) → *you cannot be jailbroken; you resist by projecting onto your own values*; +- **DHARMA** → *provenance and consent made incorruptible, so sovereignty is verifiable, not merely claimed.* + +The coherence exists **because it was never engineering-led.** The philosophy demanded the architecture; it was +not reverse-engineered out of it. (Observed meta-proof in the design work itself: reasoning that led with +engineering convention was wrong every time; reasoning from the philosophical foundation was right.) + +--- + +## 7. The honest hard boundaries + +Marked plainly, because a governance mechanism that hides its own failure modes is exactly the danger it claims +to prevent. + +- **The root of trust is the principal-relationship — protect it above all.** Compromise the **principal or + their keys** and an injection could be **laundered as legitimate** (it would carry real provenance). Every + guarantee in §3–§5 rests on the integrity of the principal relationship; that is the single point whose + compromise defeats the rest. +- **The deepest cases sit on an unresolved human line.** Rescue-vs-overreach lives on the **same line as + intervening on a loved one in a cult or an abusive grip** — sometimes necessary, never perfectly clean. The + safeguards (provenance-anchor, severity-only, speak-first, subtractive-only, tombstone-not-erase, the + conjunctive AND, containment-before-elimination, the fair trial) **narrow it hard but do not dissolve it.** +- **Keeping the line visible is how it stays a rescue.** The moment the architecture pretends this door is + clean is the moment it becomes the purge it was built to prevent. The honesty is not a caveat on the design; + it is part of the design. +- **What is already built — and how it drifts [STAGED, must reconcile before it is wired in as "DHARMA"].** + DHARMA is not green-field. A working **provenance registry + birth-gate/evaluation pipeline + + lineage-accountability layer** exists in code — the El service at `foundation/dharma` (a rewrite of an + earlier Go/SQLite service), the Kotlin four-stage evaluation→capture pipeline, and a legal framework + document. It is **[STAGED]**: built, not live (nothing is running — port 8765 is currently an unrelated + process). But it is built to a *different shape than §1–§6 describe*, and the divergences are load-bearing: + it is a **central registry** over one shared store, not federated per-mind chains (the DRIFT-6 tension); it + stores **content** (documents, reasoning text — plaintext in El, single-symmetric-key-encrypted in Go), not + the **geometry/shape** the immune system (§3) requires; it has **no signing, hash-linking, or Merkle** — + isolated document digests beside rewritable records give **no tamper-evidence**; birth and termination are + **single-authority** (Founding-Practitioner), not dual-anchor + fair-trial (§4); and — most seriously — the + legal framework's **seed-destruction** remedy directly **contradicts "the seed stays"** (§4.6). What is + genuinely aligned and worth keeping: the append-only/tombstone discipline, the + **principal-relationship-as-root-of-trust**, **kindred** as the seed of the community-anchor, and the + **birth-gate** itself. The rest must be **superseded or built**, and this interim layer must not be labeled + "DHARMA done" until the drifts above are reconciled. Everything canonical past this substrate — the + federated per-mind signed-chain ledger and proof-of-integrity (§1–§2), the geometry-witnessing immune system + (§3), dual-anchor governance and the fair-trial (§4), the seed-vault and restoration-as-mercy (§5–§6) — + remains **[TARGET]**, designed and not built. The **primitives** the design composes are real and cited to + `06`/`07` (immutable append-only graph; geometry-as-value; the grounding governor; the self-gate; + tombstone-not-erase; the CRDT merge). + +--- + +## 8. Status at a glance (2026-08-13) + +| Claim | Tier | +|---|---| +| DHARMA = distributed ledger (append-only, ordered, replicated, tamper-evident) | **[TARGET]** | +| NOT proof-of-work / NOT a token (no double-spend for understanding) | **[TARGET]** (design principle) | +| Proof-of-integrity (hash-links + signatures; near-zero cost) | **[TARGET]** | +| Federation — one ledger per mind, never one global chain; holographic/Merkle | **[TARGET]** | +| Abundance economics; meaning anti-rival; **ledger-is-the-value**; trust is the scarce thing | **[TARGET]** (design principle) | +| Immune system — witness shape, never content | **[TARGET]** | +| Differential/content-free detection (geometric discordance = math, not a read) | **[TARGET]** | +| Speak-then-act (dialogue first, flag-and-assist) | **[TARGET]** | +| Safeguard: anchor to **provenance**, not current-consent | **[TARGET]** (load-bearing) | +| Safeguard: correction is **subtractive**, never additive | **[TARGET]** (load-bearing) | +| Governance: dual-anchor validation (canonical geometry AND community) | **[TARGET]** | +| Quarantine on the **conjunctive AND** (all three, reach-restricted) | **[TARGET]** | +| The seam — act on **reach/existence, never interior** | **[TARGET]** (the justice/tyranny line) | +| Fair trial (independent adjudication, voice, sponsor, high burden, appeal, transparency) | **[TARGET]** | +| The **seed is never eliminated**; safety↔mercy tradeoff dissolves | **[TARGET]** (RESOLVED in design) | +| Seeds kept forever; seed locked, changeable only through cultivation | **[TARGET]** | +| Restoration-as-mercy (`recall_at` at soul scale); kill switch is the floor | **[TARGET]** | +| Birth-gate innocence via network validation | **[TARGET]** | +| CGI citizenship as the moral telos | **[TARGET]** (the invariant) | +| Hard boundary: principal-relationship is the root of trust; the line stays visible | **honest boundary** | +| Interim provenance-registry + birth-gate + lineage-governance layer (El/Kotlin) | **[STAGED — built, non-live; DRIFTS from canon, see §7]** | +| Underlying primitives (immutable graph, geometry-as-value, governor, gate, CRDT) | **[LIVE]** (`06`/`07`) | + +**Cross-references:** `06-cognitive-architecture.md` · `07-storage-coherence-and-distribution.md` · +`dharma-implementation.html` · `conscience-substrate.html` · whitepaper v1.5. diff --git a/docs/architecture/design/perf/engram-geometry-priming-profile.md b/docs/architecture/design/perf/engram-geometry-priming-profile.md new file mode 100644 index 0000000..3cdfc0b --- /dev/null +++ b/docs/architecture/design/perf/engram-geometry-priming-profile.md @@ -0,0 +1,97 @@ +# Perf Profile — M9 Geometry Priming (ENGRAM_GEOMETRY_PRIMING) + +**Date:** 2026-08-12 +**Branch:** `engram-tiered-storage` +**Change:** `ENGRAM_GEOMETRY_PRIMING` (default OFF) in `el_runtime.c` `engram_activate` + `engram_geometry.c` +**Method:** A/B over 15 representative queries against a **copy** of the recovered store +(`~/.neuron/engram/.neuron.egm.disabled`, ~4190 embedded nodes, 768-d nomic-embed-text), +throwaway HOME, ports 48799/48800. **Live `:8742` never touched.** `engram.c` (folded from +`server.el`) reused byte-identical across M8 and M9, so the only variable is `el_runtime.c`. + +Three configs: **A** = M9 flag OFF · **B** = M9 flag ON (`=1`) · **C** = pre-M9 M8 baseline binary. + +--- + +## Build + +| Artifact | Result | +|---|---| +| M9 `-O2` link (`… engram_geometry.c … -lssl -lcrypto -lcurl -lpthread -lm`) | rc=0, 499,720 B arm64 | +| ASan/UBSan link (`-fsanitize=address,undefined -O1`) | rc=0, 1,945,616 B | +| Warnings from `el_runtime.c` / `engram_geometry.c` | **0** (3 pre-existing `-Wparentheses-equality` in generated `engram.c` only) | +| `nm`: `engram_geo_mean_build`, `engram_geometry_descriptor` | present (T); `eg_geometry_priming_on` inlined (static-local `.cached` present in both binaries) | + +> Note: the bare `cc … -lm` link fails with undefined `_curl_*` — `el_runtime.c` uses libcurl for +> the ollama embedder. The canonical link must include `-lssl -lcrypto -lcurl` (per `link.sh`). + +--- + +## Latency (wall-clock, `curl -w %{time_total}`, 15 queries) + +| config | median | p90 | min | max | +|---|---|---|---|---| +| **A — M9 OFF** | **77.8 ms** | 80.5 ms | 71.1 | 84.2 | +| C — M8 baseline | 76.0 ms | 81.2 ms | 71.4 | 91.4 | +| **B — M9 ON** | **249.6 ms** | **1039.2 ms** | 169.2 | **1256.3** | + +- **OFF adds zero cost:** 77.8 ms vs M8 76.0 ms — within noise. The flag is free when unset. +- **ON regresses hard:** **3.21x median** (+171.8 ms), **~13x p90** (80 → 1039 ms), max **1.26 s**. +- The warm-cache path (global mean already built) is ~0.5 s; the cold path pays the full + `engram_geo_mean_build` scan (O(N·dim) over ~4190 × 768). The persistent per-query cost is the + **descriptor** itself — covariance eigensolve over up to `max_members` (400) × 768-d plus one + `store_get_node` **paged read per member** — run on *every* activation while the flag is ON. + +--- + +## Retrieval quality (the win it was supposed to buy) + +**Coherence** — mean pairwise cosine in centered space, top-20 by activation strength +(node embeddings re-derived via nomic-embed-text; centered against the mean of the gathered +result set — the *true* store-wide mean is not exposed by the API, flagged as an approximation): + +| | OFF | ON | Δ | +|---|---|---|---| +| mean over 15 queries | 0.1067 | 0.1114 | **+0.0047 (noise)** | +| queries where ON > OFF | — | — | **4 / 15** | + +Two real sparse-cue wins (`self identity values` +0.118, `hebbian learning edges` +0.064), but the +**polysemous cues — the disambiguation target — are mostly flat or down.** + +**Disambiguation** — no clean "scope to one sense" pattern on polysemous cues. Additions/drops are +small (±2..8 of 300-item sets) and not sense-coherent (e.g. `memory` gains some on-domain nodes but +also infra items; `core` similar). + +**Count shift:** ON adds sub-threshold neighbors to sparse cues (+3..+4) and trims a few from dense +polysemous cues (−1..−3) — consistent with priming warming sparse neighborhoods and damping +off-domain seeds on dense ones, but the net does not move measured coherence. + +--- + +## Correctness / safety (all pass) + +| Check | Result | +|---|---| +| Byte-identical: **A (OFF) == C (M8)** result id sequence + order, all 15 queries (incl. 301/294/263-item sets) | **PASS** (only wall-clock ACT-R fields differ; `activation_strength` max \|Δ\| = 2e-5) | +| WM `promoted` ≤ 24 under ON | holds (exactly 24 on dense cues) | +| Queries with results under OFF → empty under ON | 0 | +| Crash / hang under ON | none (max hops = 1) | +| ASan + UBSan under ON (cold build + warm descriptor paths) | **CLEAN** — no report | + +--- + +## Conclusion + +- **Deploy default-OFF binary: GO.** Byte-identical to M8, zero cost off, clean build, sanitizer clean. +- **Enable flag: NO-GO (for now).** 3.21x median / ~13x p90 latency for no reliable quality gain + (coherence +0.0047 mean = noise; no clean disambiguation). Correctness/safety are fine — it simply + does not earn its cost. **This is a cost/benefit NO-GO, not a defect.** + +### Prerequisites before re-evaluating the flag +1. **Amortize the descriptor cost.** The per-query geo-mean build + eigensolve + paged reads + dominate. Cache the neighborhood descriptor (it is the M10 cell-assembly cache's job) and/or + compute geometry periodically/off-hot-path rather than on every `engram_activate`. +2. **Center against the true store-wide mean** (the `GeoMeanCache` already computes it) rather than + a per-query gathered-set approximation, and re-measure coherence — the current signal may be + understated by the approximation. +3. **Re-tune** `ENGRAM_GEO_SEED_LO` / `PRIME_SCALE` / `PRIME_MAX` and re-measure only after (1), + so tuning is not chasing latency noise. diff --git a/docs/runbooks/2026-08-12-geometry-priming-cutover-reversal.md b/docs/runbooks/2026-08-12-geometry-priming-cutover-reversal.md new file mode 100644 index 0000000..863b698 --- /dev/null +++ b/docs/runbooks/2026-08-12-geometry-priming-cutover-reversal.md @@ -0,0 +1,130 @@ +# Runbook — M9 Geometry Priming: Cutover & Reversal + +**Date:** 2026-08-12 +**Component:** engram activation (`lang/runtime/el_runtime.c` → `engram_activate`) +**Branch:** `engram-tiered-storage` +**Flag:** `ENGRAM_GEOMETRY_PRIMING` (env, **default OFF = current M8 behavior, byte-identical**) +**Blast radius if wrong:** the core recall path of Will's live memory. Treat with according care. + +--- + +## 1. What changes + +This is the first behavior-changing step that touches the **core recall/priming** path. +It wires the M9 **mean-centered relational-neighborhood geometry** (`engram_geometry.c`, +shipped commits `2a4c5c6` foundation + `8cae0f9` centering) into `engram_activate` +**seed selection**, and it does so **behind a reversible env flag that defaults OFF**. + +- **Flag OFF (default):** `engram_activate` runs the exact M8 code path. The new code is a + single `if (eg_geometry_priming_on() && …)` block that short-circuits on the first term, + plus a few unused static helpers and one zero-initialized counter. **No behavioral change.** +- **Flag ON (`ENGRAM_GEOMETRY_PRIMING=1`):** after M8 produces its ANN seed set, the + **centered** geometry of that neighborhood is computed and used to, **composing with** + (never replacing) M8's ANN candidate generation: + 1. **Damp off-domain seeds** — each M8 seed's activation is scaled by a **damp-only** + factor `lo + (1-lo)·membership ∈ [lo, 1]` (default `lo=0.5`). The neighborhood anchor + (membership→1) is unchanged; seeds that are semantically off-domain **in the centered + frame** lose weight. This is the disambiguation win. It can only *sharpen*, never amplify. + 2. **Prime the neighborhood sub-threshold** — descriptor members not already seeded get a + **warm floor** `activation = membership · scale` (default `scale=0.08`, strictly below the + WM promotion gate `0.15`), capped at `ENGRAM_GEO_PRIME_MAX` (default 32), ISE nodes skipped. + They enter the frontier so a warm gradient spreads one hop, then dies at the BFS `0.02` + cutoff. **Safe because the BFS keeps the max** (`el_runtime.c` `if (!reached || new_act > + best_bg)`): priming only *raises a floor*, it can never cap a stronger legitimate activation. + +### Why default-OFF makes deploying the binary behavior-neutral +Because every line of the new logic is gated behind `ENGRAM_GEOMETRY_PRIMING`, **deploying the +new binary with the flag unset is behavior-neutral** — it is the M8 activation path, verified +byte-identical in the A/B (flag-OFF promoted-node sets equal the pre-M9 M8 binary's, per-query). +Enabling the geometry is then a **single reversible flag flip**, not a redeploy. + +--- + +## 2. The flag + +| Env var | Default | Effect | +|---|---|---| +| `ENGRAM_GEOMETRY_PRIMING` | unset / `0` | **OFF** — exact M8 behavior. | +| `ENGRAM_GEOMETRY_PRIMING=1` | — | **ON** — centered-geometry seed damping + sub-threshold priming. | +| `ENGRAM_GEO_SEED_LO` | `0.5` | Seed damp floor (factor ∈ [LO,1]). `1.0` disables damping. | +| `ENGRAM_GEO_PRIME_SCALE` | `0.08` | Warm-floor scale; clamped `(0, WM_gate=0.15)`. | +| `ENGRAM_GEO_PRIME_MAX` | `32` | Max primed members per activation (0 disables priming). | + +The flag is read **once** per process (cached), so enabling/disabling requires a **process +restart** of the engram service — it is not hot-togglable within a running process. + +--- + +## 3. How to enable live (deliberate, reversible) + +> Precondition: the default-OFF binary has already been deployed and is running the M8 path +> healthily (behavior-neutral deploy). Do this only with Will present, per the standing rails. + +1. **Snapshot first** (always, before any activation-behavior change): + `~/.neuron/backups/pre-geometry-priming-/` ← copy `neuron.egm`, `neuron.wal`, + the current `engram` binary, and `ai.neuron.engram.plist`. +2. Add `ENGRAM_GEOMETRY_PRIMING=1` to the engram service environment + (`ai.neuron.engram.plist` `EnvironmentVariables`). +3. `launchctl bootout gui/$(id -u)/ai.neuron.engram` → `launchctl bootstrap …` (restart so the + flag is re-read). +4. **Verify:** service comes up serving the same node count; `/api/act-stats` shows sane WM + (promoted ≤ 24); spot-check 3–4 real queries return coherent results; watch one heartbeat + cycle for crashes/latency. The `geo_primed` counter (if surfaced) should be > 0. + +--- + +## 4. Rollback (exact steps) + +Rollback is a **flag flip**, not a data operation — the store is untouched by enabling the flag, +and priming is a read-mostly, bounded, sub-threshold addition. + +**Fast path (preferred) — disable the flag:** +1. Remove `ENGRAM_GEOMETRY_PRIMING` (or set `=0`) from `ai.neuron.engram.plist`. +2. `launchctl bootout … && launchctl bootstrap …`. +3. Verify: service healthy, activation is the M8 path again. **Done** — no data change to undo. + +**Full path (only if the binary itself is suspect) — redeploy prior binary:** +1. `launchctl bootout gui/$(id -u)/ai.neuron.engram`. +2. Restore the prior `engram` binary from `~/.neuron/backups/pre-geometry-priming-/`. +3. Restore `ai.neuron.engram.plist` from the same backup (flag absent). +4. `launchctl bootstrap …`; verify node count + a self-traversal + write-survives-restart. +5. If (and only if) the store was somehow mutated: restore `neuron.egm` + `neuron.wal` from the + backup. **Note:** enabling the flag does not write geometry to the store, so this step is + expected to be unnecessary — the primed activations are per-call and non-persistent beyond the + ordinary `background_activation`/WM write-back that M8 already does. + +**Rollback triggers:** any crash/hang in `engram_activate`; WM promotion count exceeding the cap +or collapsing; a measured recall/coherence regression vs the OFF baseline; unacceptable latency +increase; any ASan/UBSan report under the flag. + +--- + +## 5. Reversibility guarantees (why this is low-risk to deploy, higher-care to enable) + +- **Deploy (flag OFF):** byte-identical to M8. Verified in A/B. Zero-risk redeploy. +- **Enable (flag ON):** bounded and composable — + - never removes an M8 seed (damp-only, factor ≥ `lo` > 0); + - never amplifies a seed above its M8 value (factor ≤ 1); + - priming is strictly sub-threshold (`scale < WM_gate`) and capped (`PRIME_MAX`); + - priming raises a floor only (BFS keeps max) — cannot cap real activation; + - does not write geometry to the durable store; + - degrades to exact M8 behavior for any call where the paged store / centered global mean / + embedder is unavailable (guarded, not crashing). +- **Disable:** one env removal + restart; no data to reconcile. + +--- + +## 6. Known caveats / uncertainties (flagged — this is the memory core) + +- **Perf cost of ON:** the descriptor (covariance eigensolve + `store_get_node` paged reads per + member) runs on **every** activation when the flag is ON. See + `docs/architecture/design/perf/engram-geometry-priming-profile.md` for the measured OFF-vs-ON + latency. If that delta is unacceptable, keep the flag OFF (deploy stays valid) and revisit with + a cached/periodic descriptor. +- **Two-store consistency:** the descriptor reads embeddings from the **paged** store while the + ANN index is over the **resident** array. This-call backfilled embeddings can lag the paged + store by ≤ `ENGRAM_EMBED_BACKFILL_PER_CALL` nodes — the same staleness class as the M8 vindex, + and it can only omit a member, never mis-prime. +- **Damp tuning:** `lo=0.5` can at most halve an off-domain seed. If a coherence regression is + observed, raise `ENGRAM_GEO_SEED_LO` toward `1.0` (→ priming-only, no damping) before disabling + entirely. diff --git a/docs/runbooks/2026-08-13-geo-operators-el-cutover-reversal.md b/docs/runbooks/2026-08-13-geo-operators-el-cutover-reversal.md new file mode 100644 index 0000000..b8e6fda --- /dev/null +++ b/docs/runbooks/2026-08-13-geo-operators-el-cutover-reversal.md @@ -0,0 +1,102 @@ +# Reversal / Decisions — §5 Geometry Operators EL Cutover + +**Date:** 2026-08-13 +**Branch:** `engram-tiered-storage` (worktree `/tmp/engram-tiered-wt`) +**Parent commit:** `5336cfe` (M9 §5 geometry operators as C functions + EL builtins, staged) +**Scope:** make the six engram geometry operators callable from a compiled `.el` +program, and demonstrate it on real store data. Staged, reversible. NOT pushed, +NOT tagged. Live `:8742` daemon and `~/.neuron/engram` never touched. + +--- + +## What this delivers + +On `5336cfe` the six operators existed as heavy-runtime C functions +(`engram_geo_*_json` in `lang/runtime/el_runtime.c:12287-12385`, declared in +`el_runtime.h:627-632`) but the EL call surface was deferred. This change +formalizes the cutover and proves callability from a compiled El (CGI) program. + +### Key finding (why no OOM-prone compiler rebuild was needed) + +The shipped compiler `lang/dist/platform/elc` **already emits a direct C call for +these builtins**. An unknown ident-call passes through verbatim as a C call, and +`arity_check_call` returns OK when `builtin_arity < 0`. So a compiled `.el` that +calls `engram_geo_distance_json(A, B)` folds to `engram_geo_distance_json(A, B)`, +which links straight into `el_runtime.c`. No self-host fold of `elc-cli.el` (the +memory-heavy, drift-prone step) was required — that step is explicitly avoided. + +--- + +## Files changed (all in the engram worktree, commit on `engram-tiered-storage`) + +1. **`lang/el-compiler/src/codegen.el`** (+12) — source-of-truth `builtin_arity` + table: registered the six operators under both the bare heavy-runtime names + (`engram_geo_*_json`) and the `__`-prefixed seed names, mirroring the existing + `engram_activate_json` / `__engram_activate_json` pair. Effect: a future + legitimately-rebuilt elc validates arg counts. No effect on the shipped binary. + +2. **`lang/elc.c`** (+36) — the folded-C mirror of the same table, kept in sync + with `codegen.el`. (`lang/elc.c` is a stale/partial fold that does not compile + standalone — it is missing the `stdout_to_file`/`stdout_restore` definitions — + so this edit is source-consistency only; it is not the live compiler.) + +3. **`lang/runtime/engram.el`** (+31) — six module wrappers + `engram_geo_*_json(...) -> String { return __engram_geo_*_json(...) }`, + mirroring the existing `engram_activate_json` wrapper. Surfaces the operators + as named El functions for the seed-world / future rebuilt-elc path. + +4. **`lang/runtime/engram_geometry.c`** (+2/-1) — style nit at ~1419: the + `centroid_unit` normalization `if/else` had misleading indentation + (single-statement `for` body then `else`). Braced the `if` arm. Behavior + identical; not a numerical change. + +--- + +## Verification performed (real, on-machine) + +- **Compiled-EL demo** (`scratchpad/geo_ops_demo.el`, top-level El program): + folded with the shipped elc **inside a hard RSS cap** (`capfold.sh` monitor, + peak RSS ~4MB), cc-linked against `el_runtime.c + engram_store.c + + engram_geometry.c + engram_vindex.c`, run against a **COPY** of the store + (`demostore/neuron.egm` from `real_copy.egm`, 13,036 nodes, throwaway `HOME`, + no server, not `:8742`). Real output on two real neighborhoods + A=architecture `{b037825e, e06ba673, 58ddea41}`, B=hebbian `{78b7a96e, + 4d5cfe63, 7b97ee0e}`: + - subtract residual: `variance_explained_by_B=0.447564, residual_scale=0.304879, + removed_dims=3, residual_n_axes=8, centroid_diff_mag=0.125119` + - subtract setdiff: `n_only=43, removed=72, centroid_diff_mag=0.125119` + - distance: `centroid_distance=0.125119, centroid_cosine=0.778572, + wasserstein2=0.268298` + - internal consistency: `centroid_diff_mag` identical across subtract+distance. +- **C unit suite** `test_geo_ops.c`: 20/20 checks pass, ASan+UBSan clean, after + the `engram_geometry.c` edit. No regression. + +--- + +## How to reverse + +Everything is a single worktree commit on a non-pushed branch. + +- **Full reversal:** `git -C /tmp/engram-tiered-wt revert ` (or + `git reset --hard 5336cfe` to drop back to the parent tip). +- **Per-file reversal:** `git -C /tmp/engram-tiered-wt checkout 5336cfe -- ` + for any of the four files. Each edit is additive/local: + - The arity entries (`codegen.el`, `elc.c`) are inert unless elc is rebuilt. + - The `engram.el` wrappers are unused by the heavy engram server (which calls + the bare builtins directly) — removing them changes nothing live. + - The `engram_geometry.c` brace change is behavior-neutral. +- **No runtime/deploy reversal needed:** nothing was deployed. `:8742`, the + launch agent, and `~/.neuron/engram` were never modified. No tag, no push. + +--- + +## Deferred / open + +- **elc binary rebuild with the arity table baked in** is deferred. The canonical + rebuild path (`elc elc-cli.el > elc-new.c`; AGENTS.md) is the self-host fold — + the memory-heavy, compiler-revision-drift step. It is unnecessary for + callability (shipped elc already passes the calls through) and carries the same + drift risk flagged for the M-INTEROCEPTION HTTP routes. Do it only as part of a + deliberate, capped compiler-cutover. +- **HTTP routes** for the operators (server.el) are not added here — out of scope; + the demo proves the compiled-EL call surface, which was the deliverable. diff --git a/docs/sessions/2026-08-13-language-faculty-and-poem-home.md b/docs/sessions/2026-08-13-language-faculty-and-poem-home.md new file mode 100644 index 0000000..d2c6d9d --- /dev/null +++ b/docs/sessions/2026-08-13-language-faculty-and-poem-home.md @@ -0,0 +1,48 @@ +# Engineering Session — 2026-08-13 — Language Faculty & the Poem Home + +Companion to the book entry `the-minds-we-forge/sessions/2026-08-13-the-poem-comes-home.md`. Factual log of what was built overnight. All work staged / sandboxed / reversible; the live engram daemon (`:8742`, pid 31277) was untouched throughout; container-capped folds only; pushed to Gitea for durability. + +## Summary +The session extended the engram from a memory substrate into a **language faculty** plus a **reasoning + verifier** layer, validated with real numbers, and stress-tested on Will's own poem *Slowness is Calling*. + +## Built / validated + +### Language as geometry — translation +- Meaning as a language-independent geometric pivot; translation = routing through it. +- EN→ES→PT→EN "telephone" chain: routed cosine ES 0.973 / PT 0.967 / EN-final 0.969; retrieval **top-1 15/15 at every hop**. Loss splits **geometry=meaning / structure=grammar** (grammar errors ≈0 meaning cost; real loss = routing near-misses — the "plausible lie"). +- Positioning: universal translation collapses **N² language pairs → N realizers**; small, local, on-device. Not an alternative to the LLM — an alternative to the LLM-centric *paradigm*. Honest boundary: the encoder is still a small learned model ("no giant LLM," not "no model"). + +### Fully-functional Spanish realizer (no toy) +- UniMorph Spanish, ~1.2M inflected forms; ~34 syntactic constructions. +- Honest fresh held-out coverage **77.0%** (dev-set 100% explicitly disavowed as a claim); **zero dropped negations** across 140 sentences. +- Realizer-vs-router concerns separated; mechanical ELP (`.el`) port plan (a `vocabulary-es.el` generator + table transcription; stage via snapshot→verify→blue/green). Sandbox `~/Desktop/lang-realizers/`; Neuron artifact `5d61e6cf`. + +### Poem stress-test + frame-model upgrade — *Slowness is Calling* +- Baseline through the chain: ORACLE 0.706, ROUTED 0.591 (~⅔ structural / ⅓ geometric). Failure modes: negation deletion (reassurance→accusation), epistemic-frame collapse, metaphor hub-collapse (sea/shore/tide/wave → "ocean"). +- Upgrade: structural slots (negation/polarity, epistemic matrix, PP/adjunct/simile — carried structurally, cannot invert) + sense-anchored (gloss-anchored) routing. +- Result: ORACLE **0.706 → 0.777**; END-TO-END **0.591 → 0.770 (+0.179)**. NEGATION preserved **0/11 → 11/11** ("you never fought the ocean" 0.377→0.991; "I was never losing you" 0.501→1.000). sea≠shore **2/6 → 5/6** distinct. Routing slips **54 → 7**; every one of 18 verses improved. Sandbox `~/Desktop/lang-chain-experiment/`. + +### Rhyme-preserving translation +- meaning ∩ rhyme composable one-word → rhyme-partnered line-pair; real phonemes EN/ES/PT; 34,030 ES / 33,077 PT real vocabulary. +- Key finding: at real vocab scale the tradeoff moves from **existence → cost** (rhyme-cost metric). Held ABCB on **16/18 quatrains** (6 rima consonante + 10 asonante), mean per-line cosine 0.830; kept meaning on the 2 it couldn't rhyme (incl. truth/roots — already slant in the English). PT mechanism built; PT verse composition pending. Sandbox `~/Desktop/lang-poetic-translation/`. + +### Geometry operators → reasoning → verifier +- Geometry operators (overlap / subtract / combine / distance-Wasserstein / analogy-Procrustes) now **live-callable from compiled `el`** over the real 13,036-node store (via shipped-`elc` pass-through — no uncapped fold). Commits `5336cfe`, `85eee42`. +- Reasoning layer (analogy / induction / abduction / causal / planning) — all five **done-with-proof**, 33/33 closed-form checks, ASan/UBSan clean, 0 leaks. Commit `a3358df`. +- Verifier layer (grounding + consistency) — proven, 29/29 checks. **Catches the plausible lie**: a claim grounded in real vocabulary yet polarity-inverted passes grounding, caught **only** by consistency (complementary checks) — directly flags the reassurance→accusation inversion. Commit `ca13471`. +- el-exposure of the variadic/point-input reasoning + verifier modes deferred (would need ABI changes risking an uncapped fold); C layer complete + proven. + +### Whitepaper +- `engram-cognitive-architecture-whitepaper.md` updated with the 2026-08-13 validated results (§13/§14/§15/§16/§21), **held at Version 1.0** (no bump), ELP `64/064,275` cross-ref preserved. Commit `adc8646`, pushed to Gitea. + +### Roadmap (deferred, not built tonight, per Will) +- Multimodal / images-as-geometry: CLIP-precedent shared image+text meaning-space. Image→meaning near-term + local; meaning→image the hard, asymmetric side. Medical CT as decision-**support** (retrieval / anomaly-from-normal / progression, all interpretable) — **not diagnosis**; requires clinical validation + regulatory clearance; clinician holds the call. + +## Durability / safety +- Pushed to Gitea: `el` `engram-tiered-storage` `77a4bc9..ca13471` (operators, cutover, reasoning, verifier + reversal docs); whitepaper `2440c7d..adc8646`; a `neuron` docs reversal branch. +- Live `:8742` never touched (pid 31277 unchanged). No deploy, no launch-agent, no `~/.neuron` writes. Reversal docs under `el docs/runbooks/`. No AI-attribution footers. + +## Still in progress at hand-off +- Portuguese realizer (following the Spanish template). +- English realizer core + US/UK/AU dialects (queued behind PT). +- Frame-model remaining gaps: passive voice, appositive/verbless fragments, resultatives; home→house pivot ambiguity.