docs: design proposal — searchable, recency-aware conversation memory

Grounds the 'summarize my recent conversations returns nothing' issue: it's a
RETRIEVAL gap, not storage (conversations ARE persisted per-turn via auto_persist;
live engram has 59 conversation nodes). Proposes recency-windowed retrieval +
per-session threading + (roadmap) semantic search. No code — proposal for Tim + Will.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitea/workflows/deploy-gke.yaml

@@ -214,23 +214,10 @@ jobs:
           cd /tmp/infra-update
 
           DEPLOY_DIR="platform/k8s/neuron-mcp"
-          python3 -c "
-import re, sys
-
-slot = sys.argv[1]
-idle = sys.argv[2]
-
-def set_replicas(path, count):
-    with open(path) as f:
-        content = f.read()
-    content = re.sub(r'^(  replicas: )\d+', r'\g<1>' + str(count), content, count=1, flags=re.MULTILINE)
-    with open(path, 'w') as f:
-        f.write(content)
-    print(f'  {path}: replicas set to {count}')
-
-set_replicas(f'{DEPLOY_DIR}/deployment-{slot}.yaml', 1)
-set_replicas(f'{DEPLOY_DIR}/deployment-{idle}.yaml', 0)
-" "$SLOT" "$IDLE"
+          sed -i "s/^  replicas: .*/  replicas: 1/" "${DEPLOY_DIR}/deployment-${SLOT}.yaml"
+          sed -i "s/^  replicas: .*/  replicas: 0/" "${DEPLOY_DIR}/deployment-${IDLE}.yaml"
+          echo "  deployment-${SLOT}.yaml: replicas set to 1"
+          echo "  deployment-${IDLE}.yaml: replicas set to 0"
 
           git config user.email "ci@neurontechnologies.ai"
           git config user.name "Neuron CI"
@@ -246,7 +233,7 @@ set_replicas(f'{DEPLOY_DIR}/deployment-{idle}.yaml', 0)
           echo "Verifying neuron-mcp-${SLOT} is healthy..."
           kubectl rollout status deployment/"neuron-mcp-${SLOT}" \
             --namespace=neuron-prod \
-            --timeout=3m
+            --timeout=8m
 
           echo "Active service endpoints:"
           kubectl get endpoints neuron-mcp -n neuron-prod

awareness.el

@@ -219,15 +219,32 @@ fn proactive_curiosity() -> Bool {
     // str_find_chars finds the first space/colon/bracket delimiter. sp > 3 guards against
     // very short or bracket-prefixed labels like "[BacklogItem]" (sp=0, not > 3 → skipped).
     // EL scoping: state_set/state_get pattern used because let inside if creates inner scope.
-    // (2026-06-11 self-review)
+    //
+    // NODE TYPE FILTER (2026-06-19 self-review): only derive auto_term from Memory,
+    // BacklogItem, or Entity nodes. Knowledge nodes are stable reference material —
+    // using their first word as a curiosity seed creates a self-reinforcing loop: e.g.
+    // "Numeric tier strings in Engram..." (a Knowledge node) -> auto_term="Numeric" ->
+    // activates all "Numeric" nodes -> keeps that Knowledge node dominant in WM forever.
+    // Knowledge nodes should be REACHED by curiosity seeds, not drive them. Only dynamic
+    // personal/work nodes (Memory, BacklogItem, Entity) carry live contextual salience
+    // worth radiating from. (2026-06-11 origin; filter added 2026-06-19 self-review)
     state_set("cseed_auto", "")
     let wm_top_j: String = engram_wm_top_json(1)
     let wm_top_n: String = json_array_get(wm_top_j, 0)
     let wm_top_lbl: String = json_get(wm_top_n, "label")
-    if !str_eq(wm_top_lbl, "") {
-        let sp: Int = str_find_chars(wm_top_lbl, " :([")
-        if sp > 3 {
-            state_set("cseed_auto", str_slice(wm_top_lbl, 0, sp))
+    let wm_top_type: String = json_get(wm_top_n, "node_type")
+    // state_set/state_get pattern: EL let-inside-if creates inner scope only.
+    state_set("allow_auto", "0")
+    if str_eq(wm_top_type, "Memory") { state_set("allow_auto", "1") }
+    if str_eq(wm_top_type, "BacklogItem") { state_set("allow_auto", "1") }
+    if str_eq(wm_top_type, "Entity") { state_set("allow_auto", "1") }
+    let allow_auto: String = state_get("allow_auto")
+    if str_eq(allow_auto, "1") {
+        if !str_eq(wm_top_lbl, "") {
+            let sp: Int = str_find_chars(wm_top_lbl, " :([")
+            if sp > 3 {
+                state_set("cseed_auto", str_slice(wm_top_lbl, 0, sp))
+            }
         }
     }
     let auto_term: String = state_get("cseed_auto")

dist/awareness.c

@@ -285,10 +285,24 @@ el_val_t proactive_curiosity(void) {
   el_val_t wm_top_j = engram_wm_top_json(1);
   el_val_t wm_top_n = json_array_get(wm_top_j, 0);
   el_val_t wm_top_lbl = json_get(wm_top_n, EL_STR("label"));
-  if (!str_eq(wm_top_lbl, EL_STR(""))) {
-    el_val_t sp = str_find_chars(wm_top_lbl, EL_STR(" :(["));
-    if (sp > 3) {
-      state_set(EL_STR("cseed_auto"), str_slice(wm_top_lbl, 0, sp));
+  el_val_t wm_top_type = json_get(wm_top_n, EL_STR("node_type"));
+  state_set(EL_STR("allow_auto"), EL_STR("0"));
+  if (str_eq(wm_top_type, EL_STR("Memory"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  if (str_eq(wm_top_type, EL_STR("BacklogItem"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  if (str_eq(wm_top_type, EL_STR("Entity"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  el_val_t allow_auto = state_get(EL_STR("allow_auto"));
+  if (str_eq(allow_auto, EL_STR("1"))) {
+    if (!str_eq(wm_top_lbl, EL_STR(""))) {
+      el_val_t sp = str_find_chars(wm_top_lbl, EL_STR(" :(["));
+      if (sp > 3) {
+        state_set(EL_STR("cseed_auto"), str_slice(wm_top_lbl, 0, sp));
+      }
     }
   }
   el_val_t auto_term = state_get(EL_STR("cseed_auto"));

dist/neuron.c

@@ -1042,12 +1042,36 @@ el_val_t call_neuron_mcp(el_val_t tool_name, el_val_t args_json);
 el_val_t agentic_tools_literal(void);
 el_val_t agentic_tools_with_web(void);
 el_val_t dispatch_tool(el_val_t tool_name, el_val_t tool_input);
+el_val_t json_array_append(el_val_t arr, el_val_t item);
+el_val_t append_tool_log(el_val_t log, el_val_t name);
+el_val_t exec_tool_block(el_val_t block);
+el_val_t agentic_blob(el_val_t model, el_val_t system, el_val_t tools_json, el_val_t messages, el_val_t origin, el_val_t approval, el_val_t iteration, el_val_t tools_log, el_val_t content, el_val_t queue, el_val_t results, el_val_t next);
+el_val_t extract_all_text(el_val_t s);
+el_val_t strip_citations(el_val_t s);
+el_val_t agentic_api_turn(el_val_t model, el_val_t safe_sys, el_val_t tools_json, el_val_t messages);
+el_val_t agentic_engine(el_val_t session_id, el_val_t blob);
 el_val_t handle_chat_agentic(el_val_t body);
+el_val_t handle_session_approve(el_val_t session_id, el_val_t body);
 el_val_t handle_chat_as_soul(el_val_t body);
 el_val_t handle_dharma_room_turn(el_val_t body);
 el_val_t handle_dharma_room_turn_agentic(el_val_t body);
 el_val_t auto_persist(el_val_t req, el_val_t resp);
 el_val_t strengthen_chat_nodes(el_val_t activation_nodes);
+el_val_t safety_self_harm_phrases(void);
+el_val_t safety_abuse_phrases(void);
+el_val_t safety_general_hard_phrases(void);
+el_val_t safety_soft_phrases(void);
+el_val_t safety_normalize(el_val_t message);
+el_val_t safety_any_match(el_val_t text, el_val_t phrases_json);
+el_val_t safety_count_match(el_val_t text, el_val_t phrases_json);
+el_val_t safety_detect_bell_level(el_val_t message);
+el_val_t safety_classify_hard_bell(el_val_t message);
+el_val_t safety_soft_directive(void);
+el_val_t safety_hard_directive(el_val_t hard_type);
+el_val_t safety_augment_system(el_val_t system, el_val_t user_msg);
+el_val_t safety_contact_path(void);
+el_val_t handle_safety_contact_get(void);
+el_val_t handle_safety_contact_post(el_val_t body);
 el_val_t auth_headers(el_val_t tok);
 el_val_t axon_get(el_val_t path);
 el_val_t axon_post(el_val_t path, el_val_t body);
@@ -1110,6 +1134,7 @@ el_val_t session_update_meta_timestamp(el_val_t session_id);
 el_val_t session_auto_title(el_val_t session_id, el_val_t first_message);
 el_val_t handle_session_approve(el_val_t session_id, el_val_t body);
 el_val_t strip_query(el_val_t path);
+el_val_t flag_true(el_val_t body, el_val_t key);
 el_val_t err_404(el_val_t path);
 el_val_t err_405(el_val_t method, el_val_t path);
 el_val_t route_health(void);
@@ -1144,6 +1169,9 @@ el_val_t local_node_count;
 el_val_t snapshot_usable;
 el_val_t boot_num;
 el_val_t is_genesis;
+el_val_t guard_disk;
+el_val_t guard_disk_len;
+el_val_t safe_to_seed;
 
 el_val_t lang_profile(el_val_t code, el_val_t word_order, el_val_t morph_type, el_val_t has_case, el_val_t has_gender, el_val_t script_dir, el_val_t agreement, el_val_t null_subject) {
   el_val_t r = native_list_empty();
@@ -25890,14 +25918,28 @@ el_val_t proactive_curiosity(void) {
   el_val_t wm_top_j = engram_wm_top_json(1);
   el_val_t wm_top_n = json_array_get(wm_top_j, 0);
   el_val_t wm_top_lbl = json_get(wm_top_n, EL_STR("label"));
-  if (!str_eq(wm_top_lbl, EL_STR(""))) {
-    el_val_t sp = str_find_chars(wm_top_lbl, EL_STR(" :(["));
-    if (sp > 3) {
-      state_set(EL_STR("cseed_auto"), str_slice(wm_top_lbl, 0, sp));
+  el_val_t wm_top_type = json_get(wm_top_n, EL_STR("node_type"));
+  state_set(EL_STR("allow_auto"), EL_STR("0"));
+  if (str_eq(wm_top_type, EL_STR("Memory"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  if (str_eq(wm_top_type, EL_STR("BacklogItem"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  if (str_eq(wm_top_type, EL_STR("Entity"))) {
+    state_set(EL_STR("allow_auto"), EL_STR("1"));
+  }
+  el_val_t allow_auto = state_get(EL_STR("allow_auto"));
+  if (str_eq(allow_auto, EL_STR("1"))) {
+    if (!str_eq(wm_top_lbl, EL_STR(""))) {
+      el_val_t sp = str_find_chars(wm_top_lbl, EL_STR(" :(["));
+      if (sp > 3) {
+        state_set(EL_STR("cseed_auto"), str_slice(wm_top_lbl, 0, sp));
+      }
     }
   }
   el_val_t auto_term = state_get(EL_STR("cseed_auto"));
-  el_val_t results_auto = ({ el_val_t _if_result_101 = 0; if (str_eq(auto_term, EL_STR(""))) { _if_result_101 = (EL_STR("[]")); } else { _if_result_101 = (engram_activate_json(auto_term, 1)); } _if_result_101; });
+  el_val_t results_auto = ({ el_val_t _if_result_3 = 0; if (str_eq(auto_term, EL_STR(""))) { _if_result_3 = (EL_STR("[]")); } else { _if_result_3 = (engram_activate_json(auto_term, 1)); } _if_result_3; });
   el_val_t found_auto = json_array_len(results_auto);
   el_val_t total_found = (found + found_auto);
   el_val_t safe_auto = str_replace(auto_term, EL_STR("\""), EL_STR("'"));
@@ -25908,6 +25950,7 @@ el_val_t proactive_curiosity(void) {
   return 0;
 }
 
+
 el_val_t pulse_count(void) {
   el_val_t s = state_get(EL_STR("soul.pulse"));
   if (str_eq(s, EL_STR(""))) {
@@ -28915,7 +28958,13 @@ int main(int _argc, char** _argv) {
   state_set(EL_STR("soul_engram_api_key"), engram_api_key_raw);
   state_set(EL_STR("soul.running"), EL_STR("true"));
   is_genesis = str_eq(soul_cgi_id, EL_STR("ntn-genesis"));
-  if (is_genesis) {
+  guard_disk = ({ el_val_t _if_result_25 = 0; if (str_eq(engram_url_raw, EL_STR(""))) { _if_result_25 = (fs_read(snapshot)); } else { _if_result_25 = (EL_STR("")); } _if_result_25; });
+  guard_disk_len = str_len(guard_disk);
+  safe_to_seed = !((guard_disk_len > 200000) && (engram_node_count() < (guard_disk_len / 16000)));
+  if (is_genesis && !safe_to_seed) {
+    println(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("[soul] GUARD: loaded "), int_to_str(engram_node_count())), EL_STR(" nodes but snapshot file is ")), int_to_str(guard_disk_len)), EL_STR(" bytes \xe2\x80\x94 refusing to seed/save over a real graph")));
+  }
+  if (is_genesis && safe_to_seed) {
     el_val_t edge_count_now = engram_edge_count();
     if (edge_count_now < 100) {
       init_soul_edges();
@@ -28926,7 +28975,7 @@ int main(int _argc, char** _argv) {
     state_set(EL_STR("soul_snapshot_path"), snapshot);
     engram_save(snapshot);
   }
-  if (is_genesis) {
+  if (is_genesis && safe_to_seed) {
     el_val_t snap = state_get(EL_STR("soul_snapshot_path"));
     if (!str_eq(snap, EL_STR(""))) {
       engram_save(snap);

docs/DESIGN-conversation-retrieval.md

@@ -0,0 +1,100 @@
+# Design proposal: searchable, recency-aware conversation memory
+
+Status: **proposal — for Tim + Will, no code yet**
+Author: Neuron (Claude Opus 4.8), 2026-06-21
+Trigger: "Summarize the key themes across my recent conversations" returns nothing useful.
+
+---
+
+## TL;DR
+
+Conversations **are** being persisted — `auto_persist` writes every turn as a
+timestamped `Conversation`/`Episodic` node. The failure is **retrieval**, not
+storage. Two gaps:
+
+1. **No recency-ordered retrieval.** There is no way to ask "give me my last N
+   conversation turns by time." Search is keyword-ranked only.
+2. **Lexical-only search.** `search_memory` → `engram_search_json` is BM25/lexical.
+   A semantic/thematic query ("themes across recent conversations") doesn't share
+   keywords with the actual topic content, so it misses.
+
+The model literally tried to express the missing capability in the fake tool call
+it hallucinated: `"recency_weight": 0.8`, `"sort_by": "recency"`,
+`node_type: "ConversationTurn"`. It wanted a recency-windowed conversation fetch
+that doesn't exist.
+
+## What exists today (verified)
+
+- `auto_persist(req, resp)` (chat.el): after each non-agentic turn, stores
+  `{"q","a","created_at","source":"chat","label":"chat:<ts>"}` as
+  `engram_node_full(... "Conversation" ... "Episodic" ...)`, tags
+  `["Conversation","chat","timestamped"]`.
+- `conv_history_persist` (chat.el): a **single overwriting** `conv:history`
+  Episodic node holding the rolling JSON history (continuity across restarts) —
+  not per-turn, not individually searchable.
+- Live engram (founder instance): **5,113 nodes, 59 conversation nodes** — a mix
+  of `chat:<ts>`, several `conv:history` copies, and older `Q:/A:` nodes.
+- Retrieval surface for the agentic loop: `search_memory`, `recall`,
+  `neuron_search_knowledge`, `neuron_recall` — all **query-keyword** based.
+  None is "most recent N by time," none is embedding/semantic.
+
+## The gap, precisely
+
+| User intent | Needs | Have today |
+|---|---|---|
+| "summarize my recent conversations" | last-N-by-time fetch | ✗ (keyword only) |
+| "what did we discuss about X" | semantic match on topic | ~ (lexical only; misses paraphrase) |
+| "themes across everything" | semantic cluster over corpus | ✗ |
+
+`auto_persist` only fires on the **non-agentic** path (`handle_chat`). Worth
+confirming the **agentic** path (`handle_chat_agentic`) persists turns too — if
+not, agentic conversations never get stored, a second (smaller) gap.
+
+## Proposal
+
+Three layers, smallest-first. (1) alone fixes the headline use case.
+
+### 1. Recency-windowed conversation retrieval (the high-value, low-cost win)
+A runtime/engram primitive + an agentic tool:
+
+- **Engram**: `engram_recent_by_type(node_type, limit, since_ts?)` → newest-first
+  by `created_at`. (Conversation nodes already carry `created_at`.)
+- **Agentic tool**: `recent_conversations(limit=20, since?)` →
+  `[{q,a,created_at}, …]`, newest first. Exposed in `agentic_tools_all`.
+- **System-prompt hint**: for "recent / lately / this week / summarize our
+  conversations," prefer `recent_conversations` over `search_memory`.
+
+This directly answers "summarize my recent conversations" — fetch last N, hand
+the model the actual turns, let it cluster themes. No embeddings required.
+
+### 2. Stable per-session threading
+Today each turn is an independent `chat:<ts>` node; there's no session grouping.
+Add `session_id` + a monotonic turn index to the persisted content (the UI already
+sends `session_id`). Enables "summarize *this* conversation" and per-session recall,
+and lets retrieval return coherent threads instead of loose turns.
+
+### 3. Semantic retrieval (the real fix for thematic queries)
+Lexical BM25 can't do "themes." Options, in order of effort:
+- **a.** Embeddings on Conversation nodes + a vector search tool
+  (`semantic_search`). Biggest lift; also fixes knowledge recall broadly.
+- **b.** Interim: a two-pass "map-reduce" — `recent_conversations` to pull the
+  window, then let the model cluster. Cheap, ships with (1), no infra.
+
+Recommend **(1) + (2) now, (3b) as the interim thematic answer, (3a) as the
+roadmap item** once embeddings land (this dovetails with the GraphRAG/embedding
+work already noted in memory: substring 1.7% P@5 vs BM25 55% vs graph 21.7%).
+
+## Open questions for Will
+1. ~~Does the agentic path persist turns?~~ **Resolved: yes** — the dispatcher
+   calls `auto_persist` after both the agentic and non-agentic branches
+   (`routes.el` lines 156/298). Both paths store per-turn nodes.
+2. `conv:history` is accumulating duplicate overwriting nodes (saw several in the
+   live engram) — intended, or should it truly overwrite/dedupe?
+3. Is there appetite for the `engram_recent_by_type` primitive in the runtime, or
+   should recency be done in `.el` by scanning + sorting (fine at 59 nodes, weak
+   at scale)?
+4. Embeddings (3a): on the roadmap timeline, or defer and ship (1)+(2)+(3b)?
+
+## Not in scope
+Persistence itself (it works), and the separate **confabulation** fix (model
+faking tool calls in Just-chat mode) — that's `neuron` PR #29.