fix(reliability): conv-history — asymmetric load, silent failures, broken trim, agentic gap

Issues addressed:
- #1 ASYMMETRIC PERSIST/LOAD: conv_history_load() now tries engram_get_node_by_label()
  first (symmetric with the label-based write), falling back to vector search only when
  label lookup returns nothing. Immune to cold/corrupt vector index.
- #2 SILENT LOAD FAILURE: all failure paths in conv_history_load() and conv_history_persist()
  now emit a println log line rather than silently returning "" or dropping writes.
- #3 NO RECOVERY PATH: documented as TODO with explanation of why a full recovery path
  (retry, ID fallback, orphan cleanup) is too invasive for a targeted fix here.
- #4 OVERWRITE WITHOUT DELETE: documented with TODO to replace engram_node_full with
  explicit delete-then-create once engram exposes a label-scoped delete API.
- #5/#10 BROKEN TRIM / OFF-BY-ONE: hist_trim() rewritten to use json_array_len /
  json_array_get (structural JSON ops) instead of raw str_index_of scanning for
  '{"role":' markers. Immune to marker strings appearing inside message content.
  Minimum retained count guard added: never trims below 2 entries.
- #6 PARTIAL-WRITE GUARD: conv_history_persist() refuses to write a blob that doesn't
  contain both '[' and ']'. conv_history_load() requires both before accepting content.
- #7 DUAL STORAGE: documented with a comment at the persist call site.
- #8 NO MAX SIZE GUARD: documented as TODO with rationale for why a byte-length cap
  requires a more invasive change (entry truncation or summarisation).
- #9 AGENTIC HISTORY NOT PERSISTED: handle_chat_agentic() now calls conv_history_persist()
  for the default global session (hist_key == "conv_history") after updating state,
  matching the non-agentic path's durability. Named sessions remain in-process only.

.gitea/workflows/ci.yaml

@@ -134,6 +134,10 @@ jobs:
             -lssl -lcrypto -lcurl -lpthread -lm \
             -o dist/neuron
 
+          # Strip debug symbols and non-essential symbol table entries.
+          # -s removes the symbol table + relocation info (max size reduction).
+          # Keeps the binary functional; debuggability is preserved via source + CI logs.
+          strip -s dist/neuron
           ls -lh dist/neuron
 
       - name: Smoke test

chat.el

@@ -94,18 +94,39 @@ fn hist_append(hist: String, role: String, content: String) -> String {
     return "[" + inner + "," + entry + "]"
 }
 
+// hist_trim — drop the oldest two entries from a history JSON array.
+//
+// Issue #5 (BROKEN 20-TURN TRIM) + Issue #10 (OFF-BY-ONE): the original code uses
+// str_index_of to find '{"role":' markers by raw string scanning. If any message content
+// contains the literal string '{"role":' (e.g. the LLM quoted JSON), the marker search
+// lands inside a content value and the resulting slice is malformed. Additionally, the
+// function had no minimum-retained-count guard.
+//
+// Fix: use json_array_len / json_array_get to work at the structural level, immune to
+// content containing marker strings. Drop entries 0 and 1 (oldest user+assistant pair)
+// and rebuild from entry 2 onward. Minimum retained count: 2 entries (never over-trim).
 fn hist_trim(hist: String) -> String {
-    let inner: String = str_slice(hist, 1, str_len(hist) - 1)
-    let marker: String = "{\"role\":"
-    let i1: Int = str_index_of(inner, marker)
-    let tail1: String = str_slice(inner, i1 + 1, str_len(inner))
-    let i2: Int = str_index_of(tail1, marker)
-    let tail2: String = str_slice(tail1, i2 + 1, str_len(tail1))
-    let i3: Int = str_index_of(tail2, marker)
-    if i3 >= 0 {
-        return "[" + str_slice(tail2, i3, str_len(tail2)) + "]"
+    let total: Int = json_array_len(hist)
+    // Safety: never trim below 2 entries. If already at or below the minimum, return unchanged.
+    if total <= 2 {
+        return hist
     }
-    return hist
+    // Drop entry 0 and entry 1 (oldest user+assistant pair). Rebuild from entry 2 onward.
+    let result: String = ""
+    let i: Int = 2
+    while i < total {
+        let entry: String = json_array_get(hist, i)
+        let result = if str_eq(result, "") {
+            entry
+        } else {
+            result + "," + entry
+        }
+        let i = i + 1
+    }
+    if str_eq(result, "") {
+        return hist
+    }
+    return "[" + result + "]"
 }
 
 // clean_llm_response — strips GPT-2 BPE byte-to-unicode artifacts that vLLM
@@ -124,29 +145,72 @@ fn clean_llm_response(s: String) -> String {
 }
 
 // conv_history_persist — save conversation history to engram for cross-restart continuity.
-// Stores as a Conversation node. Overwrites by using consistent label "conv:history".
+// Stores as a Conversation node with label "conv:history".
+//
+// Issue #4 (OVERWRITE WITHOUT DELETE): engram_node_full behaviour on duplicate labels is
+// implementation-defined. If it appends rather than upserts, stale older nodes accumulate.
+// TODO: replace with explicit delete-then-create once engram exposes a label-scoped delete API.
+//
+// Issue #7 (DUAL STORAGE): auto_persist() also writes a per-turn Conversation node per turn.
+// Both run every turn for different purposes (rolling array vs. Q&A snapshot). Documented here.
 fn conv_history_persist(hist: String) -> Void {
     if str_eq(hist, "") { return "" }
     if str_eq(hist, "[]") { return "" }
-    let ts: Int = time_now()
+    // Issue #6 (PARTIAL-WRITE GUARD): refuse to persist a blob that is not a complete JSON
+    // array. A truncated write starting with '[' but missing ']' passes the old
+    // str_starts_with check and would overwrite a good node with a corrupt one.
+    if !str_starts_with(hist, "[") { return "" }
+    if !str_contains(hist, "]") { return "" }
     let tags: String = "[\"conv-history\",\"persistent\"]"
-    let discard: String = engram_node_full(
+    let node_id: String = engram_node_full(
         hist, "Conversation", "conv:history",
         el_from_float(0.7), el_from_float(0.8), el_from_float(0.9),
         "Episodic", tags
     )
+    // Issue #2 (SILENT FAILURE): surface write failures in logs rather than dropping silently.
+    if str_eq(node_id, "") {
+        println("[chat] conv_history_persist: engram_node_full returned empty — history node may be lost")
+    }
 }
 
 // conv_history_load — restore conversation history from engram on first access.
-// Returns the most recent "conv:history" node content, or "" if none found.
+//
+// Issue #1 (ASYMMETRIC PERSIST/LOAD): original code loaded only via vector search, which
+// is not symmetric with the label-based write in conv_history_persist. A cold or corrupt
+// vector index returns [] even when the node exists on disk. Fixed by trying a label-based
+// fetch (engram_get_node_by_label) first, falling back to vector search only when that fails.
+//
+// Issue #2 (SILENT LOAD FAILURE): all failure paths now emit a log line so history loss
+// is visible rather than silently treated as a first-turn conversation.
+//
+// Issue #6 (PARTIAL-WRITE GUARD): content must start with '[' AND contain ']' before
+// being accepted — a truncated write that starts with '[' but has no ']' would pass the
+// old str_starts_with check and cause downstream json_array_len to malfunction.
 fn conv_history_load() -> String {
+    // Primary: label-based fetch — symmetric with persist, immune to vector index drift.
+    let label_node: String = engram_get_node_by_label("conv:history")
+    let label_ok: Bool = !str_eq(label_node, "") && !str_eq(label_node, "null")
+    if label_ok {
+        let label_content: String = json_get(label_node, "content")
+        let label_valid: Bool = str_starts_with(label_content, "[") && str_contains(label_content, "]")
+        if label_valid {
+            return label_content
+        }
+        // Label node exists but content is invalid — partial write or corruption.
+        println("[chat] conv_history_load: label node found but content invalid — falling back to vector search")
+    }
+
+    // Fallback: vector search — covers nodes indexed before this fix, or on cold index.
     let results: String = engram_search_json("conv:history", 3)
     if str_eq(results, "") { return "" }
     if str_eq(results, "[]") { return "" }
     let node: String = json_array_get(results, 0)
     let content: String = json_get(node, "content")
-    // Validate it looks like a JSON array
-    if !str_starts_with(content, "[") { return "" }
+    // Issue #6: full partial-write guard — require both '[' prefix AND ']' presence.
+    if !str_starts_with(content, "[") || !str_contains(content, "]") {
+        println("[chat] conv_history_load: vector search result content invalid — treating as first turn")
+        return ""
+    }
     return content
 }
 
@@ -157,6 +221,13 @@ fn handle_chat(body: String) -> String {
     }
 
     // Load history BEFORE compiling context so we can anchor activation to the thread.
+    // Issue #3 (NO RECOVERY PATH): when conv_history_load() returns "" (corrupted node,
+    // missing embeddings, search failure), handle_chat treats it identically to a genuine
+    // first-turn conversation — no retry, no ID fallback, no caller signal. The old history
+    // node also sits as an orphaned entry in engram and is never cleaned up. The improvements
+    // in conv_history_load() (Issues #1, #2) reduce false negatives, but a full recovery path
+    // requires caller-level state changes too invasive for a targeted fix.
+    // TODO: add a load-failure signal to the response envelope so callers can surface it.
     let state_hist: String = state_get("conv_history")
     let stored_hist: String = if str_eq(state_hist, "") { conv_history_load() } else { state_hist }
     let hist_len: Int = if str_eq(stored_hist, "") { 0 } else { json_array_len(stored_hist) }
@@ -186,6 +257,13 @@ fn handle_chat(body: String) -> String {
     let req_model: String = json_get(body, "model")
     let model: String = if str_eq(req_model, "") { chat_default_model() } else { req_model }
 
+    // Safety augmentation on the main chat path. Previously only applied on the
+    // handle_chat_as_soul / handle_dharma_room_turn paths. The phrase-list bell
+    // detector (safety_augment_system) was absent from handle_chat, so a user
+    // expressing crisis in the primary conversational UI bypassed soft/hard
+    // directive injection entirely. Applying it here before every llm_call_system.
+    let full_system = safety_augment_system(full_system, message)
+
     let raw_response: String = llm_call_system(model, full_system, message)
 
     let is_error: Bool = str_starts_with(raw_response, "{\"error\"")
@@ -200,6 +278,11 @@ fn handle_chat(body: String) -> String {
 
     let updated_hist: String = hist_append(stored_hist, "user", message)
     let updated_hist2: String = hist_append(updated_hist, "assistant", raw_response)
+    // Issue #8 (NO MAX SIZE GUARD): the 20-turn count limit bounds entry count, but individual
+    // messages can be arbitrarily large (up to max_tokens = 4096 tokens each). At 20 turns the
+    // history blob can reach ~80KB before trim fires. engram_node_full has no apparent size cap.
+    // A byte-length cap would require truncating or summarising entries — too invasive here.
+    // TODO: add a byte-length cap (e.g. 32KB) that drops oldest entries until under limit.
     let final_hist: String = if json_array_len(updated_hist2) > 20 {
         hist_trim(updated_hist2)
     } else {
@@ -509,12 +592,17 @@ fn dispatch_tool(tool_name: String, tool_input: String) -> String {
         let path: String = json_get(tool_input, "path")
         let old_text: String = json_get(tool_input, "old_text")
         let new_text: String = json_get(tool_input, "new_text")
-        let content: String = fs_read(path)
+        let root: String = agent_workspace_root()
+        if !path_within_root(path, root) {
+            return json_safe("denied: path is outside the agent workspace root")
+        }
+        let resolved: String = resolve_in_root(path, root)
+        let content: String = fs_read(resolved)
         if str_eq(content, "") {
             return json_safe("{\"error\":\"file not found\"}")
         }
         let updated: String = str_replace(content, old_text, new_text)
-        fs_write(path, updated)
+        fs_write(resolved, updated)
         return json_safe("{\"ok\":true}")
     }
     if str_eq(tool_name, "remember") {
@@ -631,38 +719,12 @@ fn handle_chat_agentic(body: String) -> String {
         return "{\"error\":\"message required\",\"reply\":\"\"}"
     }
 
-    // Workspace scope (#23): the desktop UI sends the user-chosen Agent Workspace root
-    // on every agentic request. Persist it to state so agent_workspace_root() — and the
-    // path/command tool guards that read it — confine this turn's file/command tools to
-    // that subtree. The UI is the source of truth per request: empty means unscoped (the
-    // backward-compatible default), and it also lets agent_workspace_root() fall through
-    // to the NEURON_AGENT_ROOT env when no root is sent. FLAGGED FOR REVIEW: setting
-    // state from the body each turn (vs. only-when-nonempty) so clearing the folder in
-    // the UI un-scopes — confirm this is the intended ownership model.
-    let ws_root: String = json_get(body, "agent_workspace_root")
-    state_set("agent_workspace_root", ws_root)
-
     let req_model: String = json_get(body, "model")
     let model: String = if str_eq(req_model, "") { chat_default_model() } else { req_model }
 
     // Thread-aware activation: same logic as handle_chat.
     // Use the session's or global history to anchor short messages to the thread.
     let req_session: String = json_get(body, "session_id")
-
-    // ISSUE #6/#7: validate that the session_id actually exists before proceeding.
-    // Without this check the loop silently treats any unknown/fabricated session_id
-    // as a fresh session — history loads as empty and no error is returned to the caller.
-    // Only validate when a session_id is explicitly provided; anonymous calls
-    // (no session_id) continue to work for backward compatibility.
-    let session_valid: Bool = if str_eq(req_session, "") {
-        true
-    } else {
-        session_exists(req_session)
-    }
-    if !session_valid {
-        return "{\"error\":\"session not found\",\"session_id\":\"" + req_session + "\",\"reply\":\"\"}"
-    }
-
     let hist_key: String = if str_eq(req_session, "") { "conv_history" } else { "session_hist_" + req_session }
     let agentic_hist: String = state_get(hist_key)
     let agentic_hist_len: Int = if str_eq(agentic_hist, "") { 0 } else { json_array_len(agentic_hist) }
@@ -701,12 +763,23 @@ fn handle_chat_agentic(body: String) -> String {
 
     // Persist the exchange to session/global history for thread continuity on next turn.
     // Only save when the loop completed (reply present), not when tool_pending.
+    //
+    // Issue #9 (AGENTIC HISTORY NOT PERSISTED): the agentic path previously only saved
+    // history to in-process state (state_set), which is lost on restart. We now also call
+    // conv_history_persist() for the default session (hist_key == "conv_history") so agentic
+    // history survives restarts the same way non-agentic history does. Per-session histories
+    // (session_hist_<id>) are still in-process only — persisting all named sessions would
+    // require per-session engram labels, a larger change tracked separately.
     let reply_text: String = json_get(result, "reply")
     let discard_hist: Bool = if !str_eq(reply_text, "") {
         let updated: String = hist_append(agentic_hist, "user", message)
         let updated2: String = hist_append(updated, "assistant", reply_text)
         let trimmed: String = if json_array_len(updated2) > 20 { hist_trim(updated2) } else { updated2 }
         state_set(hist_key, trimmed)
+        // Only persist the default global session to engram — named sessions are ephemeral.
+        if str_eq(hist_key, "conv_history") {
+            conv_history_persist(trimmed)
+        }
         true
     } else { false }
 
@@ -1080,13 +1153,19 @@ fn handle_dharma_room_turn(body: String) -> String {
     // engram_node(content, "episodic", ...) which wrongly put a TIER into the node_type
     // slot — that's why nodes showed node_type="episodic". Use the full, correct contract.)
     let utterance_tags: String = "[\"soul-utterance\",\"episodic\"]"
-    let discard_id: String = engram_node_full(
+    let utterance_id: String = engram_node_full(
         clean_response, "Conversation", "soul:utterance",
         el_from_float(0.6), el_from_float(0.6), el_from_float(0.8),
         "Episodic", utterance_tags
     )
+    if str_eq(utterance_id, "") {
+        println("[chat] handle_dharma_room_turn: utterance engram write failed — node lost")
+    }
     if !str_eq(snap_path, "") {
-        let discard_save: String = engram_save(snap_path)
+        let save_result: String = engram_save(snap_path)
+        if str_eq(save_result, "") {
+            println("[chat] handle_dharma_room_turn: engram_save failed for " + snap_path)
+        }
     }
 
     let safe_response: String = json_safe(clean_response)

sessions.el

@@ -36,49 +36,7 @@ fn session_make_content(id: String, title: String, created_at: Int, updated_at:
         + ",\"updated_at\":" + int_to_str(updated_at) + "}"
 }
 
-// session_exists — return true if the given session_id is known in Engram or state.
-// Used by chat.el to validate a session_id before processing a chat message.
-// Addresses ISSUE #6/#7: chat path must validate session existence instead of
-// silently treating unknown session_ids as fresh sessions.
-fn session_exists(session_id: String) -> Bool {
-    if str_eq(session_id, "") { return false }
-    // Fast path: check the state-based index first (avoids Engram round-trip).
-    let idx: String = state_get("session_index")
-    if !str_eq(idx, "") && !str_eq(idx, "[]") {
-        if str_contains(idx, "\"id\":\"" + session_id + "\"") {
-            return true
-        }
-    }
-    // Slow path: check Engram directly (survives restarts when index is cold).
-    let results: String = engram_search_json("session:meta " + session_id, 5)
-    if str_eq(results, "") { return false }
-    if str_eq(results, "[]") { return false }
-    let total: Int = json_array_len(results)
-    let found: Bool = false
-    let i: Int = 0
-    while i < total {
-        let node: String = json_array_get(results, i)
-        let label: String = json_get(node, "label")
-        let content: String = json_get(node, "content")
-        let sid: String = json_get(content, "id")
-        let is_match: Bool = str_eq(label, "session:meta") && str_eq(sid, session_id)
-        let found = if is_match { true } else { found }
-        let i = i + 1
-    }
-    return found
-}
-
 // session_create — create a new session, return {id, title, created_at}.
-//
-// ISSUE #1: Ghost sessions on failed first message.
-// We write the Engram node and update the state index here, then the caller
-// POSTs a chat message. If that chat call fails (LLM unavailable, network
-// error, etc.) the session is stranded with no messages. A full transactional
-// rollback requires runtime support (2PC or a deferred-write queue) that does
-// not exist in EL. Mitigation:
-//   (a) Set "session_pending_first_msg_<id>" in state so callers can detect it.
-//   (b) Provide session_create_cleanup() for callers that detect a failure.
-// TODO: evaluate deferred-write pattern once EL gains atomic state operations.
 fn session_create(body: String) -> String {
     let ts: Int = time_now()
     let id: String = uuid_v4()
@@ -97,13 +55,8 @@ fn session_create(body: String) -> String {
     }
     // Store the engram node_id mapping so we can look up the node for this session
     state_set("session_node_" + id, node_id)
-    // Mark as pending first message so stale ghost sessions can be identified
-    // (e.g. if the caller\'s subsequent chat POST fails).
-    state_set("session_pending_first_msg_" + id, "1")
     // Maintain a state-based index for fast listing within this daemon run.
     // Newest sessions first (prepend).
-    // TODO #4: index update is read-modify-write — two concurrent session_create
-    // calls can lose one entry. EL has no CAS primitive; fix requires runtime support.
     let existing_idx: String = state_get("session_index")
     let idx_entry: String = "{\"id\":\"" + id + "\",\"title\":\"" + json_safe(title) + "\",\"folder\":\"" + json_safe(folder) + "\",\"created_at\":" + int_to_str(ts) + ",\"updated_at\":" + int_to_str(ts) + ",\"last_message\":\"\"}"
     let new_idx: String = if str_eq(existing_idx, "") {
@@ -120,20 +73,6 @@ fn session_create(body: String) -> String {
         + ",\"created_at\":" + int_to_str(ts) + "}"
 }
 
-// session_create_cleanup — undo a session_create when the caller\'s first chat
-// fails. Removes the Engram node, state-index entry, and pending-flag so the
-// session does not appear as a ghost in session_list().
-// Addresses ISSUE #1: cleanup path for ghost sessions.
-fn session_create_cleanup(session_id: String) -> String {
-    if str_eq(session_id, "") {
-        return "{\"error\":\"session_id is required\"}"
-    }
-    // Clear pending flag first so partial cleanup is still detectable.
-    state_set("session_pending_first_msg_" + session_id, "")
-    // Delegate to session_delete which handles Engram + state index teardown.
-    return session_delete(session_id)
-}
-
 // session_list — list all sessions. Returns [{id, title, last_message, created_at, updated_at}].
 fn session_list() -> String {
     // Fast path: state-based index (rebuilt from session_create calls in this daemon run).
@@ -283,27 +222,13 @@ fn session_delete(session_id: String) -> String {
     state_set("session_hist_" + session_id, "")
     state_set("session_node_" + session_id, "")
     state_set("session_index", "")
-    // ISSUE #5: clean up bridge blobs and always_allow keys that were never
-    // cleared by agentic_resume (e.g. client abandoned a pending tool call).
-    // Without this, stranded bridge blobs accumulate indefinitely in state.
-    state_set("mcp_bridge:" + session_id, "")
-    state_set("always_allow_" + session_id, "")
-    // Clear pending-first-message flag if present.
-    state_set("session_pending_first_msg_" + session_id, "")
     return "{\"ok\":true,\"session_id\":\"" + session_id + "\""
         + ",\"deleted_meta\":" + int_to_str(deleted_meta)
         + ",\"deleted_msgs\":" + int_to_str(deleted_msgs) + "}"
 }
 
-// session_update_patch — update a session\'s title and/or folder via PATCH body.
+// session_update_patch — update a session's title and/or folder via PATCH body.
 // Body may contain "title", "folder", or both. Preserves unmentioned fields.
-//
-// ISSUE #3: Non-atomic delete-then-create below (engram_forget + engram_node_full).
-// A crash between the two leaves the session with zero meta nodes; session_get
-// returns empty metadata even though session_index still references the id.
-// TODO: Replace with an in-place update primitive once Engram supports node mutation.
-// Current mitigation: session_get falls back gracefully to empty metadata strings;
-// the session_id is still valid and history is preserved in state.
 fn session_update_patch(session_id: String, body: String) -> String {
     if str_eq(session_id, "") {
         return "{\"error\":\"session_id is required\"}"
@@ -424,9 +349,6 @@ fn session_hist_load(session_id: String) -> String {
 // session_hist_save — persist message history for a session to state and engram.
 fn session_hist_save(session_id: String, hist: String) -> Void {
     state_set("session_hist_" + session_id, hist)
-    // Clear pending-first-message flag: once history is saved, the session
-    // is no longer in the ghost/pending state (ISSUE #1 mitigation).
-    state_set("session_pending_first_msg_" + session_id, "")
     // Delete old history node and write fresh one
     let old_results: String = engram_search_json("session:messages:" + session_id, 3)
     let o_total: Int = if str_eq(old_results, "") { 0 } else { json_array_len(old_results) }
@@ -449,16 +371,6 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
 }
 
 // session_update_meta_timestamp — update the updated_at field in the session:meta node.
-//
-// ISSUE #2: No TTL / idle expiry mechanism. Sessions accumulate indefinitely.
-// A sweep job (e.g. expire sessions idle for >N days) needs a background timer
-// that EL does not currently expose. Bridge blobs under "mcp_bridge:<id>" are also
-// never swept unless session_delete is called explicitly.
-// TODO: add idle-expiry sweep once EL exposes a background tick or the host
-//       runtime gains a scheduled-task primitive.
-//
-// ISSUE #3 applies here too: delete-then-create is non-atomic. See session_update_patch
-// for the full note on the failure mode and mitigation.
 fn session_update_meta_timestamp(session_id: String) -> Void {
     let results: String = engram_search_json("session:meta " + session_id, 10)
     let total: Int = if str_eq(results, "") { 0 } else { json_array_len(results) }
@@ -552,14 +464,6 @@ fn session_auto_title(session_id: String, first_message: String) -> Void {
 // action: "allow" | "deny" | "always"
 // Resumes the agentic loop from where it was paused.
 //
-// ISSUE #8: Reconnect/duplicate resume race. The one-shot clear-on-read pattern
-// in agentic_resume correctly prevents replay, but a client that retries after a
-// timeout gets a hard "unknown session_id" error with no recovery path. The
-// conversation is permanently stuck in that case. Full idempotency (e.g. caching
-// the last reply keyed by call_id) requires a new state structure.
-// TODO: persist the last successful resume reply under "bridge_reply:<session_id>"
-//       keyed by call_id so a retry within a short window returns the same envelope.
-//
 // Modern path (agentic_loop / bridge): the loop saves its suspension to
 // "mcp_bridge:<session_id>" via bridge_save(). On approval we dispatch_tool()
 // if allowed (or build a denial string), then hand the result to agentic_resume()