self-review 2026-07-18: fix soul SIGABRT double-free + engram route scoping sweep

1. engram_neighbors_json (release runtime): BFS frontier/visited strings were el_strdup'd (arena-tracked) but manually freed, so el_request_end() double-freed every one — SIGABRT in http_worker under load (2 prod crashes today via /api/neuron/session/begin and /api/neuron/graph; reproduced and verified fixed with ASAN). Introduced when porting from the dev runtime, which correctly uses plain strdup. Third instance of the arena-vs-manual-free class (after EngramNode 07-15 and idmap keys 07-16). 2. server.el: let-in-if scoping sweep — defaults assigned inside if-blocks never mutated the outer binding, so /api/search and /api/activate always ran with q="", created nodes got node_type=""/salience=0.0, edges got relation=""/weight=0.0, and save/load with no path hit engram_save(""). Rewritten to the let-if-else expression form. /api/activate now also rejects empty queries instead of wiping carried WM weights. 3. engram_activate: retrieval reinforcement (ACT-R base-level learning) — nodes promoted to WM that survive both capacity caps now get last_activated/activation_count updated, so frequently retrieved memories decay slower than abandoned ones. Scoped to promoted-only to avoid flattening dampening across BFS fan-out.
2026-07-18 08:48:04 -05:00
parent e3dabe3e08
commit ab6b52a0b4
3 changed files with 651 additions and 205 deletions
@@ -76,12 +76,21 @@ fn route_stats(method: String, path: String, body: String) -> String {
    engram_stats_json()
 }

+// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
+// inner scope only — it does NOT mutate the outer binding (documented with
+// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
+// that broken pattern, so defaults never applied: nodes were created with
+// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with
+// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and
+// save/load with no "path" hit engram_save(""). Rewritten to the
+// `let x = if cond { a } else { b }` expression form (the pattern the newer
+// routes route_emit_ise/route_capture_knowledge already use correctly).
 fn route_create_node(method: String, path: String, body: String) -> String {
    let content: String = json_get_string(body, "content")
-    let node_type: String = json_get_string(body, "node_type")
-    if str_eq(node_type, "") { let node_type = "Memory" }
-    let salience: Float = json_get_float(body, "salience")
-    if salience == 0.0 { let salience = 0.5 }
+    let nt_raw: String = json_get_string(body, "node_type")
+    let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
+    let sal_raw: Float = json_get_float(body, "salience")
+    let salience: Float = if sal_raw == 0.0 { 0.5 } else { sal_raw }
    let id: String = engram_node(content, node_type, salience)
    "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
 }
@@ -107,8 +116,8 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
 // runtime keeps in lockstep with the in-memory graph. Live against the
 // running graph, not a stale export.
 fn route_scan_edges(method: String, path: String, body: String) -> String {
-    let dir: String = env("ENGRAM_DATA_DIR")
-    if str_eq(dir, "") { let dir = "/tmp/engram" }
+    let dir_raw: String = env("ENGRAM_DATA_DIR")
+    let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
    let snap_path: String = dir + "/snapshot.json"
    engram_save(snap_path)
    let snap: String = fs_read(snap_path)
@@ -122,39 +131,32 @@ fn route_scan_edges(method: String, path: String, body: String) -> String {
 }

 fn route_search(method: String, path: String, body: String) -> String {
-    let q: String = ""
-    if str_eq(method, "GET") {
-        let q = query_param(path, "q")
-    } else {
-        let q = json_get_string(body, "query")
-    }
-    let limit: Int = query_int(path, "limit", 20)
-    if limit == 0 { let limit = json_get_int(body, "limit") }
-    if limit == 0 { let limit = 20 }
+    let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
+    let lim_url: Int = query_int(path, "limit", 0)
+    let lim_body: Int = json_get_int(body, "limit")
+    let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body }
+    let limit: Int = if lim_either > 0 { lim_either } else { 20 }
    return engram_search_json(q, limit)
 }

 fn route_activate(method: String, path: String, body: String) -> String {
-    let q: String = ""
-    let depth: Int = 3
-    if str_eq(method, "GET") {
-        let q = query_param(path, "q")
-        let depth = query_int(path, "depth", 3)
-    } else {
-        let q = json_get_string(body, "query")
-        let bd: Int = json_get_int(body, "depth")
-        if bd > 0 { let depth = bd }
-    }
+    let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
+    // Guard: engram_activate with an empty query matches zero seeds, which
+    // zeroes ALL carried working-memory weights (documented in awareness.el
+    // perceive()). Never let an empty activation through to wipe WM.
+    if str_eq(q, "") { return err_json("missing query") }
+    let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") }
+    let depth: Int = if d_raw > 0 { d_raw } else { 3 }
    return "{\"results\":" + engram_activate_json(q, depth) + "}"
 }

 fn route_create_edge(method: String, path: String, body: String) -> String {
    let from_id: String = json_get_string(body, "from_id")
    let to_id: String = json_get_string(body, "to_id")
-    let relation: String = json_get_string(body, "relation")
-    if str_eq(relation, "") { let relation = "associates" }
-    let weight: Float = json_get_float(body, "weight")
-    if weight == 0.0 { let weight = 0.5 }
+    let rel_raw: String = json_get_string(body, "relation")
+    let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
+    let w_raw: Float = json_get_float(body, "weight")
+    let weight: Float = if w_raw == 0.0 { 0.5 } else { w_raw }
    engram_connect(from_id, to_id, weight, relation)
    "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
 }
@@ -181,23 +183,19 @@ fn route_forget(method: String, path: String, body: String) -> String {
 }

 fn route_save(method: String, path: String, body: String) -> String {
-    let p: String = json_get_string(body, "path")
-    if str_eq(p, "") {
-        let dir: String = env("ENGRAM_DATA_DIR")
-        if str_eq(dir, "") { let dir = "/tmp/engram" }
-        let p = dir + "/snapshot.json"
-    }
+    let p_raw: String = json_get_string(body, "path")
+    let dir_raw: String = env("ENGRAM_DATA_DIR")
+    let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
+    let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
    engram_save(p)
    "{\"ok\":true,\"path\":\"" + p + "\"}"
 }

 fn route_load(method: String, path: String, body: String) -> String {
-    let p: String = json_get_string(body, "path")
-    if str_eq(p, "") {
-        let dir: String = env("ENGRAM_DATA_DIR")
-        if str_eq(dir, "") { let dir = "/tmp/engram" }
-        let p = dir + "/snapshot.json"
-    }
+    let p_raw: String = json_get_string(body, "path")
+    let dir_raw: String = env("ENGRAM_DATA_DIR")
+    let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
+    let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
    engram_load(p)
    ok_json()
 }
@@ -219,8 +217,8 @@ fn route_health(method: String, path: String, body: String) -> String {
 // (it skips nodes already present by ID). Auth-exempt: same-host internal call.
 // (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
 fn route_sync(method: String, path: String, body: String) -> String {
-    let dir: String = env("ENGRAM_DATA_DIR")
-    if str_eq(dir, "") { let dir = "/tmp/engram" }
+    let dir_raw: String = env("ENGRAM_DATA_DIR")
+    let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
    let snap_path: String = dir + "/snapshot.json"
    engram_save(snap_path)
    let snap: String = fs_read(snap_path)
@@ -241,10 +239,20 @@ fn route_sync(method: String, path: String, body: String) -> String {
 //
 // Salience/importance set to match engram_node_full ISE defaults used by the
 // in-process fallback path in awareness.el (salience=0.3, importance=0.3,
-// confidence=0.8, tier=Episodic). High temporal_decay_rate (1.617) — ISEs
-// are inherently transient; they should decay faster than structural knowledge.
+// confidence=0.8, tier=Episodic).
 // (2026-06-26 self-review: added this route after discovering ise_post was
 // silently failing — the soul posts here but the endpoint didn't exist.)
+//
+// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs
+// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full
+// hardcodes 0.0), and per-node decay only dampens activation anyway; it never
+// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522
+// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in
+// engram_activate, so the fix is retention, not decay: every insert calls
+// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that
+// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting
+// "session-start" labels and self_review events as durable history. At
+// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever.
 fn route_emit_ise(method: String, path: String, body: String) -> String {
    let content: String = json_get_string(body, "content")
    if str_eq(content, "") { return err_json("missing content") }
@@ -256,6 +264,63 @@ fn route_emit_ise(method: String, path: String, body: String) -> String {
        sal, imp, conf,
        "Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
    )
+    let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
+    let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
+    let pruned: Int = engram_prune_telemetry(ret_ms)
+    "{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
+}
+
+// ── Knowledge capture ─────────────────────────────────────────────────────────
+//
+// route_capture_knowledge — direct Knowledge-node capture over HTTP.
+//
+// Endpoint: POST /api/neuron/knowledge/capture   (auth required: "_auth" in body)
+// Body:     {"content": "...", "title": "...", "category": "...",
+//            "tier": "note|lesson|canonical", "tags": [...], "project": "...",
+//            "_auth": "<key>"}
+//
+// WHY (2026-07-15 self-review): the world-ingestor integrator was designed
+// against this endpoint (its MCP-unavailable fallback), but the route never
+// existed — every direct push 404'd, and because the auth gate ran before
+// routing, the failure surfaced as {"error":"unauthorized"} and was
+// misdiagnosed for two weeks while world knowledge silently dropped.
+// POST /api/nodes was no substitute: it discards label/tags/tier, which
+// makes captured knowledge invisible to tag-scoped search and curiosity.
+//
+// The incoming knowledge tier (note/lesson/canonical) is preserved as a
+// "tier:<x>" tag rather than mapped onto Engram's cognitive tiers — Knowledge
+// nodes land in Semantic (stable reference), and the epistemic tier stays
+// queryable without inventing a lossy mapping.
+fn route_capture_knowledge(method: String, path: String, body: String) -> String {
+    let content: String = json_get_string(body, "content")
+    if str_eq(content, "") { return err_json("missing content") }
+    let title: String = json_get_string(body, "title")
+    let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title }
+    let category_raw: String = json_get_string(body, "category")
+    let category: String = if str_eq(category_raw, "") { "other" } else { category_raw }
+    let ktier_raw: String = json_get_string(body, "tier")
+    let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw }
+    let project: String = json_get_string(body, "project")
+    let tags_raw: String = json_get_raw(body, "tags")
+    let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw }
+    // Merge category/tier/project markers into the tag array. Search matches
+    // against the tags string, so these make captures findable by facet.
+    let base_len: Int = str_len(tags_base)
+    let head: String = str_slice(tags_base, 0, base_len - 1)
+    let sep: String = if str_eq(head, "[") { "" } else { "," }
+    let safe_cat: String = str_replace(category, "\"", "'")
+    let safe_tier: String = str_replace(ktier, "\"", "'")
+    let safe_proj: String = str_replace(project, "\"", "'")
+    let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" }
+    let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]"
+    let sal: Float = 0.5
+    let imp: Float = 0.5
+    let conf: Float = 0.9
+    let id: String = engram_node_full(
+        content, "Knowledge", label,
+        sal, imp, conf,
+        "Semantic", tags
+    )
    "{\"ok\":true,\"id\":\"" + id + "\"}"
 }

@@ -295,6 +360,12 @@ fn handle_request(method: String, path: String, body: String) -> String {
        return err_json("unauthorized")
    }

+    // Knowledge capture (auth enforced above; the world-ingestor integrator
+    // and any headless session without MCP push knowledge through this)
+    if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") {
+        return route_capture_knowledge(method, path, body)
+    }
+
    // Stats
    if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
        return route_stats(method, path, body)
@@ -362,13 +433,13 @@ fn handle_request(method: String, path: String, body: String) -> String {

 // ── Entry ─────────────────────────────────────────────────────────────────────

-let bind_str: String = env("ENGRAM_BIND")
-if str_eq(bind_str, "") { let bind_str = ":8742" }
+let bind_raw: String = env("ENGRAM_BIND")
+let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
 let port: Int = parse_port(bind_str)

 // On startup, try to load any existing snapshot (best effort).
-let data_dir: String = env("ENGRAM_DATA_DIR")
-if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
+let data_dir_raw: String = env("ENGRAM_DATA_DIR")
+let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
 let snapshot_path: String = data_dir + "/snapshot.json"
 engram_load(snapshot_path)