Make engram deletes/updates/forgets immutable on the launch branch

The ship-soul builds from this branch, which has the bounded-persona floor (#93) but never received the tombstone/supersede immutability fix (that went to main; hotfix diverged before it). So the launch soul failed verify-soul-contract IMMUTABILITY on the delete/update/forget routes — they hard-removed engram nodes via engram_forget/mem_forget. Apply the same fix, mirroring the knowledge routes' supersede pattern: - node/update -> create new node + "supersedes" edge to the original, KEEP the original (no engram_forget). - node/delete, memory/delete, memory/forget, cultivate forget, and the autonomous awareness forget -> TOMBSTONE via the canonical mem_tombstone (memory.el): keep the node + its edges, record a Tombstone marker, hide from default bounded list reads (?include_deleted recovers). Never engram_forget. The MCP forget tool now routes to the tombstoning delete instead of faking a delete. Internal GC that genuinely removes transient nodes (awareness inbox-trigger consume, consolidation dedup, session-summary replace, telemetry pruning) still calls engram_forget directly and is unchanged. Regenerated dist/soul.c (single-TU) + per-module dist/{memory,awareness, neuron-api}.c from THIS branch's sources under a 3GB physical-RSS watchdog (peak ~32MB), built against the release el_runtime (v1.0.0-20260501). The bounded-persona floor is preserved — verified in the emitted C and the linked binary (BOUNDED PERSONA / SOUL_PERSONA_NAME strings present). verify-soul-contract.sh: GATE PASS — PRESENCE all 27 routes, IMMUTABILITY 5/5 KEPT (memory-update, memory-delete, node-update, node-delete, memory-forget).
2026-07-21 10:55:46 -05:00
parent 1442ce21a6
commit 6527988eb9
8 changed files with 877 additions and 558 deletions
@@ -104,6 +104,66 @@ fn api_not_persisted(id: String) -> String {
    return "{\"ok\":false,\"error\":\"write_not_persisted\",\"id\":\"" + id + "\"}"
 }

+// ── Immutability: tombstone instead of hard-delete ────────────────────────────
+//
+// Day-one rule: engram nodes are immutable. A "delete" must never engram_forget
+// (which frees the node and drops its incident edges). Instead we TOMBSTONE: the
+// original node and all its edges are KEPT and stay traversable; a small
+// Tombstone marker node records the deletion (content = target id, label
+// "tombstone:<id>"), wired to the target with a "tombstones" edge. Default
+// bounded list reads hide tombstoned nodes (memory_hide_tombstoned); internal
+// cognition and explicit ?include_deleted reads still see them.
+fn tombstone_node(id: String) -> String {
+    // Delegates to the canonical helper in memory.el (single source of truth).
+    return mem_tombstone(id)
+}
+
+// tombstoned_id_set — delimited "|id1|id2|" of every tombstoned target id.
+// Empty string when nothing is tombstoned (callers fast-path on that).
+fn tombstoned_id_set() -> String {
+    let markers: String = engram_scan_nodes_by_type_json("Tombstone", 5000, 0)
+    if str_eq(markers, "") || str_eq(markers, "[]") { return "" }
+    let n: Int = json_array_len(markers)
+    let acc: String = "|"
+    let i: Int = 0
+    while i < n {
+        let m: String = json_array_get(markers, i)
+        let tid: String = json_get(m, "content")
+        let acc = if str_eq(tid, "") { acc } else { acc + tid + "|" }
+        let i = i + 1
+    }
+    return acc
+}
+
+// memory_hide_tombstoned — drop tombstone markers and tombstoned nodes from a
+// scanned node array. BOUNDED use only (typed/paginated lists), NOT the full
+// graph scan: json_array_get is O(index), so a full pass is O(n^2). Safe for the
+// ~50-item memory list; a hard cap protects against a large limit. The full
+// /api/graph/nodes hide needs a runtime scan filter and is deferred (see PR).
+// ?include_deleted bypasses the filter (explicit traversal).
+fn memory_hide_tombstoned(raw: String, path: String) -> String {
+    if str_contains(path, "include_deleted") { return raw }
+    if str_eq(raw, "") || str_eq(raw, "[]") { return raw }
+    let dead: String = tombstoned_id_set()
+    if str_eq(dead, "") { return raw }
+    let n: Int = json_array_len(raw)
+    if n > 1000 { return raw }
+    let out: String = "["
+    let first: Bool = true
+    let i: Int = 0
+    while i < n {
+        let node: String = json_array_get(raw, i)
+        let nid: String = json_get(node, "id")
+        let ntype: String = json_get(node, "node_type")
+        let is_dead: Bool = !str_eq(nid, "") && str_contains(dead, "|" + nid + "|")
+        let keep: Bool = !str_eq(ntype, "Tombstone") && !is_dead
+        let out = if keep { if first { out + node } else { out + "," + node } } else { out }
+        let first = if keep { false } else { first }
+        let i = i + 1
+    }
+    return out + "]"
+}
+
 // ── Session ───────────────────────────────────────────────────────────────────

 // handle_api_begin_session — full context bootstrap.
@@ -191,25 +251,26 @@ fn handle_api_node_create(body: String) -> String {
    return "{\"id\":\"" + id + "\",\"ok\":true}"
 }

-// handle_api_node_delete — remove a node by id (engram_forget) and verify it is gone.
+// handle_api_node_delete — TOMBSTONE a node by id (immutable delete).
 // Backs /api/neuron/node/delete and the /api/neuron/memory/delete alias the UI calls.
+// The node and all its incident edges are KEPT; a Tombstone marker records the
+// deletion. Never engram_forget — engram nodes are immutable by design.
 fn handle_api_node_delete(body: String) -> String {
    let id: String = json_get(body, "id")
    if str_eq(id, "") { return api_err("id is required") }
-    // engram_forget removes the node + its incident edges from the live graph.
-    // Delete is NOT read-back-verified: engram_get_node_json can return a stale hit
-    // for a just-forgotten id because the id→index map is not rebuilt on forget.
-    // A stale hit would cause a false "delete_failed" on a successful deletion.
-    // This exception is correct: read-back-verify guards WRITES; for deletes,
-    // the graph endpoints (/api/graph/nodes) reflect the removal and are the source of truth.
-    engram_forget(id)
-    return "{\"ok\":true,\"id\":\"" + id + "\"}"
+    if is_protected_node(id) { return api_err_protected(id) }
+    let existing: String = engram_get_node_json(id)
+    if str_eq(existing, "{}") { return api_err("node not found: " + id) }
+    let marker: String = tombstone_node(id)
+    if str_eq(marker, "") { return api_err("tombstone failed: " + id) }
+    return "{\"ok\":true,\"id\":\"" + id + "\",\"tombstoned\":true}"
 }

 // handle_api_node_update — update a node's content/fields. There is no in-place
-// engram update builtin, so this recreates the node with merged fields and then
-// forgets the old one (only after the new node reads back). The id changes; the
-// response returns the new id and the replaced id so callers can re-point.
+// engram update builtin, so this creates a new node with merged fields and wires
+// a "supersedes" edge new->old. The original is KEPT (immutable); the id changes,
+// and the response returns the new id and the superseded id so callers re-point.
+// Mirrors handle_api_memory_update / evolve exactly. Never engram_forget.
 fn handle_api_node_update(body: String) -> String {
    let id: String = json_get(body, "id")
    if str_eq(id, "") { return api_err("id is required") }
@@ -240,8 +301,8 @@ fn handle_api_node_update(body: String) -> String {
        el_from_float(0.5), el_from_float(0.5), el_from_float(0.8),
        tier, tags)
    if !api_persisted(new_id) { return api_not_persisted(new_id) }
-    engram_forget(id)
-    return "{\"id\":\"" + new_id + "\",\"replaced\":\"" + id + "\",\"ok\":true}"
+    engram_connect(new_id, id, el_from_float(0.9), "supersedes")
+    return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + id + "\",\"ok\":true}"
 }

 // handle_api_recall — search or activate memory by query.
@@ -504,13 +565,15 @@ fn handle_api_link_entities(body: String) -> String {
    return "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + eff_relation + "\"}"
 }

-// handle_api_forget — delete a node by ID. Blocked for protected identity nodes.
+// handle_api_forget — TOMBSTONE a node by ID (immutable; mem_forget now
+// tombstones). The node + edges are kept and recoverable. Blocked for protected
+// identity nodes.
 fn handle_api_forget(body: String) -> String {
    let node_id: String = json_get(body, "id")
    if str_eq(node_id, "") { return api_err("id is required") }
    if is_protected_node(node_id) { return api_err_protected(node_id) }
    mem_forget(node_id)
-    return "{\"ok\":true,\"id\":\"" + node_id + "\"}"
+    return "{\"ok\":true,\"id\":\"" + node_id + "\",\"tombstoned\":true}"
 }

 // handle_api_evolve_memory — evolve a Memory node. Blocked for protected identity nodes.
@@ -541,10 +604,10 @@ fn handle_api_evolve_memory(body: String) -> String {
 }

 // handle_api_memory_delete — POST /api/neuron/memory/delete {"id":"..."}.
-// Hard delete: engram_forget (via mem_forget) removes the node and all
-// incident edges from the engram store, so no soft-delete fallback is
-// needed. Existence is checked first because engram_forget silently
-// no-ops on unknown ids — a bad id must return an error, not fake success.
+// Immutable delete: TOMBSTONE via tombstone_node — the node and all its incident
+// edges are KEPT and stay traversable; a Tombstone marker records the deletion
+// and default bounded list reads hide it. Never engram_forget. Existence is
+// checked first so a bad id errors rather than faking success.
 // Blocked for protected identity nodes, same as /memory/forget.
 fn handle_api_memory_delete(body: String) -> String {
    let node_id: String = json_get(body, "id")
@@ -552,8 +615,10 @@ fn handle_api_memory_delete(body: String) -> String {
    if is_protected_node(node_id) { return api_err_protected(node_id) }
    let existing: String = engram_get_node_json(node_id)
    if str_eq(existing, "{}") { return api_err("memory not found: " + node_id) }
-    mem_forget(node_id)
-    return "{\"ok\":true,\"id\":\"" + node_id + "\",\"deleted\":true}"
+    // Immutable delete: tombstone, never mem_forget/engram_forget. Node + edges KEPT.
+    let marker: String = tombstone_node(node_id)
+    if str_eq(marker, "") { return api_err("tombstone failed: " + node_id) }
+    return "{\"ok\":true,\"id\":\"" + node_id + "\",\"tombstoned\":true}"
 }

 // handle_api_memory_update — POST /api/neuron/memory/update {"id","content"}.
@@ -623,8 +688,9 @@ fn handle_api_cultivate(body: String) -> String {
    if str_eq(op, "forget") {
        let node_id: String = json_get(body, "id")
        if str_eq(node_id, "") { return api_err("id is required") }
+        // Immutable: mem_forget now tombstones (keep node + edges), never hard-delete.
        mem_forget(node_id)
-        return "{\"ok\":true,\"id\":\"" + node_id + "\",\"cultivated\":true}"
+        return "{\"ok\":true,\"id\":\"" + node_id + "\",\"tombstoned\":true,\"cultivated\":true}"
    }

    if str_eq(op, "link_entities") {
@@ -646,7 +712,10 @@ fn handle_api_cultivate(body: String) -> String {
 // handle_api_list_typed — list nodes by node_type.
 fn handle_api_list_typed(node_type: String, path: String, body: String) -> String {
    let limit: Int = api_query_int(path, "limit", 50)
-    return api_or_empty(engram_scan_nodes_by_type_json(node_type, limit, 0))
+    let raw: String = api_or_empty(engram_scan_nodes_by_type_json(node_type, limit, 0))
+    // Hide tombstoned nodes from the default (bounded) memory list.
+    // ?include_deleted=1 returns them for explicit traversal.
+    return memory_hide_tombstoned(raw, path)
 }

 // ── Consolidate ───────────────────────────────────────────────────────────────