engram: WAL persistence + integrity hardening + single canonical runtime
El SDK CI - dev / build-and-test (pull_request) Failing after 13m17s

Establish lang/runtime/ as the ONE canonical el runtime (from the active
runtime that carries hebb/emb persistence + the new WAL); repoint the el CI
publish, engram build, elb default, and in-repo build scripts to it; delete
the el-compiler/runtime + lang/releases/ forks; add scripts/check-single-runtime.sh
drift guard.

Fixes a live prod bug: the el CI published el-runtime-c/-h from the LAGGING
el-compiler fork (0 hebb refs), so the shipped soul never persisted Hebbian
edge weights — learned co-activation was wiped on every restart. Publishing
from canonical ships the stranded 'learning that cannot outlive the process'
fix.

WAL storage engine + integrity fixes (DELETE->tombstone + store-layer
protection, safe data-dir default) ride in behind ENGRAM_WAL (default off =
byte-identical to today). Verified: engram elb per-module build clean, WAL
gate 66/66, native smoke ok, drift-guard green.
This commit is contained in:
2026-08-11 21:31:10 -05:00
parent 791b0880b7
commit 0a72fced28
57 changed files with 1481 additions and 24485 deletions
+108 -17
View File
@@ -132,7 +132,7 @@ fn route_text_health(method: String, path: String, body: String) -> String {
// any durable write that follows persists the pruning too).
fn persist_canonical() -> Int {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
let dir: String = engram_resolve_data_dir()
// (2026-08-10 self-review) This returned a hardcoded 1, which made every
// caller's `let saved: Int = persist_canonical()` a dead variable six
// durable write paths each believed they had confirmation of a successful
@@ -140,6 +140,57 @@ fn persist_canonical() -> Int {
return engram_save(dir + "/snapshot.json")
}
// WAL persistence (design doc §§3-14; gated behind ENGRAM_WAL=on) ──────────
// Default OFF every persist path below is byte-identical to the historical
// per-write full-snapshot behavior. When ON, structural mutations append O(1)
// WAL records instead of rewriting the whole graph, with threshold compaction.
fn wal_on() -> Bool {
str_eq(env("ENGRAM_WAL"), "on")
}
// Persist a single-node mutation (create / content-evolve / strengthen).
fn persist_node(id: String) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_node_put(d, id)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Persist edges appended at index >= start (covers single-edge and batch).
fn persist_edges_since(start: Int) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_edges_since(d, start)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Persist a Hebbian consolidation batch as ONE WAL record (single fsync, §5-B).
fn persist_hebb_batch(start: Int) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_hebb_batch(d, start)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Bulk mutation (embedding backfill, load-merge): write a fresh compaction base
// so the many-node change is durable in one atomic snapshot; WAL is truncated.
fn persist_bulk() -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
return engram_wal_compact(d)
}
return persist_canonical()
}
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
// label, importance, tier, and tags engram_node() defaults label to content
// and importance to 0.5, so every node created over HTTP lost its metadata.
@@ -181,7 +232,7 @@ fn route_create_node(method: String, path: String, body: String) -> String {
salience, importance, confidence,
tier, tags
)
let saved: Int = persist_canonical()
let saved: Int = persist_node(id)
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
}
@@ -208,7 +259,7 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
// clobbered the good snapshot. Read routes must never write the canonical path.)
fn route_scan_edges(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
let dir: String = engram_resolve_data_dir()
let snap_path: String = dir + "/.scan-export.json"
engram_save(snap_path)
let snap: String = fs_read(snap_path)
@@ -250,8 +301,9 @@ fn route_create_edge(method: String, path: String, body: String) -> String {
// (dormant association); only default when the key is absent.
let w_present: String = json_get_raw(body, "weight")
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(body, "weight") }
let ec0: Int = engram_edge_count()
engram_connect(from_id, to_id, weight, relation)
let saved: Int = persist_canonical()
let saved: Int = persist_edges_since(ec0)
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
}
@@ -276,6 +328,7 @@ fn route_create_edges_batch(method: String, path: String, body: String) -> Strin
if str_eq(arr, "") { return err_json("missing edges array") }
let n: Int = json_array_len(arr)
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
let ec0: Int = engram_edge_count()
let i: Int = 0
let accepted: Int = 0
let skipped: Int = 0
@@ -299,7 +352,7 @@ fn route_create_edges_batch(method: String, path: String, body: String) -> Strin
// Skip it when nothing was accepted: an all-malformed payload must not
// trigger a 60MB write.
if accepted > 0 {
let saved: Int = persist_canonical()
let saved: Int = persist_hebb_batch(ec0)
}
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
}
@@ -315,22 +368,50 @@ fn route_strengthen(method: String, path: String, body: String) -> String {
let id: String = json_get_string(body, "node_id")
if str_eq(id, "") { return err_json("missing node_id") }
engram_strengthen(id)
let saved: Int = persist_canonical()
let saved: Int = persist_node(id)
ok_json()
}
// route_forget DELETE /api/nodes/:id INTEGRITY HARDENED (design doc §18.1).
//
// Two invariants now enforced AT THE STORE (not one layer up in neuron-api.el,
// which a direct HTTP client could bypass):
// 1. Write-protection: protected identity/value nodes (derived from the self
// graph self root + values hub + their neighbors, §18.3) cannot be
// deleted over HTTP. Returns 403, node untouched.
// 2. No hard delete over the wire, ever: an ordinary delete creates a
// Tombstone marker node + `tombstones` edge and KEEPS the original node
// and its edges (recoverable), instead of the old destructive
// engram_forget() shift-delete. Raw engram_forget is now internal-GC only
// and no longer reachable from any HTTP route.
fn route_forget(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/nodes/")
if str_eq(id, "") { return err_json("missing id") }
engram_forget(id)
let saved: Int = persist_canonical()
ok_json()
if engram_is_protected(id) == 1 {
return "{\"__status__\":403,\"error\":\"protected node; deletion refused\",\"id\":\"" + id + "\"}"
}
let tomb_id: String = engram_node_full(
"tombstone:" + id, "Tombstone", "tombstone:" + id,
0.1, 0.1, 1.0, "Episodic", "[\"tombstone\"]"
)
let ec0: Int = engram_edge_count()
engram_connect(tomb_id, id, 1.0, "tombstones")
let saved: Int = if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_node_put(d, tomb_id)
let b: Int = engram_wal_edges_since(d, ec0)
let c: Int = engram_wal_maybe_compact(d)
a
} else {
persist_canonical()
}
"{\"ok\":true,\"tombstoned\":\"" + id + "\",\"tombstone_id\":\"" + tomb_id + "\"}"
}
fn route_save(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) engram_save returns 0 on an empty path and the
// route discarded it, so the response was a literal "ok":true regardless
@@ -346,7 +427,7 @@ fn route_save(method: String, path: String, body: String) -> String {
fn route_load(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) This was a stub response over the single most
// destructive operation in the server. engram_load returns 0 on an empty
@@ -398,7 +479,7 @@ fn route_embed_backfill(method: String, path: String, body: String) -> String {
let result: String = engram_embed_backfill(n)
let done: Float = json_get_float(result, "embedded")
if done > 0.0 {
let saved: Int = persist_canonical()
let saved: Int = persist_bulk()
}
return result
}
@@ -417,7 +498,7 @@ fn route_embed_backfill(method: String, path: String, body: String) -> String {
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
fn route_sync(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
let dir: String = engram_resolve_data_dir()
// 2026-07-21 self-review: export to a scratch path, never the canonical
// snapshot.json read routes must not be able to clobber the good snapshot.
let snap_path: String = dir + "/.sync-export.json"
@@ -451,7 +532,7 @@ fn route_load_merge(method: String, path: String, body: String) -> String {
engram_load_merge(p)
let added_n: Int = engram_node_count() - before_n
let added_e: Int = engram_edge_count() - before_e
let saved: Int = persist_canonical()
let saved: Int = persist_bulk()
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
}
@@ -550,7 +631,7 @@ fn route_capture_knowledge(method: String, path: String, body: String) -> String
sal, imp, conf,
"Semantic", tags
)
let saved: Int = persist_canonical()
let saved: Int = persist_node(id)
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
@@ -713,11 +794,21 @@ let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
let port: Int = parse_port(bind_str)
// On startup, try to load any existing snapshot (best effort).
let data_dir_raw: String = env("ENGRAM_DATA_DIR")
let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
// §18.2: resolve the data dir safely unset ENGRAM_DATA_DIR $HOME/.neuron/engram,
// never /tmp; fail loud if HOME is unresolvable (engram_resolve_data_dir exits).
let data_dir: String = engram_resolve_data_dir()
let snapshot_path: String = data_dir + "/snapshot.json"
engram_load(snapshot_path)
// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy
// snapshot-only boot. When ON, the snapshot above is the compaction BASE and
// the WAL carries every mutation since; replay reconstructs state to the last
// CRC-valid record, then opens the WAL for appending.
if wal_on() {
let replayed: Int = engram_wal_boot(data_dir)
println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records")
}
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
// Preserve the evidence and warn loudly and since read routes no longer write