From bb64a236ed9d2906d3bfaa41638848ea93897a26 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 12 Aug 2026 14:14:20 -0500 Subject: [PATCH] engram tiered storage: engram-service wiring + elc fold-hang fix + prune-store mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire paged store into the ENGRAM SERVICE (server.el, the authoritative durable owner): boot->engram_store_boot, persist_canonical->engram_store_checkpoint, gated by ENGRAM_STORE. - elc (lang/elc.c + src/parser.el + codegen.el + elc-combined.el): OOB guard in tok_kind/tok_value + parse_block progress backstop — fixes the pre-existing unbounded-memory fold hang on sessions.el. - engram_prune_telemetry mirrors ISE prune to the store (store_forget) so store live-count tracks resident and stale telemetry stays bounded. - Deployed live 2026-08-12: engram :8742 on neuron.egm+WAL, count reconciled 11552. --- engram/src/server.el | 72 ++++++++++++++------- lang/el-compiler/src/codegen.el | 63 ++++++++++++++++++- lang/el-compiler/src/parser.el | 24 +++++++ lang/elc-combined.el | 3 + lang/elc.c | 107 +++++++++++++++++++++++++++++++- lang/runtime/el_runtime.c | 13 ++++ lang/runtime/el_runtime.h | 7 +++ 7 files changed, 263 insertions(+), 26 deletions(-) diff --git a/engram/src/server.el b/engram/src/server.el index c9bc036..06d4d40 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -117,6 +117,17 @@ fn route_text_health(method: String, path: String, body: String) -> String { // save/load with no "path" hit engram_save(""). Rewritten to the // `let x = if cond { a } else { b }` expression form (the pattern the newer // routes route_emit_ise/route_capture_knowledge already use correctly). +// store_on — ENGRAM_STORE flag (tiered paged store as the durable owner). Matches +// engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off → +// every persistence path below is byte-for-byte the historical snapshot behavior. +fn store_on() -> Bool { + let v: String = env("ENGRAM_STORE") + if str_eq(v, "1") { return true } + if str_eq(v, "on") { return true } + if str_eq(v, "true") { return true } + return false +} + // persist_canonical — save the canonical snapshot after a durable write. // // WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ @@ -131,6 +142,14 @@ fn route_text_health(method: String, path: String, body: String) -> String { // tolerant, ~2/min — snapshotting the whole store per heartbeat is waste; // any durable write that follows persists the pruning too). fn persist_canonical() -> Int { + // ENGRAM_STORE: the paged store is the durable owner — a checkpoint flushes + // dirty pages behind a WAL-durable record (durable the moment the WAL fsyncs). + // This is the fix for the "restart reverted to a 17h-old snapshot" data loss: + // durable writes no longer depend on a full snapshot.json rewrite. Returns 1 + // on a successful checkpoint, 0 otherwise. Flag-off: unchanged (writes JSON). + if store_on() { + return engram_store_checkpoint() + } let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() // (2026-08-10 self-review) This returned a hardcoded 1, which made every @@ -798,29 +817,40 @@ let port: Int = parse_port(bind_str) // never /tmp; fail loud if HOME is unresolvable (engram_resolve_data_dir exits). let data_dir: String = engram_resolve_data_dir() let snapshot_path: String = data_dir + "/snapshot.json" -engram_load(snapshot_path) +// ENGRAM_STORE (tiered paged store — engram-tiered-storage-engine.md). When set, +// the durable owner is the paged store (neuron.egm + neuron.wal): engram_store_boot +// imports snapshot.json ONCE into a fresh neuron.egm, else replays the WAL and loads +// the store resident — snapshot.json is never read again as the ongoing store. This +// closes the "restart reverted to a 17h-old snapshot" data-loss window. Flag-off +// (default): byte-for-byte the historical snapshot + optional-WAL boot below. +if store_on() { + engram_store_boot(data_dir) + println("[engram] ENGRAM_STORE enabled — tiered paged store is the durable owner") +} else { + engram_load(snapshot_path) -// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy -// snapshot-only boot. When ON, the snapshot above is the compaction BASE and -// the WAL carries every mutation since; replay reconstructs state to the last -// CRC-valid record, then opens the WAL for appending. -if wal_on() { - let replayed: Int = engram_wal_boot(data_dir) - println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records") -} + // WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy + // snapshot-only boot. When ON, the snapshot above is the compaction BASE and + // the WAL carries every mutation since; replay reconstructs state to the last + // CRC-valid record, then opens the WAL for appending. + if wal_on() { + let replayed: Int = engram_wal_boot(data_dir) + println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records") + } -// 2026-07-21 self-review boot guard: if the snapshot file has content but the -// load produced 0 nodes, something is wrong (corrupt file / parse failure). -// Preserve the evidence and warn loudly — and since read routes no longer write -// the canonical path, a bad boot can no longer clobber the good snapshot. -let boot_snap: String = fs_read(snapshot_path) -if !str_eq(boot_snap, "") { - if engram_node_count() == 0 { - println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json") - fs_write(data_dir + "/snapshot.failed-load.json", boot_snap) - } else { - // Good load: keep a boot-time backup of the snapshot as loaded. - fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap) + // 2026-07-21 self-review boot guard: if the snapshot file has content but the + // load produced 0 nodes, something is wrong (corrupt file / parse failure). + // Preserve the evidence and warn loudly — and since read routes no longer write + // the canonical path, a bad boot can no longer clobber the good snapshot. + let boot_snap: String = fs_read(snapshot_path) + if !str_eq(boot_snap, "") { + if engram_node_count() == 0 { + println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json") + fs_write(data_dir + "/snapshot.failed-load.json", boot_snap) + } else { + // Good load: keep a boot-time backup of the snapshot as loaded. + fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap) + } } } diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 917fb31..f6290dd 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -1292,6 +1292,43 @@ fn next_if_id() -> String { native_int_to_str(n) } +// is_void_builtin — true for runtime builtins declared `void` in el_runtime.h. +// User `-> Void` functions are emitted as el_val_t (return 0) so they are safe +// to assign; only these C-level void builtins are not. +fn is_void_builtin(name: String) -> Bool { + if str_eq(name, "println") { return true } + if str_eq(name, "print") { return true } + if str_eq(name, "engram_strengthen") { return true } + if str_eq(name, "engram_forget") { return true } + if str_eq(name, "engram_connect") { return true } + if str_eq(name, "dharma_emit") { return true } + if str_eq(name, "dharma_strengthen") { return true } + if str_eq(name, "llm_register_tool") { return true } + if str_eq(name, "exit_program") { return true } + if str_eq(name, "http_serve") { return true } + if str_eq(name, "http_set_handler") { return true } + if str_eq(name, "http_serve_async") { return true } + if str_eq(name, "el_cgi_init") { return true } + if str_eq(name, "el_retain") { return true } + if str_eq(name, "el_release") { return true } + false +} + +// cg_expr_is_void — true if `val` is a direct call to a void builtin, so the +// if-expression arm must emit it as a bare statement rather than assigning its +// (nonexistent) value to the result var. +fn cg_expr_is_void(val: Map) -> Bool { + let vk: String = val["expr"] + if str_eq(vk, "Call") { + let f = val["func"] + let fk: String = f["expr"] + if str_eq(fk, "Ident") { + return is_void_builtin(f["name"]) + } + } + false +} + // Render a single arm of the if-as-expression: emit each statement-before-last // as a side-effecting expression, then assign the final Expr's value to the // result var. If the arm body is empty or its last stmt isn't an Expr, the @@ -1300,6 +1337,10 @@ fn cg_if_expr_arm(stmts: [Map], result_var: String) -> String { let n: Int = native_list_len(stmts) // Collect statement fragments into a list to avoid O(n-) string growth. let parts: [String] = native_list_empty() + // Track names already declared in this arm's C block. El permits `let x` + // to redeclare/rebind x in the same scope, but C forbids redeclaring the + // same name in one block: emit `el_val_t x = ...` first, `x = ...` after. + let declared: [String] = native_list_empty() let i = 0 while i < n { let s = native_list_get(stmts, i) @@ -1310,18 +1351,31 @@ fn cg_if_expr_arm(stmts: [Map], result_var: String) -> String { let name: String = s["name"] let val = s["value"] let val_c: String = cg_expr(val) - let parts = native_list_append(parts, "el_val_t " + name + " = " + val_c + "; ") + if list_contains(declared, name) { + let parts = native_list_append(parts, name + " = " + val_c + "; ") + } else { + let declared = native_list_append(declared, name) + let parts = native_list_append(parts, "el_val_t " + name + " = " + val_c + "; ") + } } else { if str_eq(sk, "Return") { let val = s["value"] let val_c: String = cg_expr(val) - let parts = native_list_append(parts, result_var + " = (" + val_c + "); ") + if cg_expr_is_void(val) { + let parts = native_list_append(parts, val_c + "; ") + } else { + let parts = native_list_append(parts, result_var + " = (" + val_c + "); ") + } } else { if str_eq(sk, "Expr") { let val = s["value"] let val_c: String = cg_expr(val) if is_last { - let parts = native_list_append(parts, result_var + " = (" + val_c + "); ") + if cg_expr_is_void(val) { + let parts = native_list_append(parts, val_c + "; ") + } else { + let parts = native_list_append(parts, result_var + " = (" + val_c + "); ") + } } else { let parts = native_list_append(parts, "(void)(" + val_c + "); ") } @@ -2669,6 +2723,9 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_activate") { return 2 } if str_eq(name, "engram_save") { return 1 } if str_eq(name, "engram_load") { return 1 } + if str_eq(name, "engram_store_boot") { return 1 } + if str_eq(name, "engram_store_checkpoint") { return 0 } + if str_eq(name, "engram_store_close") { return 0 } if str_eq(name, "engram_get_node_json") { return 1 } if str_eq(name, "engram_search_json") { return 2 } if str_eq(name, "engram_scan_nodes_json") { return 2 } diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index d93fcba..a064bad 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -49,6 +49,21 @@ fn tok_value(tokens: [Any], pos: Int) -> String { native_list_get(tokens, pos * 2 + 1) } +// parse_progress_fatal — robustness backstop. Called by the token-consuming +// driver loops when they detect they have iterated more times than there are +// tokens (impossible for a well-formed program, where every iteration consumes +// at least one token). Names the offending token and exits non-zero instead of +// looping forever / exhausting memory. +fn parse_progress_fatal(where: String, tokens: [Any], pos: Int) -> Void { + let k: String = tok_kind(tokens, pos) + let v: String = tok_value(tokens, pos) + println("elc: FATAL: parser made no forward progress in " + where + + " at token index " + native_int_to_str(pos) + " (kind=" + k + ")") + println("elc: likely a malformed construct near '" + v + + "' — e.g. an unterminated string or an unescaped double-quote inside a string literal (use \\\" ).") + exit(1) +} + fn expect(tokens: [Any], pos: Int, kind: String) -> Int { let k = tok_kind(tokens, pos) if k == kind { @@ -1212,7 +1227,16 @@ fn parse_block(tokens: [Any], pos: Int) -> Map { let p = expect(tokens, pos, "LBrace") let stmts: [Map] = native_list_empty() let running = true + // Runaway backstop: a block can hold at most (token count) statements, since + // every iteration consumes >= 1 token. If we exceed that, the cursor has run + // off the end without terminating (malformed input) -> fail fast, don't hang. + let blk_total: Int = native_list_len(tokens) / 2 + let blk_iters: Int = 0 while running { + let blk_iters = blk_iters + 1 + if blk_iters > blk_total + 8 { + parse_progress_fatal("parse_block", tokens, p) + } let k = tok_kind(tokens, p) if k == "RBrace" { let running = false diff --git a/lang/elc-combined.el b/lang/elc-combined.el index 3216641..295e062 100644 --- a/lang/elc-combined.el +++ b/lang/elc-combined.el @@ -3797,6 +3797,9 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_activate") { return 2 } if str_eq(name, "engram_save") { return 1 } if str_eq(name, "engram_load") { return 1 } + if str_eq(name, "engram_store_boot") { return 1 } + if str_eq(name, "engram_store_checkpoint") { return 0 } + if str_eq(name, "engram_store_close") { return 0 } if str_eq(name, "engram_get_node_json") { return 1 } if str_eq(name, "engram_search_json") { return 2 } if str_eq(name, "engram_scan_nodes_json") { return 2 } diff --git a/lang/elc.c b/lang/elc.c index 5104d13..865c5ec 100644 --- a/lang/elc.c +++ b/lang/elc.c @@ -1423,15 +1423,53 @@ el_val_t tok_at(el_val_t tokens, el_val_t pos) { } el_val_t tok_kind(el_val_t tokens, el_val_t pos) { + /* Out-of-range reads MUST report the Eof sentinel so every `== "Eof"` + termination guard in the parser fires. Without this, reading past the + trailing Eof token returns runtime null (native_list_get OOB -> 0), which + matches no delimiter, letting inner parse loops (parse_block, parse_binop) + append AST nodes forever on malformed input -> unbounded allocation -> OOM. */ + el_val_t n = (native_list_len(tokens) / 2); + if (pos < 0) { + return EL_STR("Eof"); + } + if (pos >= n) { + return EL_STR("Eof"); + } return native_list_get(tokens, (pos * 2)); return 0; } el_val_t tok_value(el_val_t tokens, el_val_t pos) { + el_val_t n = (native_list_len(tokens) / 2); + if (pos < 0) { + return EL_STR(""); + } + if (pos >= n) { + return EL_STR(""); + } return native_list_get(tokens, ((pos * 2) + 1)); return 0; } +/* parse_progress_fatal — robustness backstop. Called by the token-consuming + driver loops when they detect they have iterated more times than there are + tokens (an impossibility for a well-formed program, where every iteration + consumes at least one token). Names the offending token and exits non-zero + instead of looping forever / exhausting memory. */ +el_val_t parse_progress_fatal(el_val_t where, el_val_t tokens, el_val_t pos) { + el_val_t k = tok_kind(tokens, pos); + el_val_t v = tok_value(tokens, pos); + println(el_str_concat(el_str_concat(el_str_concat(el_str_concat( + EL_STR("elc: FATAL: parser made no forward progress in "), where), + EL_STR(" at token index ")), native_int_to_str(pos)), + el_str_concat(EL_STR(" (kind="), el_str_concat(k, EL_STR(")"))))); + println(el_str_concat(el_str_concat( + EL_STR("elc: likely a malformed construct near '"), v), + EL_STR("' — e.g. an unterminated string or an unescaped double-quote inside a string literal (use \\\" )."))); + exit(1); + return 0; +} + el_val_t expect(el_val_t tokens, el_val_t pos, el_val_t kind) { el_val_t k = tok_kind(tokens, pos); if (str_eq(k, kind)) { @@ -2689,7 +2727,16 @@ el_val_t parse_block(el_val_t tokens, el_val_t pos) { el_val_t p = expect(tokens, pos, EL_STR("LBrace")); el_val_t stmts = native_list_empty(); el_val_t running = 1; + /* Runaway backstop: a block can hold at most (token count) statements, since + every iteration consumes >= 1 token. If we exceed that, the cursor has run + off the end without terminating (malformed input) -> fail fast, don't hang. */ + el_val_t __blk_total = (native_list_len(tokens) / 2); + el_val_t __blk_iters = 0; while (running) { + __blk_iters = (__blk_iters + 1); + if (__blk_iters > (__blk_total + 8)) { + parse_progress_fatal(EL_STR("parse_block"), tokens, p); + } el_val_t k = tok_kind(tokens, p); if (str_eq(k, EL_STR("RBrace"))) { running = 0; @@ -4838,9 +4885,51 @@ el_val_t next_if_id(void) { return 0; } +/* is_void_builtin — true for runtime builtins declared `void` in el_runtime.h. + User `-> Void` functions are emitted as el_val_t (return 0) so they are safe + to assign; only these C-level void builtins are not. */ +el_val_t is_void_builtin(el_val_t name) { + if (str_eq(name, EL_STR("println"))) { return 1; } + if (str_eq(name, EL_STR("print"))) { return 1; } + if (str_eq(name, EL_STR("engram_strengthen"))) { return 1; } + if (str_eq(name, EL_STR("engram_forget"))) { return 1; } + if (str_eq(name, EL_STR("engram_connect"))) { return 1; } + if (str_eq(name, EL_STR("dharma_emit"))) { return 1; } + if (str_eq(name, EL_STR("dharma_strengthen"))) { return 1; } + if (str_eq(name, EL_STR("llm_register_tool"))) { return 1; } + if (str_eq(name, EL_STR("exit_program"))) { return 1; } + if (str_eq(name, EL_STR("http_serve"))) { return 1; } + if (str_eq(name, EL_STR("http_set_handler"))) { return 1; } + if (str_eq(name, EL_STR("http_serve_async"))) { return 1; } + if (str_eq(name, EL_STR("el_cgi_init"))) { return 1; } + if (str_eq(name, EL_STR("el_retain"))) { return 1; } + if (str_eq(name, EL_STR("el_release"))) { return 1; } + return 0; +} + +/* cg_expr_is_void — true if `val` is a direct call to a void builtin, so the + if-expression arm must emit it as a bare statement rather than assigning its + (nonexistent) value to the result var. */ +el_val_t cg_expr_is_void(el_val_t val) { + el_val_t vk = el_get_field(val, EL_STR("expr")); + if (str_eq(vk, EL_STR("Call"))) { + el_val_t f = el_get_field(val, EL_STR("func")); + el_val_t fk = el_get_field(f, EL_STR("expr")); + if (str_eq(fk, EL_STR("Ident"))) { + return is_void_builtin(el_get_field(f, EL_STR("name"))); + } + } + return 0; +} + el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) { el_val_t n = native_list_len(stmts); el_val_t parts = native_list_empty(); + /* Track names already declared in this arm's C block. El permits `let x` + to redeclare/rebind x in the same scope, but C forbids redeclaring the + same name in one block. Emit `el_val_t x = ...` the first time and a + plain `x = ...` reassignment thereafter (mirrors cg_stmt's `declared`). */ + el_val_t declared = native_list_empty(); el_val_t i = 0; while (i < n) { el_val_t s = native_list_get(stmts, i); @@ -4853,18 +4942,31 @@ el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) { el_val_t name = el_get_field(s, EL_STR("name")); el_val_t val = el_get_field(s, EL_STR("value")); el_val_t val_c = cg_expr(val); - parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("el_val_t "), name), EL_STR(" = ")), val_c), EL_STR("; "))); + if (list_contains(declared, name)) { + parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(name, EL_STR(" = ")), val_c), EL_STR("; "))); + } else { + declared = native_list_append(declared, name); + parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("el_val_t "), name), EL_STR(" = ")), val_c), EL_STR("; "))); + } } else { if (str_eq(sk, EL_STR("Return"))) { el_val_t val = el_get_field(s, EL_STR("value")); el_val_t val_c = cg_expr(val); - parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); "))); + if (cg_expr_is_void(val)) { + parts = native_list_append(parts, el_str_concat(val_c, EL_STR("; "))); + } else { + parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); "))); + } } else { if (str_eq(sk, EL_STR("Expr"))) { el_val_t val = el_get_field(s, EL_STR("value")); el_val_t val_c = cg_expr(val); if (is_last) { + if (cg_expr_is_void(val)) { + parts = native_list_append(parts, el_str_concat(val_c, EL_STR("; "))); + } else { parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); "))); + } } else { parts = native_list_append(parts, el_str_concat(el_str_concat(EL_STR("(void)("), val_c), EL_STR("); "))); } @@ -4883,6 +4985,7 @@ el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) { } el_val_t result = str_join(parts, EL_STR("")); el_release(parts); + el_release(declared); return result; return 0; } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index ed89f69..e3a2de5 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -8038,6 +8038,19 @@ el_val_t engram_prune_telemetry(el_val_t older_than_ms) { g->edge_count = ew; free(set); } + /* Mirror the telemetry prune into the durable paged store so its live + * count tracks the resident graph and stale ISE telemetry stays bounded + * in the store too. Without this, the resident graph GCs old ISE from RAM + * (node_count drops) while the store retains them (count diverges + the + * store re-accumulates the very telemetry bloat this prune was written to + * stop). Matches engram_forget's store_forget mirror. Done before the ids + * are freed below. */ + if (engram_store_enabled() && g_engram_store) { + for (int64_t i = 0; i < removed; i++) { + store_forget(g_engram_store, removed_ids[i]); + } + } + for (int64_t i = 0; i < removed; i++) free(removed_ids[i]); free(removed_ids); diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 2c07b31..0380327 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -605,6 +605,13 @@ el_val_t engram_edge_count(void); el_val_t engram_activate(el_val_t query, el_val_t depth); el_val_t engram_save(el_val_t path); el_val_t engram_load(el_val_t path); +/* Tiered paged-store entry points (ENGRAM_STORE=1). engram_store_boot opens the + * durable store (import-once / WAL-replay) and loads it resident; checkpoint pushes + * the resident graph's current field state (incl. learned hebb + activation-formed + * edges) through the WAL and flushes; close checkpoints + closes. No-ops when off. */ +el_val_t engram_store_boot(el_val_t data_dir); +el_val_t engram_store_checkpoint(void); +el_val_t engram_store_close(void); /* JSON-string accessors — return pre-serialized JSON so HTTP handlers * can pass results straight through without round-tripping ElList/ElMap