engram tiered storage: engram-service wiring + elc fold-hang fix + prune-store mirror
- Wire paged store into the ENGRAM SERVICE (server.el, the authoritative durable owner): boot->engram_store_boot, persist_canonical->engram_store_checkpoint, gated by ENGRAM_STORE. - elc (lang/elc.c + src/parser.el + codegen.el + elc-combined.el): OOB guard in tok_kind/tok_value + parse_block progress backstop — fixes the pre-existing unbounded-memory fold hang on sessions.el. - engram_prune_telemetry mirrors ISE prune to the store (store_forget) so store live-count tracks resident and stale telemetry stays bounded. - Deployed live 2026-08-12: engram :8742 on neuron.egm+WAL, count reconciled 11552.
This commit is contained in:
+51
-21
@@ -117,6 +117,17 @@ fn route_text_health(method: String, path: String, body: String) -> String {
|
||||
// save/load with no "path" hit engram_save(""). Rewritten to the
|
||||
// `let x = if cond { a } else { b }` expression form (the pattern the newer
|
||||
// routes route_emit_ise/route_capture_knowledge already use correctly).
|
||||
// store_on — ENGRAM_STORE flag (tiered paged store as the durable owner). Matches
|
||||
// engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off →
|
||||
// every persistence path below is byte-for-byte the historical snapshot behavior.
|
||||
fn store_on() -> Bool {
|
||||
let v: String = env("ENGRAM_STORE")
|
||||
if str_eq(v, "1") { return true }
|
||||
if str_eq(v, "on") { return true }
|
||||
if str_eq(v, "true") { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
// persist_canonical — save the canonical snapshot after a durable write.
|
||||
//
|
||||
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
|
||||
@@ -131,6 +142,14 @@ fn route_text_health(method: String, path: String, body: String) -> String {
|
||||
// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste;
|
||||
// any durable write that follows persists the pruning too).
|
||||
fn persist_canonical() -> Int {
|
||||
// ENGRAM_STORE: the paged store is the durable owner — a checkpoint flushes
|
||||
// dirty pages behind a WAL-durable record (durable the moment the WAL fsyncs).
|
||||
// This is the fix for the "restart reverted to a 17h-old snapshot" data loss:
|
||||
// durable writes no longer depend on a full snapshot.json rewrite. Returns 1
|
||||
// on a successful checkpoint, 0 otherwise. Flag-off: unchanged (writes JSON).
|
||||
if store_on() {
|
||||
return engram_store_checkpoint()
|
||||
}
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
// (2026-08-10 self-review) This returned a hardcoded 1, which made every
|
||||
@@ -798,29 +817,40 @@ let port: Int = parse_port(bind_str)
|
||||
// never /tmp; fail loud if HOME is unresolvable (engram_resolve_data_dir exits).
|
||||
let data_dir: String = engram_resolve_data_dir()
|
||||
let snapshot_path: String = data_dir + "/snapshot.json"
|
||||
engram_load(snapshot_path)
|
||||
// ENGRAM_STORE (tiered paged store — engram-tiered-storage-engine.md). When set,
|
||||
// the durable owner is the paged store (neuron.egm + neuron.wal): engram_store_boot
|
||||
// imports snapshot.json ONCE into a fresh neuron.egm, else replays the WAL and loads
|
||||
// the store resident — snapshot.json is never read again as the ongoing store. This
|
||||
// closes the "restart reverted to a 17h-old snapshot" data-loss window. Flag-off
|
||||
// (default): byte-for-byte the historical snapshot + optional-WAL boot below.
|
||||
if store_on() {
|
||||
engram_store_boot(data_dir)
|
||||
println("[engram] ENGRAM_STORE enabled — tiered paged store is the durable owner")
|
||||
} else {
|
||||
engram_load(snapshot_path)
|
||||
|
||||
// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy
|
||||
// snapshot-only boot. When ON, the snapshot above is the compaction BASE and
|
||||
// the WAL carries every mutation since; replay reconstructs state to the last
|
||||
// CRC-valid record, then opens the WAL for appending.
|
||||
if wal_on() {
|
||||
let replayed: Int = engram_wal_boot(data_dir)
|
||||
println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records")
|
||||
}
|
||||
// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy
|
||||
// snapshot-only boot. When ON, the snapshot above is the compaction BASE and
|
||||
// the WAL carries every mutation since; replay reconstructs state to the last
|
||||
// CRC-valid record, then opens the WAL for appending.
|
||||
if wal_on() {
|
||||
let replayed: Int = engram_wal_boot(data_dir)
|
||||
println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records")
|
||||
}
|
||||
|
||||
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
||||
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
||||
// Preserve the evidence and warn loudly — and since read routes no longer write
|
||||
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
||||
let boot_snap: String = fs_read(snapshot_path)
|
||||
if !str_eq(boot_snap, "") {
|
||||
if engram_node_count() == 0 {
|
||||
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
||||
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
||||
} else {
|
||||
// Good load: keep a boot-time backup of the snapshot as loaded.
|
||||
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
||||
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
||||
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
||||
// Preserve the evidence and warn loudly — and since read routes no longer write
|
||||
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
||||
let boot_snap: String = fs_read(snapshot_path)
|
||||
if !str_eq(boot_snap, "") {
|
||||
if engram_node_count() == 0 {
|
||||
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
||||
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
||||
} else {
|
||||
// Good load: keep a boot-time backup of the snapshot as loaded.
|
||||
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user