engram tiered storage M2: WAL + checkpoint + crash recovery + legacy import

Write-back no-steal buffer pool makes the fsync'd WAL load-bearing (M1 was write-through). Logical WAL with record-granularity page-LSN redo idempotency. Checkpoint = flush dirty pages, fsync store, advance last_checkpoint_lsn, reclaim WAL prefix. One-time snapshot.json import only when store absent; JSON never read as the ongoing store thereafter. Gates: 33/33 M1 (no regression) + 36/36 M2 — replay parity, torn-tail fuzz (every byte offset), checkpoint-crash at all 5 phases, torn-page+WAL redo, legacy-import parity, hebb-survives-crash.
2026-08-11 23:00:20 -05:00
parent fa47b98d18
commit 8affb1d6e0
4 changed files with 1542 additions and 13 deletions
@@ -132,4 +132,79 @@ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max)
 /* Introspection for tests/tools. */
 uint64_t store_page_count(const EngramPagedStore* s);

+/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
+ *
+ * The durable engram is `engram.store` (paged) fronted by `engram.wal`
+ * (append-only). A mutation is durable once its WAL record is fsync'd
+ * (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
+ * store only at a checkpoint, so the store file on disk always reflects a
+ * consistent point (`last_checkpoint_lsn`) and the WAL owns everything since.
+ * Recovery = open store, replay WAL forward, redo a record only where the target
+ * record's home page LSN < record LSN (idempotent). JSON is ONLY an import
+ * source / export artifact — never the ongoing store. */
+
+typedef enum { ENGRAM_WAL_ALWAYS = 0, ENGRAM_WAL_GROUP = 1, ENGRAM_WAL_OFF = 2 } EngramWalSync;
+
+/* Serializable layer-registry view (the `layers` array of the legacy snapshot). */
+typedef struct StoreLayer {
+    uint32_t layer_id;
+    char*    name;
+    uint32_t activation_priority;
+    int32_t  suppressible;
+    int32_t  transparent;
+    int32_t  injectable;
+    uint8_t* unknown;
+    size_t   unknown_len;
+    int      tombstoned;
+} StoreLayer;
+
+/* Boot the durable engram in `data_dir` (holds engram.store + engram.wal). If the
+ * store is absent but a legacy snapshot.json exists, it is imported ONCE into a
+ * fresh store; thereafter the store is authoritative and JSON is never read again.
+ * On open, the WAL is replayed to recover any post-checkpoint mutations. */
+EngramPagedStore* engram_open(const char* data_dir);
+int  engram_close(EngramPagedStore* s);       /* checkpoint + close */
+
+/* Force a checkpoint: flush dirty pages → fsync store → advance checkpoint LSN →
+ * reclaim the WAL prefix. Also threshold-triggered automatically on the write path. */
+int  engram_checkpoint(EngramPagedStore* s);
+
+/* WAL commit policy. engram_open honours env ENGRAM_WAL_SYNC=always|group|off. */
+void engram_set_wal_sync(EngramPagedStore* s, EngramWalSync policy);
+
+/* Layer registry. */
+int  store_put_layer(EngramPagedStore* s, const StoreLayer* L);
+int  store_get_layer(EngramPagedStore* s, uint32_t layer_id, StoreLayer* out);
+int  store_del_layer(EngramPagedStore* s, uint32_t layer_id);
+int  store_list_layers(EngramPagedStore* s, StoreLayer** out, size_t* n);
+void store_layer_free(StoreLayer* L);
+void store_layers_free(StoreLayer* arr, size_t n);
+
+/* Edge lookup by id (for hebb updates + idempotency). 1 hit / 0 absent / <0 err. */
+int  store_get_edge(EngramPagedStore* s, const char* id, StoreEdge* out);
+
+/* HEBB batch: one WAL record updating hebb (+ last_fired) on a set of edges. */
+typedef struct StoreHebbDelta { const char* edge_id; double hebb; int64_t last_fired; } StoreHebbDelta;
+int  store_hebb_batch(EngramPagedStore* s, const StoreHebbDelta* d, size_t n);
+
+/* Supersede: logs the (old,new) pair and tombstones old_id at the store; the new
+ * node + `supersedes` edge are logged separately (neuron-layer immutability). */
+int  store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id);
+
+/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
+int  store_forget(EngramPagedStore* s, const char* id);
+
+/* Introspection / test hooks. */
+uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
+uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);
+
+/* Crash-test hooks (writes only under a throwaway dir).
+ *  store__crash          — abandon all RAM state without flush/fsync (power loss).
+ *  store__flush_pages    — pwrite dirty pages to disk WITHOUT a checkpoint (steal).
+ *  store__checkpoint_crashat — run checkpoint but stop (then power-loss) after
+ *                          `phase` (0..4); phase<0 = full checkpoint. */
+void store__crash(EngramPagedStore* s);
+int  store__flush_pages(EngramPagedStore* s);
+int  store__checkpoint_crashat(EngramPagedStore* s, int phase);
+
 #endif /* ENGRAM_STORE_H */