/* engram_store.h — M1 of the engram tiered storage engine. * * The FINAL on-disk paged store format: superblock (+ mirror), slotted pages, * self-describing TLV records, overflow chains, and two B+-tree indexes * (primary id->loc, adjacency from_id/to_id->edge-locs) over a free-listed * page file. See docs/architecture/design/engram-tiered-storage-engine.md §2. * * This is a self-contained module (plain C, standard libs only). It defines its * own serializable views of a node/edge (StoreNode/StoreEdge) that mirror every * persisted field of EngramNode/EngramEdge in el_runtime.c. M3 maps between the * live runtime structs and these; M1 does not touch el_runtime.c. * * Format id: magic "ENGST01", format_version 1. This format is PERMANENT — the * TLV record scheme means new fields never force a migration. */ #ifndef ENGRAM_STORE_H #define ENGRAM_STORE_H #include #include /* Fixed for the life of a store; recorded in the superblock. */ #define STORE_PAGE_SIZE 16384u #define STORE_MAGIC "ENGST01" /* 7 chars + NUL stored in an 8-byte field */ #define STORE_FORMAT_VERSION 1u /* Ring-buffer length for ACT-R base-level access timestamps. * MUST equal ENGRAM_BLL_K in el_runtime.c (currently 10). Static-checked in .c. */ #define STORE_BLL_K 10 /* Page types (page header byte). */ enum { STORE_PT_NODE = 1, STORE_PT_EDGE = 2, STORE_PT_INDEX = 3, STORE_PT_OVERFLOW = 4, STORE_PT_FREE = 5 }; /* store_check flags. */ #define STORE_CHECK_CRC 1u /* ── Serializable node view: every persisted EngramNode field ─────────────── */ typedef struct StoreNode { char* id; char* content; char* node_type; char* label; char* tier; char* tags; char* metadata; double salience; double importance; double confidence; double temporal_decay_rate; int64_t activation_count; int64_t last_activated; int64_t created_at; int64_t updated_at; double background_activation; double working_memory_weight; int32_t suppression_count; uint32_t layer_id; int64_t access_ts[STORE_BLL_K]; int32_t access_head; int32_t access_filled; double wm_anchor; float* emb; /* owned; NULL if not embedded */ int32_t emb_dim; /* Forward-compat: raw bytes of any TLV fields the reader did not recognise, * concatenated verbatim ([tag][u32 len][bytes]...). Re-emitted on write so * an old reader never drops a newer writer's fields. */ uint8_t* unknown; size_t unknown_len; int tombstoned; /* set by store_get_* if the located record is dead */ /* hebb_elig / hebb_elig_ts are DELIBERATELY NOT persisted (see EngramNode). */ } StoreNode; /* ── Serializable edge view: every persisted EngramEdge field ─────────────── */ typedef struct StoreEdge { char* id; char* from_id; char* to_id; char* relation; char* metadata; double weight; double hebb; double confidence; int64_t created_at; int64_t updated_at; int64_t last_fired; int32_t inhibitory; uint32_t layer_id; uint8_t* unknown; size_t unknown_len; int tombstoned; } StoreEdge; typedef struct EngramPagedStore EngramPagedStore; /* Lifecycle. */ EngramPagedStore* store_create(const char* path); /* fails if file exists */ EngramPagedStore* store_open(const char* path); /* recovers via mirror SB */ int store_close(EngramPagedStore* s); /* syncs + frees */ int store_sync(EngramPagedStore* s); /* fsync + rewrite both superblocks */ /* Nodes. store_get_node returns 1 on hit (fills *out, caller store_node_free), * 0 if absent or tombstoned, <0 on error. */ int store_put_node(EngramPagedStore* s, const StoreNode* n); int store_get_node(EngramPagedStore* s, const char* id, StoreNode* out); int store_tombstone(EngramPagedStore* s, const char* id); /* Edges. *out is malloc'd (store_edges_free); *n set to count. */ int store_put_edge(EngramPagedStore* s, const StoreEdge* e); int store_get_edges_from(EngramPagedStore* s, const char* from_id, StoreEdge** out, size_t* n); int store_get_edges_to(EngramPagedStore* s, const char* to_id, StoreEdge** out, size_t* n); /* Integrity: verify every page's crc (and both superblocks). Returns the number * of corrupt pages (0 = clean), or <0 on I/O error. */ int store_check(EngramPagedStore* s, unsigned flags); /* Ownership helpers. */ void store_node_free(StoreNode* n); void store_edge_free(StoreEdge* e); void store_edges_free(StoreEdge* arr, size_t n); /* Test-only hook (NOT a format property — B+-tree nodes are self-describing via * their stored key count). Caps entries/keys per index node to force splits on * small datasets. 0 = natural full-page fanout. */ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max); /* Introspection for tests/tools. */ uint64_t store_page_count(const EngramPagedStore* s); /* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ───────────── * * The durable engram is `neuron.egm` (paged) fronted by `neuron.wal` * (append-only). A mutation is durable once its WAL record is fsync'd * (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the * store only at a checkpoint, so the store file on disk always reflects a * consistent point (`last_checkpoint_lsn`) and the WAL owns everything since. * Recovery = open store, replay WAL forward, redo a record only where the target * record's home page LSN < record LSN (idempotent). JSON is ONLY an import * source / export artifact — never the ongoing store. */ typedef enum { ENGRAM_WAL_ALWAYS = 0, ENGRAM_WAL_GROUP = 1, ENGRAM_WAL_OFF = 2 } EngramWalSync; /* Serializable layer-registry view (the `layers` array of the legacy snapshot). */ typedef struct StoreLayer { uint32_t layer_id; char* name; uint32_t activation_priority; int32_t suppressible; int32_t transparent; int32_t injectable; uint8_t* unknown; size_t unknown_len; int tombstoned; } StoreLayer; /* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the * store is absent but a legacy snapshot.json exists, it is imported ONCE into a * fresh store; thereafter the store is authoritative and JSON is never read again. * On open, the WAL is replayed to recover any post-checkpoint mutations. */ EngramPagedStore* engram_open(const char* data_dir); int engram_close(EngramPagedStore* s); /* checkpoint + close */ /* Force a checkpoint: flush dirty pages → fsync store → advance checkpoint LSN → * reclaim the WAL prefix. Also threshold-triggered automatically on the write path. */ int engram_checkpoint(EngramPagedStore* s); /* WAL commit policy. engram_open honours env ENGRAM_WAL_SYNC=always|group|off. */ void engram_set_wal_sync(EngramPagedStore* s, EngramWalSync policy); /* Layer registry. */ int store_put_layer(EngramPagedStore* s, const StoreLayer* L); int store_get_layer(EngramPagedStore* s, uint32_t layer_id, StoreLayer* out); int store_del_layer(EngramPagedStore* s, uint32_t layer_id); int store_list_layers(EngramPagedStore* s, StoreLayer** out, size_t* n); void store_layer_free(StoreLayer* L); void store_layers_free(StoreLayer* arr, size_t n); /* Edge lookup by id (for hebb updates + idempotency). 1 hit / 0 absent / <0 err. */ int store_get_edge(EngramPagedStore* s, const char* id, StoreEdge* out); /* HEBB batch: one WAL record updating hebb (+ last_fired) on a set of edges. */ typedef struct StoreHebbDelta { const char* edge_id; double hebb; int64_t last_fired; } StoreHebbDelta; int store_hebb_batch(EngramPagedStore* s, const StoreHebbDelta* d, size_t n); /* Supersede: logs the (old,new) pair and tombstones old_id at the store; the new * node + `supersedes` edge are logged separately (neuron-layer immutability). */ int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id); /* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */ int store_forget(EngramPagedStore* s, const char* id); /* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ── * Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a * borrowed view (the engine frees it after cb returns — the callback must copy * anything it keeps). De-duplicated by id (canonical latest-live per id, matching * point-read semantics). Returns the count emitted, or <0 on error. The engine * hands out StoreNode/StoreEdge only — it never sees a soul struct (design §10). */ typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx); typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx); int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx); int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx); /* Introspection / test hooks. */ uint64_t engram_wal_next_lsn(const EngramPagedStore* s); uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s); /* ── M4: demand-paging buffer pool (additive residency; on-disk format UNCHANGED) ── * * The write-back, no-steal cache of M2 becomes a bounded, demand-paged buffer * pool. A fixed frame budget (env ENGRAM_POOL_FRAMES; 0 = unlimited; default * large ⇒ whole store resident ⇒ identical to Phase 1) keeps only hot pages in * RAM; a page access that is not resident faults in from neuron.egm, and under * pressure a CLEAN, unpinned frame is evicted (LRU). Dirty frames are never * stolen (M2 no-steal / WAL durability), and superblocks + index root/interior * pages are auto-pinned. Prefetch (env ENGRAM_PREFETCH) reads ahead on scans. */ /* Pin / unpin an individual page (faults it in and keeps it resident until * unpinned). Pin a hot layer's pages (WM/core) as a set. Idempotent counts. */ int store_pin_page(EngramPagedStore* s, uint64_t page_id); int store_unpin_page(EngramPagedStore* s, uint64_t page_id); int store_pin_layer(EngramPagedStore* s, uint32_t layer); /* returns #pages pinned */ int store_unpin_layer(EngramPagedStore* s, uint32_t layer); /* Buffer-pool introspection. */ typedef struct StorePoolStats { size_t cap; /* frame budget (0 = unlimited) */ size_t resident; /* frames currently resident */ size_t pinned; /* frames that cannot be evicted (dirty/pinned/structural) */ size_t dirty; /* dirty (un-checkpointed) frames */ unsigned prefetch; /* read-ahead window */ uint64_t hits, misses; /* page_read cache hits / demand faults */ uint64_t evictions; /* clean frames reclaimed */ uint64_t prefetch_reads; /* pages brought in by read-ahead */ } StorePoolStats; void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out); int store_pool_resident(const EngramPagedStore* s, uint64_t page_id); /* Test hooks: set the frame budget / prefetch window at runtime (NOT format). */ void store__set_pool_frames(EngramPagedStore* s, size_t frames); void store__set_prefetch(EngramPagedStore* s, unsigned window); /* Crash-test hooks (writes only under a throwaway dir). * store__crash — abandon all RAM state without flush/fsync (power loss). * store__flush_pages — pwrite dirty pages to disk WITHOUT a checkpoint (steal). * store__checkpoint_crashat — run checkpoint but stop (then power-loss) after * `phase` (0..4); phase<0 = full checkpoint. */ void store__crash(EngramPagedStore* s); int store__flush_pages(EngramPagedStore* s); int store__checkpoint_crashat(EngramPagedStore* s, int phase); /* ── M5: online compaction + background checkpointer (additive; format UNCHANGED) ── * * COMPACTION reclaims the space held by DEAD records — tombstoned nodes/edges * (telemetry prune, forget), superseded ids, and the stale prior versions a * re-put/hebb-batch leaves behind — plus the overflow pages they orphaned. It * rewrites only the LIVE records (bit-exact) into a fresh, densely packed image * with fresh id + adjacency indexes, then commits the swap atomically, so the * .egm file physically SHRINKS and the freed pages are reclaimed. Crash-safe: * a crash at any instant recovers to either the pre- or the post-compaction * store, never a corrupt mix (atomic rename is the commit point). It cooperates * with the M4 pool (no-steal, pins) by building into a separate store whose own * pool honours ENGRAM_POOL_FRAMES, then INVALIDATING every frame of the live * pool so no stale frame survives for a relocated page. * * Requires a quiesce point: store_compact performs a checkpoint (or sync) at * entry, so it is called between mutations, not concurrently with one. */ int store_compact(EngramPagedStore* s); /* Test hook: run compaction but stop (then power-loss) after `phase`: * 0 = after the entry checkpoint, before building (→ recovers pre-compaction) * 1 = after building+fsync the new image, before rename (→ pre-compaction) * 2 = after the atomic rename, before reopening RAM state (→ post-compaction) * phase<0 = full compaction. Frees `s` on a crash phase (like the checkpoint hook). */ int store__compact_crashat(EngramPagedStore* s, int phase); /* BACKGROUND CHECKPOINTER policy. A checkpoint fires automatically on the write * path when ANY armed trigger trips, reclaiming the WAL prefix without an explicit * engram_checkpoint. 0 disables that trigger. Same checkpoint semantics as M2. * ops — mutations since last checkpoint (default 100000) * dirty_pages — dirty (un-checkpointed) pool frames * wal_bytes — bytes appended to the WAL since it was last reclaimed * interval_ms — wall-clock ms since the last checkpoint (checked on writes) */ void store_set_checkpoint_policy(EngramPagedStore* s, uint64_t ops, size_t dirty_pages, uint64_t wal_bytes, long long interval_ms); /* Introspection: number of pages currently on the free-list. */ uint64_t store_free_page_count(const EngramPagedStore* s); #endif /* ENGRAM_STORE_H */