el/lang/runtime/engram_store.h

/* engram_store.h — M1 of the engram tiered storage engine.
 *
 * The FINAL on-disk paged store format: superblock (+ mirror), slotted pages,
 * self-describing TLV records, overflow chains, and two B+-tree indexes
 * (primary id->loc, adjacency from_id/to_id->edge-locs) over a free-listed
 * page file. See docs/architecture/design/engram-tiered-storage-engine.md §2.
 *
 * This is a self-contained module (plain C, standard libs only). It defines its
 * own serializable views of a node/edge (StoreNode/StoreEdge) that mirror every
 * persisted field of EngramNode/EngramEdge in el_runtime.c. M3 maps between the
 * live runtime structs and these; M1 does not touch el_runtime.c.
 *
 * Format id: magic "ENGST01", format_version 1. This format is PERMANENT — the
 * TLV record scheme means new fields never force a migration.
 */
#ifndef ENGRAM_STORE_H
#define ENGRAM_STORE_H

#include <stddef.h>
#include <stdint.h>

/* Fixed for the life of a store; recorded in the superblock. */
#define STORE_PAGE_SIZE   16384u
#define STORE_MAGIC       "ENGST01"      /* 7 chars + NUL stored in an 8-byte field */
#define STORE_FORMAT_VERSION 1u

/* Ring-buffer length for ACT-R base-level access timestamps.
 * MUST equal ENGRAM_BLL_K in el_runtime.c (currently 10). Static-checked in .c. */
#define STORE_BLL_K 10

/* Page types (page header byte). */
enum {
    STORE_PT_NODE     = 1,
    STORE_PT_EDGE     = 2,
    STORE_PT_INDEX    = 3,
    STORE_PT_OVERFLOW = 4,
    STORE_PT_FREE     = 5
};

/* store_check flags. */
#define STORE_CHECK_CRC 1u

/* ── Serializable node view: every persisted EngramNode field ─────────────── */
typedef struct StoreNode {
    char*    id;
    char*    content;
    char*    node_type;
    char*    label;
    char*    tier;
    char*    tags;
    char*    metadata;
    double   salience;
    double   importance;
    double   confidence;
    double   temporal_decay_rate;
    int64_t  activation_count;
    int64_t  last_activated;
    int64_t  created_at;
    int64_t  updated_at;
    double   background_activation;
    double   working_memory_weight;
    int32_t  suppression_count;
    uint32_t layer_id;
    int64_t  access_ts[STORE_BLL_K];
    int32_t  access_head;
    int32_t  access_filled;
    double   wm_anchor;
    float*   emb;        /* owned; NULL if not embedded */
    int32_t  emb_dim;
    /* Forward-compat: raw bytes of any TLV fields the reader did not recognise,
     * concatenated verbatim ([tag][u32 len][bytes]...). Re-emitted on write so
     * an old reader never drops a newer writer's fields. */
    uint8_t* unknown;
    size_t   unknown_len;
    int      tombstoned; /* set by store_get_* if the located record is dead */
    /* hebb_elig / hebb_elig_ts are DELIBERATELY NOT persisted (see EngramNode). */
} StoreNode;

/* ── Serializable edge view: every persisted EngramEdge field ─────────────── */
typedef struct StoreEdge {
    char*    id;
    char*    from_id;
    char*    to_id;
    char*    relation;
    char*    metadata;
    double   weight;
    double   hebb;
    double   confidence;
    int64_t  created_at;
    int64_t  updated_at;
    int64_t  last_fired;
    int32_t  inhibitory;
    uint32_t layer_id;
    uint8_t* unknown;
    size_t   unknown_len;
    int      tombstoned;
} StoreEdge;

typedef struct EngramPagedStore EngramPagedStore;

/* Lifecycle. */
EngramPagedStore* store_create(const char* path);   /* fails if file exists */
EngramPagedStore* store_open(const char* path);     /* recovers via mirror SB */
int  store_close(EngramPagedStore* s);              /* syncs + frees */
int  store_sync(EngramPagedStore* s);               /* fsync + rewrite both superblocks */

/* Nodes. store_get_node returns 1 on hit (fills *out, caller store_node_free),
 * 0 if absent or tombstoned, <0 on error. */
int  store_put_node(EngramPagedStore* s, const StoreNode* n);
int  store_get_node(EngramPagedStore* s, const char* id, StoreNode* out);
int  store_tombstone(EngramPagedStore* s, const char* id);

/* Edges. *out is malloc'd (store_edges_free); *n set to count. */
int  store_put_edge(EngramPagedStore* s, const StoreEdge* e);
int  store_get_edges_from(EngramPagedStore* s, const char* from_id, StoreEdge** out, size_t* n);
int  store_get_edges_to(EngramPagedStore* s, const char* to_id, StoreEdge** out, size_t* n);

/* Integrity: verify every page's crc (and both superblocks). Returns the number
 * of corrupt pages (0 = clean), or <0 on I/O error. */
int  store_check(EngramPagedStore* s, unsigned flags);

/* Ownership helpers. */
void store_node_free(StoreNode* n);
void store_edge_free(StoreEdge* e);
void store_edges_free(StoreEdge* arr, size_t n);

/* Test-only hook (NOT a format property — B+-tree nodes are self-describing via
 * their stored key count). Caps entries/keys per index node to force splits on
 * small datasets. 0 = natural full-page fanout. */
void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max);

/* Introspection for tests/tools. */
uint64_t store_page_count(const EngramPagedStore* s);

/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
 *
 * The durable engram is `neuron.egm` (paged) fronted by `neuron.wal`
 * (append-only). A mutation is durable once its WAL record is fsync'd
 * (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
 * store only at a checkpoint, so the store file on disk always reflects a
 * consistent point (`last_checkpoint_lsn`) and the WAL owns everything since.
 * Recovery = open store, replay WAL forward, redo a record only where the target
 * record's home page LSN < record LSN (idempotent). JSON is ONLY an import
 * source / export artifact — never the ongoing store. */

typedef enum { ENGRAM_WAL_ALWAYS = 0, ENGRAM_WAL_GROUP = 1, ENGRAM_WAL_OFF = 2 } EngramWalSync;

/* Serializable layer-registry view (the `layers` array of the legacy snapshot). */
typedef struct StoreLayer {
    uint32_t layer_id;
    char*    name;
    uint32_t activation_priority;
    int32_t  suppressible;
    int32_t  transparent;
    int32_t  injectable;
    uint8_t* unknown;
    size_t   unknown_len;
    int      tombstoned;
} StoreLayer;

/* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the
 * store is absent but a legacy snapshot.json exists, it is imported ONCE into a
 * fresh store; thereafter the store is authoritative and JSON is never read again.
 * On open, the WAL is replayed to recover any post-checkpoint mutations. */
EngramPagedStore* engram_open(const char* data_dir);
int  engram_close(EngramPagedStore* s);       /* checkpoint + close */

/* Force a checkpoint: flush dirty pages → fsync store → advance checkpoint LSN →
 * reclaim the WAL prefix. Also threshold-triggered automatically on the write path. */
int  engram_checkpoint(EngramPagedStore* s);

/* WAL commit policy. engram_open honours env ENGRAM_WAL_SYNC=always|group|off. */
void engram_set_wal_sync(EngramPagedStore* s, EngramWalSync policy);

/* Layer registry. */
int  store_put_layer(EngramPagedStore* s, const StoreLayer* L);
int  store_get_layer(EngramPagedStore* s, uint32_t layer_id, StoreLayer* out);
int  store_del_layer(EngramPagedStore* s, uint32_t layer_id);
int  store_list_layers(EngramPagedStore* s, StoreLayer** out, size_t* n);
void store_layer_free(StoreLayer* L);
void store_layers_free(StoreLayer* arr, size_t n);

/* Edge lookup by id (for hebb updates + idempotency). 1 hit / 0 absent / <0 err. */
int  store_get_edge(EngramPagedStore* s, const char* id, StoreEdge* out);

/* HEBB batch: one WAL record updating hebb (+ last_fired) on a set of edges. */
typedef struct StoreHebbDelta { const char* edge_id; double hebb; int64_t last_fired; } StoreHebbDelta;
int  store_hebb_batch(EngramPagedStore* s, const StoreHebbDelta* d, size_t n);

/* Supersede: logs the (old,new) pair and tombstones old_id at the store; the new
 * node + `supersedes` edge are logged separately (neuron-layer immutability). */
int  store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id);

/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
int  store_forget(EngramPagedStore* s, const char* id);

/* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ──
 * Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a
 * borrowed view (the engine frees it after cb returns — the callback must copy
 * anything it keeps). De-duplicated by id (canonical latest-live per id, matching
 * point-read semantics). Returns the count emitted, or <0 on error. The engine
 * hands out StoreNode/StoreEdge only — it never sees a soul struct (design §10). */
typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx);
typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx);
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx);
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx);

/* Introspection / test hooks. */
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);

/* ── M4: demand-paging buffer pool (additive residency; on-disk format UNCHANGED) ──
 *
 * The write-back, no-steal cache of M2 becomes a bounded, demand-paged buffer
 * pool. A fixed frame budget (env ENGRAM_POOL_FRAMES; 0 = unlimited; default
 * large ⇒ whole store resident ⇒ identical to Phase 1) keeps only hot pages in
 * RAM; a page access that is not resident faults in from neuron.egm, and under
 * pressure a CLEAN, unpinned frame is evicted (LRU). Dirty frames are never
 * stolen (M2 no-steal / WAL durability), and superblocks + index root/interior
 * pages are auto-pinned. Prefetch (env ENGRAM_PREFETCH) reads ahead on scans. */

/* Pin / unpin an individual page (faults it in and keeps it resident until
 * unpinned). Pin a hot layer's pages (WM/core) as a set. Idempotent counts. */
int store_pin_page(EngramPagedStore* s, uint64_t page_id);
int store_unpin_page(EngramPagedStore* s, uint64_t page_id);
int store_pin_layer(EngramPagedStore* s, uint32_t layer);    /* returns #pages pinned */
int store_unpin_layer(EngramPagedStore* s, uint32_t layer);

/* Buffer-pool introspection. */
typedef struct StorePoolStats {
    size_t   cap;            /* frame budget (0 = unlimited) */
    size_t   resident;       /* frames currently resident */
    size_t   pinned;         /* frames that cannot be evicted (dirty/pinned/structural) */
    size_t   dirty;          /* dirty (un-checkpointed) frames */
    unsigned prefetch;       /* read-ahead window */
    uint64_t hits, misses;   /* page_read cache hits / demand faults */
    uint64_t evictions;      /* clean frames reclaimed */
    uint64_t prefetch_reads; /* pages brought in by read-ahead */
} StorePoolStats;
void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out);
int  store_pool_resident(const EngramPagedStore* s, uint64_t page_id);

/* Test hooks: set the frame budget / prefetch window at runtime (NOT format). */
void store__set_pool_frames(EngramPagedStore* s, size_t frames);
void store__set_prefetch(EngramPagedStore* s, unsigned window);

/* Crash-test hooks (writes only under a throwaway dir).
 *  store__crash          — abandon all RAM state without flush/fsync (power loss).
 *  store__flush_pages    — pwrite dirty pages to disk WITHOUT a checkpoint (steal).
 *  store__checkpoint_crashat — run checkpoint but stop (then power-loss) after
 *                          `phase` (0..4); phase<0 = full checkpoint. */
void store__crash(EngramPagedStore* s);
int  store__flush_pages(EngramPagedStore* s);
int  store__checkpoint_crashat(EngramPagedStore* s, int phase);

/* ── M5: online compaction + background checkpointer (additive; format UNCHANGED) ──
 *
 * COMPACTION reclaims the space held by DEAD records — tombstoned nodes/edges
 * (telemetry prune, forget), superseded ids, and the stale prior versions a
 * re-put/hebb-batch leaves behind — plus the overflow pages they orphaned. It
 * rewrites only the LIVE records (bit-exact) into a fresh, densely packed image
 * with fresh id + adjacency indexes, then commits the swap atomically, so the
 * .egm file physically SHRINKS and the freed pages are reclaimed. Crash-safe:
 * a crash at any instant recovers to either the pre- or the post-compaction
 * store, never a corrupt mix (atomic rename is the commit point). It cooperates
 * with the M4 pool (no-steal, pins) by building into a separate store whose own
 * pool honours ENGRAM_POOL_FRAMES, then INVALIDATING every frame of the live
 * pool so no stale frame survives for a relocated page.
 *
 * Requires a quiesce point: store_compact performs a checkpoint (or sync) at
 * entry, so it is called between mutations, not concurrently with one. */
int store_compact(EngramPagedStore* s);

/* Test hook: run compaction but stop (then power-loss) after `phase`:
 *   0 = after the entry checkpoint, before building  (→ recovers pre-compaction)
 *   1 = after building+fsync the new image, before rename (→ pre-compaction)
 *   2 = after the atomic rename, before reopening RAM state (→ post-compaction)
 *   phase<0 = full compaction. Frees `s` on a crash phase (like the checkpoint hook). */
int store__compact_crashat(EngramPagedStore* s, int phase);

/* BACKGROUND CHECKPOINTER policy. A checkpoint fires automatically on the write
 * path when ANY armed trigger trips, reclaiming the WAL prefix without an explicit
 * engram_checkpoint. 0 disables that trigger. Same checkpoint semantics as M2.
 *   ops         — mutations since last checkpoint (default 100000)
 *   dirty_pages — dirty (un-checkpointed) pool frames
 *   wal_bytes   — bytes appended to the WAL since it was last reclaimed
 *   interval_ms — wall-clock ms since the last checkpoint (checked on writes) */
void store_set_checkpoint_policy(EngramPagedStore* s, uint64_t ops,
                                 size_t dirty_pages, uint64_t wal_bytes,
                                 long long interval_ms);

/* Introspection: number of pages currently on the free-list. */
uint64_t store_free_page_count(const EngramPagedStore* s);

/* ── CCR §4 managed-memory layer (write-barrier + minor GC + observability) ─────
 *
 * All flag-gated at store open (default OFF ⇒ byte-for-byte legacy behaviour):
 *   ENGRAM_WRITE_BARRIER=1  arm the durable-content write-barrier: a store_put_node
 *       whose DURABLE fields (content/type/label/tier/tags/metadata/importance/
 *       confidence/decay/layer/emb) are byte-identical to the last persisted copy
 *       is SKIPPED entirely — no LSN, no WAL, no record, no tombstone. This kills
 *       the ~99.78% checkpoint full-walk garbage at the source (ephemeral
 *       activation/WM state is intentionally not re-persisted on think-only cycles).
 *   ENGRAM_GC=1  (a) node re-puts supersede prior copies (mark DEAD, as edges do)
 *       so stale versions become reclaimable, and (b) a MINOR GC runs at the head
 *       of every checkpoint, returning whole dead NODE/EDGE pages to the free list.
 * The MAJOR GC is the existing merge-safe store_compact (schedule on a dead-ratio
 * threshold from the soul). */

/* Run one minor-GC sweep now: reclaim whole dead NODE/EDGE pages to the free list.
 * Returns the number of pages reclaimed (>=0). Safe to call between mutations;
 * automatically invoked at each checkpoint when ENGRAM_GC is armed. */
int store_minor_gc(EngramPagedStore* s);

/* GC / cache observability census (CCR §4.4). Page tallies are point-in-time;
 * the *_writes / *_skips / *_runs / *_reclaimed counters are cumulative since open. */
typedef struct StoreGcStats {
    uint64_t node_pages, edge_pages, index_pages, overflow_pages, free_pages;
    uint64_t live_nodes, live_edges;   /* live slots on NODE / EDGE pages */
    uint64_t dead_slots;               /* superseded/tombstoned slots awaiting reclaim */
    uint64_t live_bytes, dead_bytes;   /* on-page record bytes, live vs dead */
    uint64_t durable_writes;           /* node puts that actually appended a record */
    uint64_t barrier_skips;            /* node puts skipped by the write-barrier */
    uint64_t minor_gc_runs;            /* minor-GC invocations */
    uint64_t pages_reclaimed;          /* whole pages returned to the free list by minor GC */
    int      barrier_on, gc_on;        /* which gates are armed */
} StoreGcStats;
void store_gc_stats(EngramPagedStore* s, StoreGcStats* out);

#endif /* ENGRAM_STORE_H */