will.anderson
8affb1d6e0
Write-back no-steal buffer pool makes the fsync'd WAL load-bearing (M1 was write-through). Logical WAL with record-granularity page-LSN redo idempotency. Checkpoint = flush dirty pages, fsync store, advance last_checkpoint_lsn, reclaim WAL prefix. One-time snapshot.json import only when store absent; JSON never read as the ongoing store thereafter. Gates: 33/33 M1 (no regression) + 36/36 M2 — replay parity, torn-tail fuzz (every byte offset), checkpoint-crash at all 5 phases, torn-page+WAL redo, legacy-import parity, hebb-survives-crash.
211 lines
8.8 KiB
C
211 lines
8.8 KiB
C
/* engram_store.h — M1 of the engram tiered storage engine.
|
|
*
|
|
* The FINAL on-disk paged store format: superblock (+ mirror), slotted pages,
|
|
* self-describing TLV records, overflow chains, and two B+-tree indexes
|
|
* (primary id->loc, adjacency from_id/to_id->edge-locs) over a free-listed
|
|
* page file. See docs/architecture/design/engram-tiered-storage-engine.md §2.
|
|
*
|
|
* This is a self-contained module (plain C, standard libs only). It defines its
|
|
* own serializable views of a node/edge (StoreNode/StoreEdge) that mirror every
|
|
* persisted field of EngramNode/EngramEdge in el_runtime.c. M3 maps between the
|
|
* live runtime structs and these; M1 does not touch el_runtime.c.
|
|
*
|
|
* Format id: magic "ENGST01", format_version 1. This format is PERMANENT — the
|
|
* TLV record scheme means new fields never force a migration.
|
|
*/
|
|
#ifndef ENGRAM_STORE_H
|
|
#define ENGRAM_STORE_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* Fixed for the life of a store; recorded in the superblock. */
|
|
#define STORE_PAGE_SIZE 16384u
|
|
#define STORE_MAGIC "ENGST01" /* 7 chars + NUL stored in an 8-byte field */
|
|
#define STORE_FORMAT_VERSION 1u
|
|
|
|
/* Ring-buffer length for ACT-R base-level access timestamps.
|
|
* MUST equal ENGRAM_BLL_K in el_runtime.c (currently 10). Static-checked in .c. */
|
|
#define STORE_BLL_K 10
|
|
|
|
/* Page types (page header byte). */
|
|
enum {
|
|
STORE_PT_NODE = 1,
|
|
STORE_PT_EDGE = 2,
|
|
STORE_PT_INDEX = 3,
|
|
STORE_PT_OVERFLOW = 4,
|
|
STORE_PT_FREE = 5
|
|
};
|
|
|
|
/* store_check flags. */
|
|
#define STORE_CHECK_CRC 1u
|
|
|
|
/* ── Serializable node view: every persisted EngramNode field ─────────────── */
|
|
typedef struct StoreNode {
|
|
char* id;
|
|
char* content;
|
|
char* node_type;
|
|
char* label;
|
|
char* tier;
|
|
char* tags;
|
|
char* metadata;
|
|
double salience;
|
|
double importance;
|
|
double confidence;
|
|
double temporal_decay_rate;
|
|
int64_t activation_count;
|
|
int64_t last_activated;
|
|
int64_t created_at;
|
|
int64_t updated_at;
|
|
double background_activation;
|
|
double working_memory_weight;
|
|
int32_t suppression_count;
|
|
uint32_t layer_id;
|
|
int64_t access_ts[STORE_BLL_K];
|
|
int32_t access_head;
|
|
int32_t access_filled;
|
|
double wm_anchor;
|
|
float* emb; /* owned; NULL if not embedded */
|
|
int32_t emb_dim;
|
|
/* Forward-compat: raw bytes of any TLV fields the reader did not recognise,
|
|
* concatenated verbatim ([tag][u32 len][bytes]...). Re-emitted on write so
|
|
* an old reader never drops a newer writer's fields. */
|
|
uint8_t* unknown;
|
|
size_t unknown_len;
|
|
int tombstoned; /* set by store_get_* if the located record is dead */
|
|
/* hebb_elig / hebb_elig_ts are DELIBERATELY NOT persisted (see EngramNode). */
|
|
} StoreNode;
|
|
|
|
/* ── Serializable edge view: every persisted EngramEdge field ─────────────── */
|
|
typedef struct StoreEdge {
|
|
char* id;
|
|
char* from_id;
|
|
char* to_id;
|
|
char* relation;
|
|
char* metadata;
|
|
double weight;
|
|
double hebb;
|
|
double confidence;
|
|
int64_t created_at;
|
|
int64_t updated_at;
|
|
int64_t last_fired;
|
|
int32_t inhibitory;
|
|
uint32_t layer_id;
|
|
uint8_t* unknown;
|
|
size_t unknown_len;
|
|
int tombstoned;
|
|
} StoreEdge;
|
|
|
|
typedef struct EngramPagedStore EngramPagedStore;
|
|
|
|
/* Lifecycle. */
|
|
EngramPagedStore* store_create(const char* path); /* fails if file exists */
|
|
EngramPagedStore* store_open(const char* path); /* recovers via mirror SB */
|
|
int store_close(EngramPagedStore* s); /* syncs + frees */
|
|
int store_sync(EngramPagedStore* s); /* fsync + rewrite both superblocks */
|
|
|
|
/* Nodes. store_get_node returns 1 on hit (fills *out, caller store_node_free),
|
|
* 0 if absent or tombstoned, <0 on error. */
|
|
int store_put_node(EngramPagedStore* s, const StoreNode* n);
|
|
int store_get_node(EngramPagedStore* s, const char* id, StoreNode* out);
|
|
int store_tombstone(EngramPagedStore* s, const char* id);
|
|
|
|
/* Edges. *out is malloc'd (store_edges_free); *n set to count. */
|
|
int store_put_edge(EngramPagedStore* s, const StoreEdge* e);
|
|
int store_get_edges_from(EngramPagedStore* s, const char* from_id, StoreEdge** out, size_t* n);
|
|
int store_get_edges_to(EngramPagedStore* s, const char* to_id, StoreEdge** out, size_t* n);
|
|
|
|
/* Integrity: verify every page's crc (and both superblocks). Returns the number
|
|
* of corrupt pages (0 = clean), or <0 on I/O error. */
|
|
int store_check(EngramPagedStore* s, unsigned flags);
|
|
|
|
/* Ownership helpers. */
|
|
void store_node_free(StoreNode* n);
|
|
void store_edge_free(StoreEdge* e);
|
|
void store_edges_free(StoreEdge* arr, size_t n);
|
|
|
|
/* Test-only hook (NOT a format property — B+-tree nodes are self-describing via
|
|
* their stored key count). Caps entries/keys per index node to force splits on
|
|
* small datasets. 0 = natural full-page fanout. */
|
|
void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max);
|
|
|
|
/* Introspection for tests/tools. */
|
|
uint64_t store_page_count(const EngramPagedStore* s);
|
|
|
|
/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
|
|
*
|
|
* The durable engram is `engram.store` (paged) fronted by `engram.wal`
|
|
* (append-only). A mutation is durable once its WAL record is fsync'd
|
|
* (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
|
|
* store only at a checkpoint, so the store file on disk always reflects a
|
|
* consistent point (`last_checkpoint_lsn`) and the WAL owns everything since.
|
|
* Recovery = open store, replay WAL forward, redo a record only where the target
|
|
* record's home page LSN < record LSN (idempotent). JSON is ONLY an import
|
|
* source / export artifact — never the ongoing store. */
|
|
|
|
typedef enum { ENGRAM_WAL_ALWAYS = 0, ENGRAM_WAL_GROUP = 1, ENGRAM_WAL_OFF = 2 } EngramWalSync;
|
|
|
|
/* Serializable layer-registry view (the `layers` array of the legacy snapshot). */
|
|
typedef struct StoreLayer {
|
|
uint32_t layer_id;
|
|
char* name;
|
|
uint32_t activation_priority;
|
|
int32_t suppressible;
|
|
int32_t transparent;
|
|
int32_t injectable;
|
|
uint8_t* unknown;
|
|
size_t unknown_len;
|
|
int tombstoned;
|
|
} StoreLayer;
|
|
|
|
/* Boot the durable engram in `data_dir` (holds engram.store + engram.wal). If the
|
|
* store is absent but a legacy snapshot.json exists, it is imported ONCE into a
|
|
* fresh store; thereafter the store is authoritative and JSON is never read again.
|
|
* On open, the WAL is replayed to recover any post-checkpoint mutations. */
|
|
EngramPagedStore* engram_open(const char* data_dir);
|
|
int engram_close(EngramPagedStore* s); /* checkpoint + close */
|
|
|
|
/* Force a checkpoint: flush dirty pages → fsync store → advance checkpoint LSN →
|
|
* reclaim the WAL prefix. Also threshold-triggered automatically on the write path. */
|
|
int engram_checkpoint(EngramPagedStore* s);
|
|
|
|
/* WAL commit policy. engram_open honours env ENGRAM_WAL_SYNC=always|group|off. */
|
|
void engram_set_wal_sync(EngramPagedStore* s, EngramWalSync policy);
|
|
|
|
/* Layer registry. */
|
|
int store_put_layer(EngramPagedStore* s, const StoreLayer* L);
|
|
int store_get_layer(EngramPagedStore* s, uint32_t layer_id, StoreLayer* out);
|
|
int store_del_layer(EngramPagedStore* s, uint32_t layer_id);
|
|
int store_list_layers(EngramPagedStore* s, StoreLayer** out, size_t* n);
|
|
void store_layer_free(StoreLayer* L);
|
|
void store_layers_free(StoreLayer* arr, size_t n);
|
|
|
|
/* Edge lookup by id (for hebb updates + idempotency). 1 hit / 0 absent / <0 err. */
|
|
int store_get_edge(EngramPagedStore* s, const char* id, StoreEdge* out);
|
|
|
|
/* HEBB batch: one WAL record updating hebb (+ last_fired) on a set of edges. */
|
|
typedef struct StoreHebbDelta { const char* edge_id; double hebb; int64_t last_fired; } StoreHebbDelta;
|
|
int store_hebb_batch(EngramPagedStore* s, const StoreHebbDelta* d, size_t n);
|
|
|
|
/* Supersede: logs the (old,new) pair and tombstones old_id at the store; the new
|
|
* node + `supersedes` edge are logged separately (neuron-layer immutability). */
|
|
int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id);
|
|
|
|
/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
|
|
int store_forget(EngramPagedStore* s, const char* id);
|
|
|
|
/* Introspection / test hooks. */
|
|
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
|
|
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);
|
|
|
|
/* Crash-test hooks (writes only under a throwaway dir).
|
|
* store__crash — abandon all RAM state without flush/fsync (power loss).
|
|
* store__flush_pages — pwrite dirty pages to disk WITHOUT a checkpoint (steal).
|
|
* store__checkpoint_crashat — run checkpoint but stop (then power-loss) after
|
|
* `phase` (0..4); phase<0 = full checkpoint. */
|
|
void store__crash(EngramPagedStore* s);
|
|
int store__flush_pages(EngramPagedStore* s);
|
|
int store__checkpoint_crashat(EngramPagedStore* s, int phase);
|
|
|
|
#endif /* ENGRAM_STORE_H */
|