neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

engram tiered storage M3: wire store behind ENGRAM_STORE (default off) + .egm rename

Browse Source 
Caller-side shim in el_runtime.c maps EngramNode/Edge <-> StoreNode/Edge; engine
keeps zero soul deps (libengram boundary, design §10). Flag off = today's JSON
path byte-for-byte (proven: no neuron.egm created, graph identical). Flag on =
engram_open (import snapshot.json once into neuron.egm, else WAL-replay) +
resident load; node/edge create + forget dual-write via guarded hooks. Files
renamed engram.store->neuron.egm, engram.wal->neuron.wal.

Gate: M3 parity PASS (graph on==off byte-exact modulo ordering; snapshot round-trip;
reboot-from-egm with snapshot.json deleted; activation set+sequence identical;
ASan/UBSan clean). M1 33/33 + M2 36/36 green post-rename.

Known gap (pre-flip): in-place hebb/WM/activation_count updates during activation
are not yet persisted to the store (create/connect/forget are). Must close before
live flip so learned edges survive restart.
This commit is contained in:
Will Anderson
2026-08-11 23:21:21 -05:00
parent 8affb1d6e0
commit a72145b44e
6 changed files with 588 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/el_runtime.c
+208
View File
@@ -7267,6 +7267,204 @@ static char* engram_first_n_chars(const char* s, size_t n) {
return out;
}
/* ══════════════════════════════════════════════════════════════════════════
* M3 — ENGRAM_STORE glue (CALLER side of the libengram ABI; design §10).
*
* The engine (engram_store.{c,h}) has ZERO soul dependencies and never sees an
* EngramNode/EngramEdge or a soul global. ALL mapping between the live runtime
* structs and the engine's StoreNode/StoreEdge views lives HERE, on the caller
* side of the C ABI. That is what keeps a standalone `engramd` a later additive
* choice rather than a fork.
*
* Behind the ENGRAM_STORE env flag (default OFF):
* OFF (unset / "0" / "off") — every hook below early-returns; the paged store
* is never opened or written and no store code is reached. The runtime keeps
* EXACTLY today's JSON-snapshot behavior, byte-for-byte.
* ON ("1" / "on" / "true") — engram_store_boot() imports snapshot.json ONCE
* into neuron.egm (or replays neuron.wal), loads the WHOLE store resident in
* RAM (Phase 1: no demand paging — that is M4), and every structural
* mutation (node/edge create, forget) is mirrored through the store's
* WAL-logged API so neuron.egm/neuron.wal stay authoritative.
* ══════════════════════════════════════════════════════════════════════════ */
#include "engram_store.h"
static EngramPagedStore* g_engram_store = NULL;
int engram_store_enabled(void) {
const char* f = getenv("ENGRAM_STORE");
return (f && (strcmp(f, "1") == 0 || strcmp(f, "on") == 0 ||
strcmp(f, "true") == 0)) ? 1 : 0;
}
/* EngramNode → borrowed StoreNode view (no ownership transfer; the store copies
* every field it persists, so shared string pointers are safe). */
static void eg_node_to_store(const EngramNode* n, StoreNode* sn) {
memset(sn, 0, sizeof *sn);
sn->id = n->id; sn->content = n->content; sn->node_type = n->node_type;
sn->label = n->label; sn->tier = n->tier; sn->tags = n->tags;
sn->metadata = n->metadata;
sn->salience = n->salience; sn->importance = n->importance;
sn->confidence = n->confidence; sn->temporal_decay_rate = n->temporal_decay_rate;
sn->activation_count = n->activation_count; sn->last_activated = n->last_activated;
sn->created_at = n->created_at; sn->updated_at = n->updated_at;
sn->background_activation = n->background_activation;
sn->working_memory_weight = n->working_memory_weight;
sn->suppression_count = n->suppression_count; sn->layer_id = n->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
sn->access_ts[i] = n->access_ts[i];
sn->access_head = n->access_head; sn->access_filled = n->access_filled;
sn->wm_anchor = n->wm_anchor; sn->emb = n->emb; sn->emb_dim = n->emb_dim;
}
static void eg_edge_to_store(const EngramEdge* e, StoreEdge* se) {
memset(se, 0, sizeof *se);
se->id = e->id; se->from_id = e->from_id; se->to_id = e->to_id;
se->relation = e->relation; se->metadata = e->metadata;
se->weight = e->weight; se->hebb = e->hebb; se->confidence = e->confidence;
se->created_at = e->created_at; se->updated_at = e->updated_at;
se->last_fired = e->last_fired; se->inhibitory = e->inhibitory;
se->layer_id = e->layer_id;
}
/* Structural-mutation hooks. Callers guard with `if (engram_store_enabled())`;
* these also null-check g_engram_store so a mutation before boot is a safe no-op. */
static void eg_store_put_node(const EngramNode* n) {
if (!g_engram_store || !n || !n->id) return;
StoreNode sn; eg_node_to_store(n, &sn);
store_put_node(g_engram_store, &sn);
}
static void eg_store_put_edge(const EngramEdge* e) {
if (!g_engram_store || !e || !e->id) return;
StoreEdge se; eg_edge_to_store(e, &se);
store_put_edge(g_engram_store, &se);
}
/* Resident-load callbacks: StoreNode/StoreEdge → a fresh EngramNode/EngramEdge
* appended to the in-RAM graph. Mirrors engram_load's field set (minus the
* boot-time WM laundering — the store already holds the authoritative weights). */
static void eg_load_node_cb(const StoreNode* sn, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_nodes();
EngramNode* n = &g->nodes[g->node_count];
memset(n, 0, sizeof *n);
n->id = el_strdup_persist(sn->id ? sn->id : "");
n->content = el_strdup_persist(sn->content ? sn->content : "");
n->node_type = el_strdup_persist(sn->node_type && *sn->node_type ? sn->node_type : "Memory");
n->label = el_strdup_persist(sn->label ? sn->label : "");
n->tier = el_strdup_persist(sn->tier && *sn->tier ? sn->tier : "Working");
n->tags = el_strdup_persist(sn->tags ? sn->tags : "");
n->metadata = el_strdup_persist(sn->metadata && *sn->metadata ? sn->metadata : "{}");
n->salience = sn->salience; n->importance = sn->importance;
n->confidence = sn->confidence; n->temporal_decay_rate = sn->temporal_decay_rate;
n->activation_count = sn->activation_count; n->last_activated = sn->last_activated;
n->created_at = sn->created_at; n->updated_at = sn->updated_at;
n->background_activation = sn->background_activation;
n->working_memory_weight = sn->working_memory_weight;
n->suppression_count = sn->suppression_count; n->layer_id = sn->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
n->access_ts[i] = sn->access_ts[i];
n->access_head = sn->access_head; n->access_filled = sn->access_filled;
n->wm_anchor = sn->wm_anchor;
if (sn->emb && sn->emb_dim > 0) {
n->emb = malloc(sizeof(float) * (size_t)sn->emb_dim);
if (n->emb) { memcpy(n->emb, sn->emb, sizeof(float) * (size_t)sn->emb_dim);
n->emb_dim = sn->emb_dim; }
}
int64_t idx = g->node_count; g->node_count++;
if (n->id && *n->id) engram_idmap_put(g, n->id, idx);
}
static void eg_load_edge_cb(const StoreEdge* se, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_edges();
EngramEdge* e = &g->edges[g->edge_count];
memset(e, 0, sizeof *e);
e->id = el_strdup_persist(se->id ? se->id : "");
e->from_id = el_strdup_persist(se->from_id ? se->from_id : "");
e->to_id = el_strdup_persist(se->to_id ? se->to_id : "");
e->relation = el_strdup_persist(se->relation && *se->relation ? se->relation : "associate");
e->metadata = el_strdup_persist(se->metadata && *se->metadata ? se->metadata : "{}");
e->weight = se->weight; e->hebb = se->hebb; e->confidence = se->confidence;
e->created_at = se->created_at; e->updated_at = se->updated_at;
e->last_fired = se->last_fired; e->inhibitory = se->inhibitory;
e->layer_id = se->layer_id;
g->edge_count++;
}
static void eg_load_layer_cb(EngramStore* g, const StoreLayer* L) {
if (!L->name) return;
for (size_t i = 0; i < g->layer_count; i++) /* upsert by id */
if (g->layers[i].layer_id == L->layer_id) return; /* canonical already seeded */
if (g->layer_count >= g->layer_capacity) {
size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16;
EngramLayer* nl = realloc(g->layers, nc * sizeof(EngramLayer));
if (!nl) return;
g->layers = nl; g->layer_capacity = nc;
}
g->layers[g->layer_count++] = (EngramLayer){
.layer_id = L->layer_id,
.name = el_strdup_persist(L->name),
.activation_priority = L->activation_priority,
.suppressible = L->suppressible,
.transparent = L->transparent,
.injectable = L->injectable
};
}
/* Clear the resident graph so the store becomes the sole source of truth on boot
* (mirrors engram_load's reset). */
static void eg_reset_resident(EngramStore* g) {
for (int64_t i = 0; i < g->node_count; i++) {
free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type);
free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags);
free(g->nodes[i].metadata);
free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0;
}
g->node_count = 0;
for (int64_t i = 0; i < g->edge_count; i++) {
free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id);
free(g->edges[i].relation); free(g->edges[i].metadata);
}
g->edge_count = 0;
engram_idmap_free(g);
engram_adj_free(g);
}
/* engram_store_boot(data_dir) — open (import-once or WAL-replay) the durable
* paged store and load it whole into RAM (Phase 1). No-op / returns 0 when the
* flag is off. Returns 1 on success. Idempotent (a second call is a no-op). */
el_val_t engram_store_boot(el_val_t data_dir) {
if (!engram_store_enabled()) return (el_val_t)0;
if (g_engram_store) return (el_val_t)1;
const char* d = EL_CSTR(data_dir);
if (!d || !*d) return (el_val_t)0;
g_engram_store = engram_open(d);
if (!g_engram_store) return (el_val_t)0;
EngramStore* g = engram_get();
eg_reset_resident(g);
store_scan_nodes(g_engram_store, eg_load_node_cb, g);
store_scan_edges(g_engram_store, eg_load_edge_cb, g);
StoreLayer* ls = NULL; size_t ln = 0;
if (store_list_layers(g_engram_store, &ls, &ln) == 0) {
for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]);
store_layers_free(ls, ln);
}
g->adj_dirty = 1;
return (el_val_t)1;
}
/* engram_store_checkpoint() — flush dirty pages + advance the checkpoint LSN.
* The store→JSON export path stays engram_save (the JSON is an export artifact). */
el_val_t engram_store_checkpoint(void) {
if (!engram_store_enabled() || !g_engram_store) return (el_val_t)0;
return (el_val_t)(int64_t)(engram_checkpoint(g_engram_store) == 0 ? 1 : 0);
}
/* engram_store_close() — checkpoint + close (used at shutdown / by tests). */
el_val_t engram_store_close(void) {
if (!g_engram_store) return (el_val_t)0;
int r = engram_close(g_engram_store);
g_engram_store = NULL;
return (el_val_t)(int64_t)(r == 0 ? 1 : 0);
}
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
EngramStore* g = engram_get();
engram_grow_nodes();
@@ -7296,6 +7494,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
g->node_count++;
engram_idmap_put(g, n->id, new_idx);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7425,6 +7624,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
g->node_count++;
engram_idmap_put(g, n->id, new_idx_full);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7495,6 +7695,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
g->node_count++;
engram_idmap_put(g, n->id, new_idx_layered);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7642,6 +7843,12 @@ void engram_forget(el_val_t node_id) {
EngramStore* g = engram_get();
int64_t idx = engram_find_node_index(sid);
if (idx < 0) return;
/* Mirror the removal into the durable store BEFORE the shift-delete frees
* the incident edges' ids (node tombstone; edge records are reclaimed at
* compaction — M5). Node-level FORGET matches the existing WAL semantics. */
if (engram_store_enabled() && g_engram_store) {
store_forget(g_engram_store, sid);
}
/* Free node strings */
EngramNode* n = &g->nodes[idx];
free(n->id); free(n->content); free(n->node_type); free(n->label);
@@ -7980,6 +8187,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_edge(e);
}
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) {
lang/runtime/engram_store.c
+86 -3
View File
@@ -1137,12 +1137,95 @@ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max)
}
uint64_t store_page_count(const EngramPagedStore* s){ return s ? s->page_count : 0; }
/* ── M3: full live enumeration (boundary-clean; StoreNode/StoreEdge out only) ──
* Page-walk every NODE/EDGE page, emitting each DISTINCT live record. A re-put
* leaves several live records for one id (apply_node_put appends; reads dedup),
* so we track ids already emitted by their 64-bit id-hash — the same key the
* primary B+-tree uses (design §2.4) — and fetch the canonical latest-live via
* the point-read path so a scan and a get agree exactly. Used by the caller
* (el_runtime) to load the whole store resident at boot and to export JSON. */
typedef struct { uint64_t* h; size_t n, cap; } U64Set;
static int u64set_add(U64Set* s, uint64_t v){ /* 1 = newly added, 0 = present */
if ((s->n + 1) * 4 >= s->cap * 3){
size_t nc = s->cap ? s->cap * 2 : 1024;
uint64_t* nh = (uint64_t*)calloc(nc, sizeof(uint64_t));
if (!nh) return 1; /* degrade rather than crash */
for (size_t i = 0; i < s->cap; i++){
uint64_t k = s->h[i];
if (k){ size_t j = k & (nc - 1); while (nh[j]) j = (j + 1) & (nc - 1); nh[j] = k; }
}
free(s->h); s->h = nh; s->cap = nc;
}
uint64_t k = v ? v : 1; /* 0 reserved as empty slot */
size_t j = k & (s->cap - 1);
while (s->h[j]){ if (s->h[j] == k) return 0; j = (j + 1) & (s->cap - 1); }
s->h[j] = k; s->n++; return 1;
}
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_NODE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreNode cand; node_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreNode canon;
if (store_get_node(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_node_free(&canon);
}
}
store_node_free(&cand);
}
}
free(seen.h);
return count;
}
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_EDGE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreEdge cand; edge_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreEdge canon;
if (store_get_edge(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_edge_free(&canon);
}
}
store_edge_free(&cand);
}
}
free(seen.h);
return count;
}
/* ══════════════════════════════════════════════════════════════════════════════
* M2 — WAL + write-back buffer pool + checkpoint + crash recovery + legacy import
*
* Durability model (design §2.2/§4, ARIES-lite):
* • Buffer pool is WRITE-BACK, no-steal: a mutation dirties a page in RAM; the
* page reaches engram.store ONLY at a checkpoint. So after a crash the store
* page reaches neuron.egm ONLY at a checkpoint. So after a crash the store
* file reflects exactly `last_checkpoint_lsn`, and everything since lives in
* the WAL. This is what makes the WAL load-bearing (durability = fsync'd WAL,
* not the page).
@@ -2018,8 +2101,8 @@ static int import_snapshot(EngramPagedStore* s, const char* path){
EngramPagedStore* engram_open(const char* data_dir){
if (!data_dir) return NULL;
char store_path[1200], wal_path[1200], snap_path[1200];
snprintf(store_path, sizeof store_path, "%s/engram.store", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/engram.wal", data_dir);
snprintf(store_path, sizeof store_path, "%s/neuron.egm", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/neuron.wal", data_dir);
snprintf(snap_path, sizeof snap_path, "%s/snapshot.json", data_dir);
EngramWalSync sync = ENGRAM_WAL_GROUP;
lang/runtime/engram_store.h
+13 -2
View File
@@ -134,7 +134,7 @@ uint64_t store_page_count(const EngramPagedStore* s);
/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
*
* The durable engram is `engram.store` (paged) fronted by `engram.wal`
* The durable engram is `neuron.egm` (paged) fronted by `neuron.wal`
* (append-only). A mutation is durable once its WAL record is fsync'd
* (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
* store only at a checkpoint, so the store file on disk always reflects a
@@ -158,7 +158,7 @@ typedef struct StoreLayer {
int tombstoned;
} StoreLayer;
/* Boot the durable engram in `data_dir` (holds engram.store + engram.wal). If the
/* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the
* store is absent but a legacy snapshot.json exists, it is imported ONCE into a
* fresh store; thereafter the store is authoritative and JSON is never read again.
* On open, the WAL is replayed to recover any post-checkpoint mutations. */
@@ -194,6 +194,17 @@ int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id
/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
int store_forget(EngramPagedStore* s, const char* id);
/* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ──
* Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a
* borrowed view (the engine frees it after cb returns — the callback must copy
* anything it keeps). De-duplicated by id (canonical latest-live per id, matching
* point-read semantics). Returns the count emitted, or <0 on error. The engine
* hands out StoreNode/StoreEdge only — it never sees a soul struct (design §10). */
typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx);
typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx);
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx);
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx);
/* Introspection / test hooks. */
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);