neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

engram tiered storage M3: wire store behind ENGRAM_STORE (default off) + .egm rename

Browse Source 
Caller-side shim in el_runtime.c maps EngramNode/Edge <-> StoreNode/Edge; engine
keeps zero soul deps (libengram boundary, design §10). Flag off = today's JSON
path byte-for-byte (proven: no neuron.egm created, graph identical). Flag on =
engram_open (import snapshot.json once into neuron.egm, else WAL-replay) +
resident load; node/edge create + forget dual-write via guarded hooks. Files
renamed engram.store->neuron.egm, engram.wal->neuron.wal.

Gate: M3 parity PASS (graph on==off byte-exact modulo ordering; snapshot round-trip;
reboot-from-egm with snapshot.json deleted; activation set+sequence identical;
ASan/UBSan clean). M1 33/33 + M2 36/36 green post-rename.

Known gap (pre-flip): in-place hebb/WM/activation_count updates during activation
are not yet persisted to the store (create/connect/forget are). Must close before
live flip so learned edges survive restart.
This commit is contained in:
Will Anderson
2026-08-11 23:21:21 -05:00
parent 8affb1d6e0
commit a72145b44e
6 changed files with 588 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/engram_store.c
+86 -3
View File
@@ -1137,12 +1137,95 @@ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max)
}
uint64_t store_page_count(const EngramPagedStore* s){ return s ? s->page_count : 0; }
/* ── M3: full live enumeration (boundary-clean; StoreNode/StoreEdge out only) ──
* Page-walk every NODE/EDGE page, emitting each DISTINCT live record. A re-put
* leaves several live records for one id (apply_node_put appends; reads dedup),
* so we track ids already emitted by their 64-bit id-hash — the same key the
* primary B+-tree uses (design §2.4) — and fetch the canonical latest-live via
* the point-read path so a scan and a get agree exactly. Used by the caller
* (el_runtime) to load the whole store resident at boot and to export JSON. */
typedef struct { uint64_t* h; size_t n, cap; } U64Set;
static int u64set_add(U64Set* s, uint64_t v){ /* 1 = newly added, 0 = present */
if ((s->n + 1) * 4 >= s->cap * 3){
size_t nc = s->cap ? s->cap * 2 : 1024;
uint64_t* nh = (uint64_t*)calloc(nc, sizeof(uint64_t));
if (!nh) return 1; /* degrade rather than crash */
for (size_t i = 0; i < s->cap; i++){
uint64_t k = s->h[i];
if (k){ size_t j = k & (nc - 1); while (nh[j]) j = (j + 1) & (nc - 1); nh[j] = k; }
}
free(s->h); s->h = nh; s->cap = nc;
}
uint64_t k = v ? v : 1; /* 0 reserved as empty slot */
size_t j = k & (s->cap - 1);
while (s->h[j]){ if (s->h[j] == k) return 0; j = (j + 1) & (s->cap - 1); }
s->h[j] = k; s->n++; return 1;
}
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_NODE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreNode cand; node_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreNode canon;
if (store_get_node(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_node_free(&canon);
}
}
store_node_free(&cand);
}
}
free(seen.h);
return count;
}
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_EDGE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreEdge cand; edge_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreEdge canon;
if (store_get_edge(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_edge_free(&canon);
}
}
store_edge_free(&cand);
}
}
free(seen.h);
return count;
}
/* ══════════════════════════════════════════════════════════════════════════════
* M2 — WAL + write-back buffer pool + checkpoint + crash recovery + legacy import
*
* Durability model (design §2.2/§4, ARIES-lite):
* • Buffer pool is WRITE-BACK, no-steal: a mutation dirties a page in RAM; the
* page reaches engram.store ONLY at a checkpoint. So after a crash the store
* page reaches neuron.egm ONLY at a checkpoint. So after a crash the store
* file reflects exactly `last_checkpoint_lsn`, and everything since lives in
* the WAL. This is what makes the WAL load-bearing (durability = fsync'd WAL,
* not the page).
@@ -2018,8 +2101,8 @@ static int import_snapshot(EngramPagedStore* s, const char* path){
EngramPagedStore* engram_open(const char* data_dir){
if (!data_dir) return NULL;
char store_path[1200], wal_path[1200], snap_path[1200];
snprintf(store_path, sizeof store_path, "%s/engram.store", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/engram.wal", data_dir);
snprintf(store_path, sizeof store_path, "%s/neuron.egm", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/neuron.wal", data_dir);
snprintf(snap_path, sizeof snap_path, "%s/snapshot.json", data_dir);
EngramWalSync sync = ENGRAM_WAL_GROUP;