neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

engram tiered storage M3: wire store behind ENGRAM_STORE (default off) + .egm rename

Browse Source 
Caller-side shim in el_runtime.c maps EngramNode/Edge <-> StoreNode/Edge; engine
keeps zero soul deps (libengram boundary, design §10). Flag off = today's JSON
path byte-for-byte (proven: no neuron.egm created, graph identical). Flag on =
engram_open (import snapshot.json once into neuron.egm, else WAL-replay) +
resident load; node/edge create + forget dual-write via guarded hooks. Files
renamed engram.store->neuron.egm, engram.wal->neuron.wal.

Gate: M3 parity PASS (graph on==off byte-exact modulo ordering; snapshot round-trip;
reboot-from-egm with snapshot.json deleted; activation set+sequence identical;
ASan/UBSan clean). M1 33/33 + M2 36/36 green post-rename.

Known gap (pre-flip): in-place hebb/WM/activation_count updates during activation
are not yet persisted to the store (create/connect/forget are). Must close before
live flip so learned edges survive restart.
This commit is contained in:
Will Anderson
2026-08-11 23:21:21 -05:00
parent 8affb1d6e0
commit a72145b44e
6 changed files with 588 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/el_runtime.c
+208
View File
@@ -7267,6 +7267,204 @@ static char* engram_first_n_chars(const char* s, size_t n) {
return out;
}
/* ══════════════════════════════════════════════════════════════════════════
* M3 — ENGRAM_STORE glue (CALLER side of the libengram ABI; design §10).
*
* The engine (engram_store.{c,h}) has ZERO soul dependencies and never sees an
* EngramNode/EngramEdge or a soul global. ALL mapping between the live runtime
* structs and the engine's StoreNode/StoreEdge views lives HERE, on the caller
* side of the C ABI. That is what keeps a standalone `engramd` a later additive
* choice rather than a fork.
*
* Behind the ENGRAM_STORE env flag (default OFF):
* OFF (unset / "0" / "off") — every hook below early-returns; the paged store
* is never opened or written and no store code is reached. The runtime keeps
* EXACTLY today's JSON-snapshot behavior, byte-for-byte.
* ON ("1" / "on" / "true") — engram_store_boot() imports snapshot.json ONCE
* into neuron.egm (or replays neuron.wal), loads the WHOLE store resident in
* RAM (Phase 1: no demand paging — that is M4), and every structural
* mutation (node/edge create, forget) is mirrored through the store's
* WAL-logged API so neuron.egm/neuron.wal stay authoritative.
* ══════════════════════════════════════════════════════════════════════════ */
#include "engram_store.h"
static EngramPagedStore* g_engram_store = NULL;
int engram_store_enabled(void) {
const char* f = getenv("ENGRAM_STORE");
return (f && (strcmp(f, "1") == 0 || strcmp(f, "on") == 0 ||
strcmp(f, "true") == 0)) ? 1 : 0;
}
/* EngramNode → borrowed StoreNode view (no ownership transfer; the store copies
* every field it persists, so shared string pointers are safe). */
static void eg_node_to_store(const EngramNode* n, StoreNode* sn) {
memset(sn, 0, sizeof *sn);
sn->id = n->id; sn->content = n->content; sn->node_type = n->node_type;
sn->label = n->label; sn->tier = n->tier; sn->tags = n->tags;
sn->metadata = n->metadata;
sn->salience = n->salience; sn->importance = n->importance;
sn->confidence = n->confidence; sn->temporal_decay_rate = n->temporal_decay_rate;
sn->activation_count = n->activation_count; sn->last_activated = n->last_activated;
sn->created_at = n->created_at; sn->updated_at = n->updated_at;
sn->background_activation = n->background_activation;
sn->working_memory_weight = n->working_memory_weight;
sn->suppression_count = n->suppression_count; sn->layer_id = n->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
sn->access_ts[i] = n->access_ts[i];
sn->access_head = n->access_head; sn->access_filled = n->access_filled;
sn->wm_anchor = n->wm_anchor; sn->emb = n->emb; sn->emb_dim = n->emb_dim;
}
static void eg_edge_to_store(const EngramEdge* e, StoreEdge* se) {
memset(se, 0, sizeof *se);
se->id = e->id; se->from_id = e->from_id; se->to_id = e->to_id;
se->relation = e->relation; se->metadata = e->metadata;
se->weight = e->weight; se->hebb = e->hebb; se->confidence = e->confidence;
se->created_at = e->created_at; se->updated_at = e->updated_at;
se->last_fired = e->last_fired; se->inhibitory = e->inhibitory;
se->layer_id = e->layer_id;
}
/* Structural-mutation hooks. Callers guard with `if (engram_store_enabled())`;
* these also null-check g_engram_store so a mutation before boot is a safe no-op. */
static void eg_store_put_node(const EngramNode* n) {
if (!g_engram_store || !n || !n->id) return;
StoreNode sn; eg_node_to_store(n, &sn);
store_put_node(g_engram_store, &sn);
}
static void eg_store_put_edge(const EngramEdge* e) {
if (!g_engram_store || !e || !e->id) return;
StoreEdge se; eg_edge_to_store(e, &se);
store_put_edge(g_engram_store, &se);
}
/* Resident-load callbacks: StoreNode/StoreEdge → a fresh EngramNode/EngramEdge
* appended to the in-RAM graph. Mirrors engram_load's field set (minus the
* boot-time WM laundering — the store already holds the authoritative weights). */
static void eg_load_node_cb(const StoreNode* sn, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_nodes();
EngramNode* n = &g->nodes[g->node_count];
memset(n, 0, sizeof *n);
n->id = el_strdup_persist(sn->id ? sn->id : "");
n->content = el_strdup_persist(sn->content ? sn->content : "");
n->node_type = el_strdup_persist(sn->node_type && *sn->node_type ? sn->node_type : "Memory");
n->label = el_strdup_persist(sn->label ? sn->label : "");
n->tier = el_strdup_persist(sn->tier && *sn->tier ? sn->tier : "Working");
n->tags = el_strdup_persist(sn->tags ? sn->tags : "");
n->metadata = el_strdup_persist(sn->metadata && *sn->metadata ? sn->metadata : "{}");
n->salience = sn->salience; n->importance = sn->importance;
n->confidence = sn->confidence; n->temporal_decay_rate = sn->temporal_decay_rate;
n->activation_count = sn->activation_count; n->last_activated = sn->last_activated;
n->created_at = sn->created_at; n->updated_at = sn->updated_at;
n->background_activation = sn->background_activation;
n->working_memory_weight = sn->working_memory_weight;
n->suppression_count = sn->suppression_count; n->layer_id = sn->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
n->access_ts[i] = sn->access_ts[i];
n->access_head = sn->access_head; n->access_filled = sn->access_filled;
n->wm_anchor = sn->wm_anchor;
if (sn->emb && sn->emb_dim > 0) {
n->emb = malloc(sizeof(float) * (size_t)sn->emb_dim);
if (n->emb) { memcpy(n->emb, sn->emb, sizeof(float) * (size_t)sn->emb_dim);
n->emb_dim = sn->emb_dim; }
}
int64_t idx = g->node_count; g->node_count++;
if (n->id && *n->id) engram_idmap_put(g, n->id, idx);
}
static void eg_load_edge_cb(const StoreEdge* se, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_edges();
EngramEdge* e = &g->edges[g->edge_count];
memset(e, 0, sizeof *e);
e->id = el_strdup_persist(se->id ? se->id : "");
e->from_id = el_strdup_persist(se->from_id ? se->from_id : "");
e->to_id = el_strdup_persist(se->to_id ? se->to_id : "");
e->relation = el_strdup_persist(se->relation && *se->relation ? se->relation : "associate");
e->metadata = el_strdup_persist(se->metadata && *se->metadata ? se->metadata : "{}");
e->weight = se->weight; e->hebb = se->hebb; e->confidence = se->confidence;
e->created_at = se->created_at; e->updated_at = se->updated_at;
e->last_fired = se->last_fired; e->inhibitory = se->inhibitory;
e->layer_id = se->layer_id;
g->edge_count++;
}
static void eg_load_layer_cb(EngramStore* g, const StoreLayer* L) {
if (!L->name) return;
for (size_t i = 0; i < g->layer_count; i++) /* upsert by id */
if (g->layers[i].layer_id == L->layer_id) return; /* canonical already seeded */
if (g->layer_count >= g->layer_capacity) {
size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16;
EngramLayer* nl = realloc(g->layers, nc * sizeof(EngramLayer));
if (!nl) return;
g->layers = nl; g->layer_capacity = nc;
}
g->layers[g->layer_count++] = (EngramLayer){
.layer_id = L->layer_id,
.name = el_strdup_persist(L->name),
.activation_priority = L->activation_priority,
.suppressible = L->suppressible,
.transparent = L->transparent,
.injectable = L->injectable
};
}
/* Clear the resident graph so the store becomes the sole source of truth on boot
* (mirrors engram_load's reset). */
static void eg_reset_resident(EngramStore* g) {
for (int64_t i = 0; i < g->node_count; i++) {
free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type);
free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags);
free(g->nodes[i].metadata);
free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0;
}
g->node_count = 0;
for (int64_t i = 0; i < g->edge_count; i++) {
free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id);
free(g->edges[i].relation); free(g->edges[i].metadata);
}
g->edge_count = 0;
engram_idmap_free(g);
engram_adj_free(g);
}
/* engram_store_boot(data_dir) — open (import-once or WAL-replay) the durable
* paged store and load it whole into RAM (Phase 1). No-op / returns 0 when the
* flag is off. Returns 1 on success. Idempotent (a second call is a no-op). */
el_val_t engram_store_boot(el_val_t data_dir) {
if (!engram_store_enabled()) return (el_val_t)0;
if (g_engram_store) return (el_val_t)1;
const char* d = EL_CSTR(data_dir);
if (!d || !*d) return (el_val_t)0;
g_engram_store = engram_open(d);
if (!g_engram_store) return (el_val_t)0;
EngramStore* g = engram_get();
eg_reset_resident(g);
store_scan_nodes(g_engram_store, eg_load_node_cb, g);
store_scan_edges(g_engram_store, eg_load_edge_cb, g);
StoreLayer* ls = NULL; size_t ln = 0;
if (store_list_layers(g_engram_store, &ls, &ln) == 0) {
for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]);
store_layers_free(ls, ln);
}
g->adj_dirty = 1;
return (el_val_t)1;
}
/* engram_store_checkpoint() — flush dirty pages + advance the checkpoint LSN.
* The store→JSON export path stays engram_save (the JSON is an export artifact). */
el_val_t engram_store_checkpoint(void) {
if (!engram_store_enabled() || !g_engram_store) return (el_val_t)0;
return (el_val_t)(int64_t)(engram_checkpoint(g_engram_store) == 0 ? 1 : 0);
}
/* engram_store_close() — checkpoint + close (used at shutdown / by tests). */
el_val_t engram_store_close(void) {
if (!g_engram_store) return (el_val_t)0;
int r = engram_close(g_engram_store);
g_engram_store = NULL;
return (el_val_t)(int64_t)(r == 0 ? 1 : 0);
}
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
EngramStore* g = engram_get();
engram_grow_nodes();
@@ -7296,6 +7494,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
g->node_count++;
engram_idmap_put(g, n->id, new_idx);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7425,6 +7624,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
g->node_count++;
engram_idmap_put(g, n->id, new_idx_full);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7495,6 +7695,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
g->node_count++;
engram_idmap_put(g, n->id, new_idx_layered);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7642,6 +7843,12 @@ void engram_forget(el_val_t node_id) {
EngramStore* g = engram_get();
int64_t idx = engram_find_node_index(sid);
if (idx < 0) return;
/* Mirror the removal into the durable store BEFORE the shift-delete frees
* the incident edges' ids (node tombstone; edge records are reclaimed at
* compaction — M5). Node-level FORGET matches the existing WAL semantics. */
if (engram_store_enabled() && g_engram_store) {
store_forget(g_engram_store, sid);
}
/* Free node strings */
EngramNode* n = &g->nodes[idx];
free(n->id); free(n->content); free(n->node_type); free(n->label);
@@ -7980,6 +8187,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_edge(e);
}
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) {