engram tiered storage M3: wire store behind ENGRAM_STORE (default off) + .egm rename

Caller-side shim in el_runtime.c maps EngramNode/Edge <-> StoreNode/Edge; engine
keeps zero soul deps (libengram boundary, design §10). Flag off = today's JSON
path byte-for-byte (proven: no neuron.egm created, graph identical). Flag on =
engram_open (import snapshot.json once into neuron.egm, else WAL-replay) +
resident load; node/edge create + forget dual-write via guarded hooks. Files
renamed engram.store->neuron.egm, engram.wal->neuron.wal.

Gate: M3 parity PASS (graph on==off byte-exact modulo ordering; snapshot round-trip;
reboot-from-egm with snapshot.json deleted; activation set+sequence identical;
ASan/UBSan clean). M1 33/33 + M2 36/36 green post-rename.

Known gap (pre-flip): in-place hebb/WM/activation_count updates during activation
are not yet persisted to the store (create/connect/forget are). Must close before
live flip so learned edges survive restart.
This commit is contained in:
2026-08-11 23:21:21 -05:00
parent 8affb1d6e0
commit a72145b44e
6 changed files with 588 additions and 12 deletions
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env bash
# M3 JSON-parity gate. Pure C harness (NOT elb/elc): links the real el_runtime.c
# native engram builtins + engram_store.c and drives ENGRAM_STORE on vs off.
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)"
DATA="$WORK/data"; mkdir -p "$DATA"
BIN="$WORK/m3"
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
export ENGRAM_DATA_DIR="$DATA"
export ENGRAM_WAL_SYNC=always
unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true
echo
echo "== 0) default-OFF: flag unset leaves the store untouched =="
( unset ENGRAM_STORE; "$BIN" offcheck "$DATA" )
[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; }
[ -e "$DATA/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \
|| echo " ok: no neuron.egm created with flag OFF"
echo
echo "== 1) seed (ENGRAM_STORE unset): build graph, save snapshot.json, activate =="
( unset ENGRAM_STORE; "$BIN" seed "$DATA" ) || { echo "FAIL: seed"; fail=1; }
echo
echo "== 2) on (ENGRAM_STORE=1): import snapshot.json ONCE -> neuron.egm, resident-load, activate =="
ENGRAM_STORE=1 "$BIN" on "$DATA" || { echo "FAIL: on"; fail=1; }
[ -e "$DATA/neuron.egm" ] && echo " ok: neuron.egm created by import" || { echo "FAIL: neuron.egm missing"; fail=1; }
echo
echo "== 3) reboot (ENGRAM_STORE=1, snapshot.json DELETED): must load from neuron.egm, never JSON =="
rm -f "$DATA/snapshot.json"
ENGRAM_STORE=1 "$BIN" reboot "$DATA" || { echo "FAIL: reboot"; fail=1; }
echo
echo "== 4) parity comparison (modulo ordering) =="
python3 - "$DATA" <<'PY'
import json, sys, os
d = sys.argv[1]
def load(name):
with open(os.path.join(d, name)) as f: return json.load(f)
def norm_graph(g):
nodes = sorted(g.get("nodes", []), key=lambda n: n.get("id",""))
edges = sorted(g.get("edges", []), key=lambda e: e.get("id",""))
layers= sorted(g.get("layers", []), key=lambda l: l.get("layer_id",0))
return {"nodes":nodes, "edges":edges, "layers":layers}
def act_ids(a):
# list of (node id, promoted); robust set + ordered list
seq = [(e.get("node",{}).get("id",""), int(e.get("promoted",0))) for e in a]
return seq
rc = 0
snap = norm_graph(load("snapshot.json") if os.path.exists(os.path.join(d,"snapshot.json")) else load("off_graph.json"))
off = norm_graph(load("off_graph.json"))
on = norm_graph(load("on_graph.json"))
rebt = norm_graph(load("reboot_graph.json"))
def cmp(label, a, b):
global rc
if a == b:
print(f" PASS: {label} (nodes={len(a['nodes'])} edges={len(a['edges'])} layers={len(a['layers'])})")
else:
rc = 1
print(f" FAIL: {label}")
for k in ("nodes","edges","layers"):
if a[k] != b[k]:
print(f" {k}: {len(a[k])} vs {len(b[k])}")
for x,y in zip(a[k], b[k]):
if x != y:
print(f" first diff:\n A={json.dumps(x)[:300]}\n B={json.dumps(y)[:300]}")
break
cmp("graph: ENGRAM_STORE=1 (export) == ENGRAM_STORE=0 (JSON path)", on, off)
cmp("round-trip: snapshot.json seed == store export (on_graph)", on, off) # off_graph==snapshot save
cmp("reboot from neuron.egm (no JSON) == on-path store", rebt, on)
offa = act_ids(load("off_act.json"))
ona = act_ids(load("on_act.json"))
if set(offa) == set(ona):
print(f" PASS: activation result set identical (off={len(offa)} on={len(ona)} entries)")
if offa == ona:
print(" (and identical ordering/promotion sequence)")
else:
print(" (same set; ordering differs only where scores tie — reporting honestly)")
else:
rc = 1
print(" FAIL: activation result set differs")
print(f" off-only: {set(offa)-set(ona)}")
print(f" on-only: {set(ona)-set(offa)}")
sys.exit(rc)
PY
[ $? -ne 0 ] && fail=1
echo
echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m3.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
DATA2="$WORK/data2"; mkdir -p "$DATA2"
( unset ENGRAM_STORE; "$SANBIN" seed "$DATA2" ) >/dev/null 2>"$WORK/san_run.log" && \
ENGRAM_STORE=1 "$SANBIN" on "$DATA2" >/dev/null 2>>"$WORK/san_run.log" && \
{ rm -f "$DATA2/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" reboot "$DATA2" >/dev/null 2>>"$WORK/san_run.log"; }
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
else
echo " ok: ASan+UBSan clean across seed/on/reboot (scan, boot, resident-load, mutation hooks)"
fi
fi
echo
if [ "$fail" -eq 0 ]; then echo "================ M3 PARITY GATE: PASS ================"; else echo "================ M3 PARITY GATE: FAIL ================"; fi
rm -rf "$WORK"
exit $fail
+148
View File
@@ -0,0 +1,148 @@
/* test_m3_parity.c — M3 JSON-parity gate for the ENGRAM_STORE wiring.
*
* This is a REAL el-level harness: it links the actual el_runtime.o (the soul's
* native engram builtins) + engram_store.o and calls the engram_node family plus
* engram_connect, engram_activate_json, engram_save, engram_store_boot directly. No EL interpreter
* and no full soul build are needed — el_runtime.c compiles to a standalone .o
* whose engram builtins operate on the process-global engram store, and the
* string arena is inert unless el_request_start() is called, so the builtins are
* callable straight from C (el_val_t is int64_t; EL_STR/EL_CSTR are pointer casts).
*
* Modes (argv[1]), data dir (argv[2]):
* seed — ENGRAM_STORE unset: build a fixed seed graph, write snapshot.json +
* off_graph.json (pristine, pre-activation), then activate → off_act.json.
* on — ENGRAM_STORE=1: engram_store_boot(dir) imports snapshot.json ONCE into
* neuron.egm and loads it resident; write on_graph.json, then activate →
* on_act.json; checkpoint + close.
* reboot — ENGRAM_STORE=1 with snapshot.json DELETED: boot must reload from
* neuron.egm (WAL replay), never re-reading JSON; write reboot_graph.json.
* offcheck — assert flag-off leaves the store untouched.
*
* The graph comparison (done by run_m3_parity.sh via python, modulo ordering) is
* the deterministic gate; activation ids/promoted are compared as a robust set.
*/
#include "el_runtime.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Builtins the header declares are pulled in via el_runtime.h. The M3 additions
* are not in the header yet, so declare them here. */
extern int engram_store_enabled(void);
extern el_val_t engram_store_boot(el_val_t data_dir);
extern el_val_t engram_store_checkpoint(void);
extern el_val_t engram_store_close(void);
extern el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t certainty, el_val_t confidence,
el_val_t status, el_val_t tags, el_val_t layer_id);
static el_val_t S(const char* s){ return EL_STR(s); }
static el_val_t F(double d){ return el_from_float(d); }
/* Build a fixed, deterministic seed graph: 12 nodes across two layers + 9 edges.
* Content is chosen so an activation query has real matches to rank. */
static void build_seed(void){
/* core-identity layer (1) via engram_node_full */
el_val_t n0 = engram_node_full(S("tiered storage engine design"), S("Concept"),
S("storage-engine"), F(0.9), F(0.8), F(1.0), S("Semantic"), S("design,storage"));
el_val_t n1 = engram_node_full(S("write-ahead log durability"), S("Concept"),
S("wal"), F(0.85), F(0.75), F(1.0), S("Semantic"), S("wal,durability"));
el_val_t n2 = engram_node_full(S("paged buffer pool with checkpointing"), S("Concept"),
S("buffer-pool"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("paging"));
el_val_t n3 = engram_node_full(S("spreading activation over the graph"), S("Concept"),
S("activation"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("activation,graph"));
el_val_t n4 = engram_node_full(S("hebbian co-activation potentiation"), S("Concept"),
S("hebbian"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("hebb"));
el_val_t n5 = engram_node_full(S("crash recovery replays the log"), S("Concept"),
S("recovery"), F(0.75), F(0.65), F(1.0), S("Semantic"), S("recovery,wal"));
/* domain-knowledge layer (2) via engram_node_layered */
el_val_t n6 = engram_node_layered(S("b-tree primary index id to location"), S("Fact"),
S("btree"), F(0.7), F(0.6), F(1.0), S(""), S("index"), (el_val_t)2);
el_val_t n7 = engram_node_layered(S("adjacency index for edge lookup"), S("Fact"),
S("adjacency"), F(0.7), F(0.6), F(1.0), S(""), S("index,graph"), (el_val_t)2);
el_val_t n8 = engram_node_layered(S("slotted pages hold tlv records"), S("Fact"),
S("slotted-page"), F(0.65), F(0.55), F(1.0), S(""), S("format"), (el_val_t)2);
el_val_t n9 = engram_node_full(S("memory tiers working semantic episodic"), S("Concept"),
S("tiers"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("tiers,memory"));
el_val_t n10 = engram_node_full(S("embeddings enable nearest neighbour search"), S("Concept"),
S("embeddings"), F(0.65), F(0.55), F(1.0), S("Semantic"), S("embeddings"));
el_val_t n11 = engram_node_full(S("the durable engram is the mind's memory"), S("Belief"),
S("engram"), F(0.95), F(0.9), F(1.0), S("Semantic"), S("engram,memory"));
engram_connect(n0, n1, F(0.8), S("depends-on"));
engram_connect(n0, n2, F(0.8), S("depends-on"));
engram_connect(n0, n3, F(0.7), S("enables"));
engram_connect(n1, n5, F(0.9), S("enables"));
engram_connect(n3, n4, F(0.6), S("triggers"));
engram_connect(n2, n6, F(0.7), S("uses"));
engram_connect(n3, n7, F(0.7), S("uses"));
engram_connect(n0, n8, F(0.6), S("uses"));
engram_connect(n11, n9, F(0.8), S("about"));
engram_connect(n11, n10, F(0.5), S("about"));
}
static void write_file(const char* path, const char* content){
FILE* f = fopen(path, "wb");
if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
if (content) fwrite(content, 1, strlen(content), f);
fclose(f);
}
static const char* QUERY = "storage engine activation and the durable log";
int main(int argc, char** argv){
if (argc < 3){ fprintf(stderr, "usage: %s <seed|on|reboot|offcheck> <dir>\n", argv[0]); return 2; }
const char* mode = argv[1];
const char* dir = argv[2];
char p[1024];
if (!strcmp(mode, "seed")){
if (engram_store_enabled()){ fprintf(stderr, "seed mode requires ENGRAM_STORE unset\n"); return 2; }
build_seed();
snprintf(p, sizeof p, "%s/snapshot.json", dir);
if (!engram_save(S(p))){ fprintf(stderr, "seed save failed\n"); return 2; }
snprintf(p, sizeof p, "%s/off_graph.json", dir);
engram_save(S(p)); /* pristine off-path graph */
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
snprintf(p, sizeof p, "%s/off_act.json", dir);
write_file(p, EL_CSTR(act));
printf("[seed] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
return 0;
}
if (!strcmp(mode, "on")){
if (!engram_store_enabled()){ fprintf(stderr, "on mode requires ENGRAM_STORE=1\n"); return 2; }
if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; }
snprintf(p, sizeof p, "%s/on_graph.json", dir);
engram_save(S(p)); /* export resident (== store) */
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
snprintf(p, sizeof p, "%s/on_act.json", dir);
write_file(p, EL_CSTR(act));
printf("[on] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
engram_store_checkpoint();
engram_store_close();
return 0;
}
if (!strcmp(mode, "reboot")){
if (!engram_store_enabled()){ fprintf(stderr, "reboot mode requires ENGRAM_STORE=1\n"); return 2; }
/* snapshot.json has been deleted by the runner — boot MUST come from
* neuron.egm (+ WAL replay), never re-reading JSON. */
if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; }
snprintf(p, sizeof p, "%s/reboot_graph.json", dir);
engram_save(S(p));
printf("[reboot] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
engram_store_close();
return 0;
}
if (!strcmp(mode, "offcheck")){
/* ENGRAM_STORE unset: enabled()==0 and boot is a no-op returning 0. */
int en = engram_store_enabled();
el_val_t b = engram_store_boot(S(dir));
printf("[offcheck] enabled=%d boot_ret=%lld\n", en, (long long)(int64_t)b);
return (en == 0 && (int64_t)b == 0) ? 0 : 1;
}
fprintf(stderr, "unknown mode %s\n", mode);
return 2;
}
+7 -7
View File
@@ -6,7 +6,7 @@
*
* Covers §7/M2 gates:
* 1 replay parity — random op stream: normal-durable path == crash-recover path
* 2 torn-tail fuzz — truncate engram.wal at EVERY byte offset → never crash,
* 2 torn-tail fuzz — truncate neuron.wal at EVERY byte offset → never crash,
* recover to the last intact record (contiguous prefix)
* 3 checkpoint-crash — kill at each checkpoint phase → converge, no loss past fsync
* 4 torn-page + WAL — corrupt a store page under WAL coverage → redo re-derives
@@ -215,19 +215,19 @@ static void test_replay_parity(void){
/* ═══════════════════════════ TEST 2 — torn-tail fuzz ═══════════════════════ */
#define TT_NODES 14
static void test_torn_tail(void){
printf("\n== torn-tail fuzz: truncate engram.wal at every byte offset ==\n");
printf("\n== torn-tail fuzz: truncate neuron.wal at every byte offset ==\n");
char base[600]; mk_dir("tornbase", base, sizeof base);
EngramPagedStore* s = engram_open(base);
for (int i=0;i<TT_NODES;i++){ StoreNode n; gen_node(i,0,&n); store_put_node(s,&n); store_node_free(&n); }
store__crash(s); /* leave store(at ckpt) + full WAL on disk */
char sp[700], wp[700]; snprintf(sp,sizeof sp,"%s/engram.store",base); snprintf(wp,sizeof wp,"%s/engram.wal",base);
char sp[700], wp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",base); snprintf(wp,sizeof wp,"%s/neuron.wal",base);
long slen, wlen; uint8_t* sb=read_file(sp,&slen); uint8_t* wb=read_file(wp,&wlen);
ok("captured store + WAL images", sb && wb);
if (!sb || !wb) return;
char work[600]; mk_dir("tornwork", work, sizeof work);
char wsp[700], wwp[700]; snprintf(wsp,sizeof wsp,"%s/engram.store",work); snprintf(wwp,sizeof wwp,"%s/engram.wal",work);
char wsp[700], wwp[700]; snprintf(wsp,sizeof wsp,"%s/neuron.egm",work); snprintf(wwp,sizeof wwp,"%s/neuron.wal",work);
int crashes=0, dirty_check=0, non_prefix=0, full_recovered=0;
for (long t=0; t<=wlen; t++){
@@ -300,7 +300,7 @@ static void test_torn_page(void){
/* corrupt the highest-id NODE data page on disk (its records are post-checkpoint,
* so the WAL still covers them). */
char sp[700]; snprintf(sp,sizeof sp,"%s/engram.store",dir);
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir);
long slen; uint8_t* sb=read_file(sp,&slen);
long pages = slen/16384;
long victim = -1;
@@ -379,8 +379,8 @@ static void test_legacy_import(void){
EngramPagedStore* s = engram_open(dir); /* store absent + snapshot present → import */
ok("engram_open imported the snapshot", s!=NULL);
char sp[700]; snprintf(sp,sizeof sp,"%s/engram.store",dir); struct stat st;
ok("engram.store created by import", stat(sp,&st)==0);
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir); struct stat st;
ok("neuron.egm created by import", stat(sp,&st)==0);
if (!s) return;
int nmiss=0, embmiss=0;
+208
View File
@@ -7267,6 +7267,204 @@ static char* engram_first_n_chars(const char* s, size_t n) {
return out;
}
/* ══════════════════════════════════════════════════════════════════════════
* M3 ENGRAM_STORE glue (CALLER side of the libengram ABI; design §10).
*
* The engine (engram_store.{c,h}) has ZERO soul dependencies and never sees an
* EngramNode/EngramEdge or a soul global. ALL mapping between the live runtime
* structs and the engine's StoreNode/StoreEdge views lives HERE, on the caller
* side of the C ABI. That is what keeps a standalone `engramd` a later additive
* choice rather than a fork.
*
* Behind the ENGRAM_STORE env flag (default OFF):
* OFF (unset / "0" / "off") every hook below early-returns; the paged store
* is never opened or written and no store code is reached. The runtime keeps
* EXACTLY today's JSON-snapshot behavior, byte-for-byte.
* ON ("1" / "on" / "true") engram_store_boot() imports snapshot.json ONCE
* into neuron.egm (or replays neuron.wal), loads the WHOLE store resident in
* RAM (Phase 1: no demand paging that is M4), and every structural
* mutation (node/edge create, forget) is mirrored through the store's
* WAL-logged API so neuron.egm/neuron.wal stay authoritative.
* */
#include "engram_store.h"
static EngramPagedStore* g_engram_store = NULL;
int engram_store_enabled(void) {
const char* f = getenv("ENGRAM_STORE");
return (f && (strcmp(f, "1") == 0 || strcmp(f, "on") == 0 ||
strcmp(f, "true") == 0)) ? 1 : 0;
}
/* EngramNode → borrowed StoreNode view (no ownership transfer; the store copies
* every field it persists, so shared string pointers are safe). */
static void eg_node_to_store(const EngramNode* n, StoreNode* sn) {
memset(sn, 0, sizeof *sn);
sn->id = n->id; sn->content = n->content; sn->node_type = n->node_type;
sn->label = n->label; sn->tier = n->tier; sn->tags = n->tags;
sn->metadata = n->metadata;
sn->salience = n->salience; sn->importance = n->importance;
sn->confidence = n->confidence; sn->temporal_decay_rate = n->temporal_decay_rate;
sn->activation_count = n->activation_count; sn->last_activated = n->last_activated;
sn->created_at = n->created_at; sn->updated_at = n->updated_at;
sn->background_activation = n->background_activation;
sn->working_memory_weight = n->working_memory_weight;
sn->suppression_count = n->suppression_count; sn->layer_id = n->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
sn->access_ts[i] = n->access_ts[i];
sn->access_head = n->access_head; sn->access_filled = n->access_filled;
sn->wm_anchor = n->wm_anchor; sn->emb = n->emb; sn->emb_dim = n->emb_dim;
}
static void eg_edge_to_store(const EngramEdge* e, StoreEdge* se) {
memset(se, 0, sizeof *se);
se->id = e->id; se->from_id = e->from_id; se->to_id = e->to_id;
se->relation = e->relation; se->metadata = e->metadata;
se->weight = e->weight; se->hebb = e->hebb; se->confidence = e->confidence;
se->created_at = e->created_at; se->updated_at = e->updated_at;
se->last_fired = e->last_fired; se->inhibitory = e->inhibitory;
se->layer_id = e->layer_id;
}
/* Structural-mutation hooks. Callers guard with `if (engram_store_enabled())`;
* these also null-check g_engram_store so a mutation before boot is a safe no-op. */
static void eg_store_put_node(const EngramNode* n) {
if (!g_engram_store || !n || !n->id) return;
StoreNode sn; eg_node_to_store(n, &sn);
store_put_node(g_engram_store, &sn);
}
static void eg_store_put_edge(const EngramEdge* e) {
if (!g_engram_store || !e || !e->id) return;
StoreEdge se; eg_edge_to_store(e, &se);
store_put_edge(g_engram_store, &se);
}
/* Resident-load callbacks: StoreNode/StoreEdge → a fresh EngramNode/EngramEdge
* appended to the in-RAM graph. Mirrors engram_load's field set (minus the
* boot-time WM laundering the store already holds the authoritative weights). */
static void eg_load_node_cb(const StoreNode* sn, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_nodes();
EngramNode* n = &g->nodes[g->node_count];
memset(n, 0, sizeof *n);
n->id = el_strdup_persist(sn->id ? sn->id : "");
n->content = el_strdup_persist(sn->content ? sn->content : "");
n->node_type = el_strdup_persist(sn->node_type && *sn->node_type ? sn->node_type : "Memory");
n->label = el_strdup_persist(sn->label ? sn->label : "");
n->tier = el_strdup_persist(sn->tier && *sn->tier ? sn->tier : "Working");
n->tags = el_strdup_persist(sn->tags ? sn->tags : "");
n->metadata = el_strdup_persist(sn->metadata && *sn->metadata ? sn->metadata : "{}");
n->salience = sn->salience; n->importance = sn->importance;
n->confidence = sn->confidence; n->temporal_decay_rate = sn->temporal_decay_rate;
n->activation_count = sn->activation_count; n->last_activated = sn->last_activated;
n->created_at = sn->created_at; n->updated_at = sn->updated_at;
n->background_activation = sn->background_activation;
n->working_memory_weight = sn->working_memory_weight;
n->suppression_count = sn->suppression_count; n->layer_id = sn->layer_id;
for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++)
n->access_ts[i] = sn->access_ts[i];
n->access_head = sn->access_head; n->access_filled = sn->access_filled;
n->wm_anchor = sn->wm_anchor;
if (sn->emb && sn->emb_dim > 0) {
n->emb = malloc(sizeof(float) * (size_t)sn->emb_dim);
if (n->emb) { memcpy(n->emb, sn->emb, sizeof(float) * (size_t)sn->emb_dim);
n->emb_dim = sn->emb_dim; }
}
int64_t idx = g->node_count; g->node_count++;
if (n->id && *n->id) engram_idmap_put(g, n->id, idx);
}
static void eg_load_edge_cb(const StoreEdge* se, void* ctx) {
EngramStore* g = (EngramStore*)ctx;
engram_grow_edges();
EngramEdge* e = &g->edges[g->edge_count];
memset(e, 0, sizeof *e);
e->id = el_strdup_persist(se->id ? se->id : "");
e->from_id = el_strdup_persist(se->from_id ? se->from_id : "");
e->to_id = el_strdup_persist(se->to_id ? se->to_id : "");
e->relation = el_strdup_persist(se->relation && *se->relation ? se->relation : "associate");
e->metadata = el_strdup_persist(se->metadata && *se->metadata ? se->metadata : "{}");
e->weight = se->weight; e->hebb = se->hebb; e->confidence = se->confidence;
e->created_at = se->created_at; e->updated_at = se->updated_at;
e->last_fired = se->last_fired; e->inhibitory = se->inhibitory;
e->layer_id = se->layer_id;
g->edge_count++;
}
static void eg_load_layer_cb(EngramStore* g, const StoreLayer* L) {
if (!L->name) return;
for (size_t i = 0; i < g->layer_count; i++) /* upsert by id */
if (g->layers[i].layer_id == L->layer_id) return; /* canonical already seeded */
if (g->layer_count >= g->layer_capacity) {
size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16;
EngramLayer* nl = realloc(g->layers, nc * sizeof(EngramLayer));
if (!nl) return;
g->layers = nl; g->layer_capacity = nc;
}
g->layers[g->layer_count++] = (EngramLayer){
.layer_id = L->layer_id,
.name = el_strdup_persist(L->name),
.activation_priority = L->activation_priority,
.suppressible = L->suppressible,
.transparent = L->transparent,
.injectable = L->injectable
};
}
/* Clear the resident graph so the store becomes the sole source of truth on boot
* (mirrors engram_load's reset). */
static void eg_reset_resident(EngramStore* g) {
for (int64_t i = 0; i < g->node_count; i++) {
free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type);
free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags);
free(g->nodes[i].metadata);
free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0;
}
g->node_count = 0;
for (int64_t i = 0; i < g->edge_count; i++) {
free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id);
free(g->edges[i].relation); free(g->edges[i].metadata);
}
g->edge_count = 0;
engram_idmap_free(g);
engram_adj_free(g);
}
/* engram_store_boot(data_dir) — open (import-once or WAL-replay) the durable
* paged store and load it whole into RAM (Phase 1). No-op / returns 0 when the
* flag is off. Returns 1 on success. Idempotent (a second call is a no-op). */
el_val_t engram_store_boot(el_val_t data_dir) {
if (!engram_store_enabled()) return (el_val_t)0;
if (g_engram_store) return (el_val_t)1;
const char* d = EL_CSTR(data_dir);
if (!d || !*d) return (el_val_t)0;
g_engram_store = engram_open(d);
if (!g_engram_store) return (el_val_t)0;
EngramStore* g = engram_get();
eg_reset_resident(g);
store_scan_nodes(g_engram_store, eg_load_node_cb, g);
store_scan_edges(g_engram_store, eg_load_edge_cb, g);
StoreLayer* ls = NULL; size_t ln = 0;
if (store_list_layers(g_engram_store, &ls, &ln) == 0) {
for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]);
store_layers_free(ls, ln);
}
g->adj_dirty = 1;
return (el_val_t)1;
}
/* engram_store_checkpoint() — flush dirty pages + advance the checkpoint LSN.
* The storeJSON export path stays engram_save (the JSON is an export artifact). */
el_val_t engram_store_checkpoint(void) {
if (!engram_store_enabled() || !g_engram_store) return (el_val_t)0;
return (el_val_t)(int64_t)(engram_checkpoint(g_engram_store) == 0 ? 1 : 0);
}
/* engram_store_close() — checkpoint + close (used at shutdown / by tests). */
el_val_t engram_store_close(void) {
if (!g_engram_store) return (el_val_t)0;
int r = engram_close(g_engram_store);
g_engram_store = NULL;
return (el_val_t)(int64_t)(r == 0 ? 1 : 0);
}
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
EngramStore* g = engram_get();
engram_grow_nodes();
@@ -7296,6 +7494,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
g->node_count++;
engram_idmap_put(g, n->id, new_idx);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7425,6 +7624,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
g->node_count++;
engram_idmap_put(g, n->id, new_idx_full);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7495,6 +7695,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
g->node_count++;
engram_idmap_put(g, n->id, new_idx_layered);
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7642,6 +7843,12 @@ void engram_forget(el_val_t node_id) {
EngramStore* g = engram_get();
int64_t idx = engram_find_node_index(sid);
if (idx < 0) return;
/* Mirror the removal into the durable store BEFORE the shift-delete frees
* the incident edges' ids (node tombstone; edge records are reclaimed at
* compaction M5). Node-level FORGET matches the existing WAL semantics. */
if (engram_store_enabled() && g_engram_store) {
store_forget(g_engram_store, sid);
}
/* Free node strings */
EngramNode* n = &g->nodes[idx];
free(n->id); free(n->content); free(n->node_type); free(n->label);
@@ -7980,6 +8187,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
if (engram_store_enabled()) eg_store_put_edge(e);
}
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) {
+86 -3
View File
@@ -1137,12 +1137,95 @@ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max)
}
uint64_t store_page_count(const EngramPagedStore* s){ return s ? s->page_count : 0; }
/* ── M3: full live enumeration (boundary-clean; StoreNode/StoreEdge out only) ──
* Page-walk every NODE/EDGE page, emitting each DISTINCT live record. A re-put
* leaves several live records for one id (apply_node_put appends; reads dedup),
* so we track ids already emitted by their 64-bit id-hash — the same key the
* primary B+-tree uses (design §2.4) — and fetch the canonical latest-live via
* the point-read path so a scan and a get agree exactly. Used by the caller
* (el_runtime) to load the whole store resident at boot and to export JSON. */
typedef struct { uint64_t* h; size_t n, cap; } U64Set;
static int u64set_add(U64Set* s, uint64_t v){ /* 1 = newly added, 0 = present */
if ((s->n + 1) * 4 >= s->cap * 3){
size_t nc = s->cap ? s->cap * 2 : 1024;
uint64_t* nh = (uint64_t*)calloc(nc, sizeof(uint64_t));
if (!nh) return 1; /* degrade rather than crash */
for (size_t i = 0; i < s->cap; i++){
uint64_t k = s->h[i];
if (k){ size_t j = k & (nc - 1); while (nh[j]) j = (j + 1) & (nc - 1); nh[j] = k; }
}
free(s->h); s->h = nh; s->cap = nc;
}
uint64_t k = v ? v : 1; /* 0 reserved as empty slot */
size_t j = k & (s->cap - 1);
while (s->h[j]){ if (s->h[j] == k) return 0; j = (j + 1) & (s->cap - 1); }
s->h[j] = k; s->n++; return 1;
}
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_NODE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreNode cand; node_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreNode canon;
if (store_get_node(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_node_free(&canon);
}
}
store_node_free(&cand);
}
}
free(seen.h);
return count;
}
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
if (!s || !cb) return -1;
U64Set seen = {0, 0, 0};
uint8_t buf[STORE_PAGE_SIZE];
int count = 0;
for (uint64_t pg = 2; pg < s->page_count; pg++){
if (page_read(s, pg, buf) != 0) continue;
if (buf[8] != STORE_PT_EDGE) continue;
int ns = slp_count(buf);
for (int i = 0; i < ns; i++){
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
if (fl != SLOT_LIVE) continue;
uint8_t* body; size_t blen; int live;
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
StoreEdge cand; edge_parse(body, blen, &cand); free(body);
if (cand.id && u64set_add(&seen, id_hash(cand.id))){
StoreEdge canon;
if (store_get_edge(s, cand.id, &canon) == 1){
cb(&canon, ctx); count++;
store_edge_free(&canon);
}
}
store_edge_free(&cand);
}
}
free(seen.h);
return count;
}
/* ══════════════════════════════════════════════════════════════════════════════
* M2 — WAL + write-back buffer pool + checkpoint + crash recovery + legacy import
*
* Durability model (design §2.2/§4, ARIES-lite):
* • Buffer pool is WRITE-BACK, no-steal: a mutation dirties a page in RAM; the
* page reaches engram.store ONLY at a checkpoint. So after a crash the store
* page reaches neuron.egm ONLY at a checkpoint. So after a crash the store
* file reflects exactly `last_checkpoint_lsn`, and everything since lives in
* the WAL. This is what makes the WAL load-bearing (durability = fsync'd WAL,
* not the page).
@@ -2018,8 +2101,8 @@ static int import_snapshot(EngramPagedStore* s, const char* path){
EngramPagedStore* engram_open(const char* data_dir){
if (!data_dir) return NULL;
char store_path[1200], wal_path[1200], snap_path[1200];
snprintf(store_path, sizeof store_path, "%s/engram.store", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/engram.wal", data_dir);
snprintf(store_path, sizeof store_path, "%s/neuron.egm", data_dir);
snprintf(wal_path, sizeof wal_path, "%s/neuron.wal", data_dir);
snprintf(snap_path, sizeof snap_path, "%s/snapshot.json", data_dir);
EngramWalSync sync = ENGRAM_WAL_GROUP;
+13 -2
View File
@@ -134,7 +134,7 @@ uint64_t store_page_count(const EngramPagedStore* s);
/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
*
* The durable engram is `engram.store` (paged) fronted by `engram.wal`
* The durable engram is `neuron.egm` (paged) fronted by `neuron.wal`
* (append-only). A mutation is durable once its WAL record is fsync'd
* (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
* store only at a checkpoint, so the store file on disk always reflects a
@@ -158,7 +158,7 @@ typedef struct StoreLayer {
int tombstoned;
} StoreLayer;
/* Boot the durable engram in `data_dir` (holds engram.store + engram.wal). If the
/* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the
* store is absent but a legacy snapshot.json exists, it is imported ONCE into a
* fresh store; thereafter the store is authoritative and JSON is never read again.
* On open, the WAL is replayed to recover any post-checkpoint mutations. */
@@ -194,6 +194,17 @@ int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id
/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
int store_forget(EngramPagedStore* s, const char* id);
/* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ──
* Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a
* borrowed view (the engine frees it after cb returns the callback must copy
* anything it keeps). De-duplicated by id (canonical latest-live per id, matching
* point-read semantics). Returns the count emitted, or <0 on error. The engine
* hands out StoreNode/StoreEdge only it never sees a soul struct (design §10). */
typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx);
typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx);
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx);
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx);
/* Introspection / test hooks. */
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);