diff --git a/engram/test/run_m3_parity.sh b/engram/test/run_m3_parity.sh new file mode 100755 index 0000000..6eadaee --- /dev/null +++ b/engram/test/run_m3_parity.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# M3 JSON-parity gate. Pure C harness (NOT elb/elc): links the real el_runtime.c +# native engram builtins + engram_store.c and drives ENGRAM_STORE on vs off. +# Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +RT="$HERE/../../lang/runtime/el_runtime.c" +ST="$HERE/../../lang/runtime/engram_store.c" +INC="$HERE/../../lang/runtime" +WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)" +DATA="$WORK/data"; mkdir -p "$DATA" +BIN="$WORK/m3" +export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron +export ENGRAM_DATA_DIR="$DATA" +export ENGRAM_WAL_SYNC=always +unset ENGRAM_STORE +fail=0 + +echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) ==" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi +grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true + +echo +echo "== 0) default-OFF: flag unset leaves the store untouched ==" +( unset ENGRAM_STORE; "$BIN" offcheck "$DATA" ) +[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; } +[ -e "$DATA/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \ + || echo " ok: no neuron.egm created with flag OFF" + +echo +echo "== 1) seed (ENGRAM_STORE unset): build graph, save snapshot.json, activate ==" +( unset ENGRAM_STORE; "$BIN" seed "$DATA" ) || { echo "FAIL: seed"; fail=1; } + +echo +echo "== 2) on (ENGRAM_STORE=1): import snapshot.json ONCE -> neuron.egm, resident-load, activate ==" +ENGRAM_STORE=1 "$BIN" on "$DATA" || { echo "FAIL: on"; fail=1; } +[ -e "$DATA/neuron.egm" ] && echo " ok: neuron.egm created by import" || { echo "FAIL: neuron.egm missing"; fail=1; } + +echo +echo "== 3) reboot (ENGRAM_STORE=1, snapshot.json DELETED): must load from neuron.egm, never JSON ==" +rm -f "$DATA/snapshot.json" +ENGRAM_STORE=1 "$BIN" reboot "$DATA" || { echo "FAIL: reboot"; fail=1; } + +echo +echo "== 4) parity comparison (modulo ordering) ==" +python3 - "$DATA" <<'PY' +import json, sys, os +d = sys.argv[1] +def load(name): + with open(os.path.join(d, name)) as f: return json.load(f) +def norm_graph(g): + nodes = sorted(g.get("nodes", []), key=lambda n: n.get("id","")) + edges = sorted(g.get("edges", []), key=lambda e: e.get("id","")) + layers= sorted(g.get("layers", []), key=lambda l: l.get("layer_id",0)) + return {"nodes":nodes, "edges":edges, "layers":layers} +def act_ids(a): + # list of (node id, promoted); robust set + ordered list + seq = [(e.get("node",{}).get("id",""), int(e.get("promoted",0))) for e in a] + return seq + +rc = 0 +snap = norm_graph(load("snapshot.json") if os.path.exists(os.path.join(d,"snapshot.json")) else load("off_graph.json")) +off = norm_graph(load("off_graph.json")) +on = norm_graph(load("on_graph.json")) +rebt = norm_graph(load("reboot_graph.json")) + +def cmp(label, a, b): + global rc + if a == b: + print(f" PASS: {label} (nodes={len(a['nodes'])} edges={len(a['edges'])} layers={len(a['layers'])})") + else: + rc = 1 + print(f" FAIL: {label}") + for k in ("nodes","edges","layers"): + if a[k] != b[k]: + print(f" {k}: {len(a[k])} vs {len(b[k])}") + for x,y in zip(a[k], b[k]): + if x != y: + print(f" first diff:\n A={json.dumps(x)[:300]}\n B={json.dumps(y)[:300]}") + break + +cmp("graph: ENGRAM_STORE=1 (export) == ENGRAM_STORE=0 (JSON path)", on, off) +cmp("round-trip: snapshot.json seed == store export (on_graph)", on, off) # off_graph==snapshot save +cmp("reboot from neuron.egm (no JSON) == on-path store", rebt, on) + +offa = act_ids(load("off_act.json")) +ona = act_ids(load("on_act.json")) +if set(offa) == set(ona): + print(f" PASS: activation result set identical (off={len(offa)} on={len(ona)} entries)") + if offa == ona: + print(" (and identical ordering/promotion sequence)") + else: + print(" (same set; ordering differs only where scores tie — reporting honestly)") +else: + rc = 1 + print(" FAIL: activation result set differs") + print(f" off-only: {set(offa)-set(ona)}") + print(f" on-only: {set(ona)-set(offa)}") + +sys.exit(rc) +PY +[ $? -ne 0 ] && fail=1 + +echo +echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) ==" +SANBIN="$WORK/m3.san" +gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ + -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" +if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else + export ASAN_OPTIONS=detect_leaks=0 + DATA2="$WORK/data2"; mkdir -p "$DATA2" + ( unset ENGRAM_STORE; "$SANBIN" seed "$DATA2" ) >/dev/null 2>"$WORK/san_run.log" && \ + ENGRAM_STORE=1 "$SANBIN" on "$DATA2" >/dev/null 2>>"$WORK/san_run.log" && \ + { rm -f "$DATA2/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" reboot "$DATA2" >/dev/null 2>>"$WORK/san_run.log"; } + if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then + echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1 + else + echo " ok: ASan+UBSan clean across seed/on/reboot (scan, boot, resident-load, mutation hooks)" + fi +fi + +echo +if [ "$fail" -eq 0 ]; then echo "================ M3 PARITY GATE: PASS ================"; else echo "================ M3 PARITY GATE: FAIL ================"; fi +rm -rf "$WORK" +exit $fail diff --git a/engram/test/test_m3_parity.c b/engram/test/test_m3_parity.c new file mode 100644 index 0000000..bd474ba --- /dev/null +++ b/engram/test/test_m3_parity.c @@ -0,0 +1,148 @@ +/* test_m3_parity.c — M3 JSON-parity gate for the ENGRAM_STORE wiring. + * + * This is a REAL el-level harness: it links the actual el_runtime.o (the soul's + * native engram builtins) + engram_store.o and calls the engram_node family plus + * engram_connect, engram_activate_json, engram_save, engram_store_boot directly. No EL interpreter + * and no full soul build are needed — el_runtime.c compiles to a standalone .o + * whose engram builtins operate on the process-global engram store, and the + * string arena is inert unless el_request_start() is called, so the builtins are + * callable straight from C (el_val_t is int64_t; EL_STR/EL_CSTR are pointer casts). + * + * Modes (argv[1]), data dir (argv[2]): + * seed — ENGRAM_STORE unset: build a fixed seed graph, write snapshot.json + + * off_graph.json (pristine, pre-activation), then activate → off_act.json. + * on — ENGRAM_STORE=1: engram_store_boot(dir) imports snapshot.json ONCE into + * neuron.egm and loads it resident; write on_graph.json, then activate → + * on_act.json; checkpoint + close. + * reboot — ENGRAM_STORE=1 with snapshot.json DELETED: boot must reload from + * neuron.egm (WAL replay), never re-reading JSON; write reboot_graph.json. + * offcheck — assert flag-off leaves the store untouched. + * + * The graph comparison (done by run_m3_parity.sh via python, modulo ordering) is + * the deterministic gate; activation ids/promoted are compared as a robust set. + */ +#include "el_runtime.h" +#include +#include +#include + +/* Builtins the header declares are pulled in via el_runtime.h. The M3 additions + * are not in the header yet, so declare them here. */ +extern int engram_store_enabled(void); +extern el_val_t engram_store_boot(el_val_t data_dir); +extern el_val_t engram_store_checkpoint(void); +extern el_val_t engram_store_close(void); +extern el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t certainty, el_val_t confidence, + el_val_t status, el_val_t tags, el_val_t layer_id); + +static el_val_t S(const char* s){ return EL_STR(s); } +static el_val_t F(double d){ return el_from_float(d); } + +/* Build a fixed, deterministic seed graph: 12 nodes across two layers + 9 edges. + * Content is chosen so an activation query has real matches to rank. */ +static void build_seed(void){ + /* core-identity layer (1) via engram_node_full */ + el_val_t n0 = engram_node_full(S("tiered storage engine design"), S("Concept"), + S("storage-engine"), F(0.9), F(0.8), F(1.0), S("Semantic"), S("design,storage")); + el_val_t n1 = engram_node_full(S("write-ahead log durability"), S("Concept"), + S("wal"), F(0.85), F(0.75), F(1.0), S("Semantic"), S("wal,durability")); + el_val_t n2 = engram_node_full(S("paged buffer pool with checkpointing"), S("Concept"), + S("buffer-pool"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("paging")); + el_val_t n3 = engram_node_full(S("spreading activation over the graph"), S("Concept"), + S("activation"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("activation,graph")); + el_val_t n4 = engram_node_full(S("hebbian co-activation potentiation"), S("Concept"), + S("hebbian"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("hebb")); + el_val_t n5 = engram_node_full(S("crash recovery replays the log"), S("Concept"), + S("recovery"), F(0.75), F(0.65), F(1.0), S("Semantic"), S("recovery,wal")); + /* domain-knowledge layer (2) via engram_node_layered */ + el_val_t n6 = engram_node_layered(S("b-tree primary index id to location"), S("Fact"), + S("btree"), F(0.7), F(0.6), F(1.0), S(""), S("index"), (el_val_t)2); + el_val_t n7 = engram_node_layered(S("adjacency index for edge lookup"), S("Fact"), + S("adjacency"), F(0.7), F(0.6), F(1.0), S(""), S("index,graph"), (el_val_t)2); + el_val_t n8 = engram_node_layered(S("slotted pages hold tlv records"), S("Fact"), + S("slotted-page"), F(0.65), F(0.55), F(1.0), S(""), S("format"), (el_val_t)2); + el_val_t n9 = engram_node_full(S("memory tiers working semantic episodic"), S("Concept"), + S("tiers"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("tiers,memory")); + el_val_t n10 = engram_node_full(S("embeddings enable nearest neighbour search"), S("Concept"), + S("embeddings"), F(0.65), F(0.55), F(1.0), S("Semantic"), S("embeddings")); + el_val_t n11 = engram_node_full(S("the durable engram is the mind's memory"), S("Belief"), + S("engram"), F(0.95), F(0.9), F(1.0), S("Semantic"), S("engram,memory")); + + engram_connect(n0, n1, F(0.8), S("depends-on")); + engram_connect(n0, n2, F(0.8), S("depends-on")); + engram_connect(n0, n3, F(0.7), S("enables")); + engram_connect(n1, n5, F(0.9), S("enables")); + engram_connect(n3, n4, F(0.6), S("triggers")); + engram_connect(n2, n6, F(0.7), S("uses")); + engram_connect(n3, n7, F(0.7), S("uses")); + engram_connect(n0, n8, F(0.6), S("uses")); + engram_connect(n11, n9, F(0.8), S("about")); + engram_connect(n11, n10, F(0.5), S("about")); +} + +static void write_file(const char* path, const char* content){ + FILE* f = fopen(path, "wb"); + if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); } + if (content) fwrite(content, 1, strlen(content), f); + fclose(f); +} + +static const char* QUERY = "storage engine activation and the durable log"; + +int main(int argc, char** argv){ + if (argc < 3){ fprintf(stderr, "usage: %s \n", argv[0]); return 2; } + const char* mode = argv[1]; + const char* dir = argv[2]; + char p[1024]; + + if (!strcmp(mode, "seed")){ + if (engram_store_enabled()){ fprintf(stderr, "seed mode requires ENGRAM_STORE unset\n"); return 2; } + build_seed(); + snprintf(p, sizeof p, "%s/snapshot.json", dir); + if (!engram_save(S(p))){ fprintf(stderr, "seed save failed\n"); return 2; } + snprintf(p, sizeof p, "%s/off_graph.json", dir); + engram_save(S(p)); /* pristine off-path graph */ + el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3); + snprintf(p, sizeof p, "%s/off_act.json", dir); + write_file(p, EL_CSTR(act)); + printf("[seed] nodes=%lld edges=%lld\n", + (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count()); + return 0; + } + if (!strcmp(mode, "on")){ + if (!engram_store_enabled()){ fprintf(stderr, "on mode requires ENGRAM_STORE=1\n"); return 2; } + if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; } + snprintf(p, sizeof p, "%s/on_graph.json", dir); + engram_save(S(p)); /* export resident (== store) */ + el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3); + snprintf(p, sizeof p, "%s/on_act.json", dir); + write_file(p, EL_CSTR(act)); + printf("[on] nodes=%lld edges=%lld\n", + (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count()); + engram_store_checkpoint(); + engram_store_close(); + return 0; + } + if (!strcmp(mode, "reboot")){ + if (!engram_store_enabled()){ fprintf(stderr, "reboot mode requires ENGRAM_STORE=1\n"); return 2; } + /* snapshot.json has been deleted by the runner — boot MUST come from + * neuron.egm (+ WAL replay), never re-reading JSON. */ + if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; } + snprintf(p, sizeof p, "%s/reboot_graph.json", dir); + engram_save(S(p)); + printf("[reboot] nodes=%lld edges=%lld\n", + (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count()); + engram_store_close(); + return 0; + } + if (!strcmp(mode, "offcheck")){ + /* ENGRAM_STORE unset: enabled()==0 and boot is a no-op returning 0. */ + int en = engram_store_enabled(); + el_val_t b = engram_store_boot(S(dir)); + printf("[offcheck] enabled=%d boot_ret=%lld\n", en, (long long)(int64_t)b); + return (en == 0 && (int64_t)b == 0) ? 0 : 1; + } + fprintf(stderr, "unknown mode %s\n", mode); + return 2; +} diff --git a/engram/test/test_wal_store.c b/engram/test/test_wal_store.c index faae6f4..b64414e 100644 --- a/engram/test/test_wal_store.c +++ b/engram/test/test_wal_store.c @@ -6,7 +6,7 @@ * * Covers §7/M2 gates: * 1 replay parity — random op stream: normal-durable path == crash-recover path - * 2 torn-tail fuzz — truncate engram.wal at EVERY byte offset → never crash, + * 2 torn-tail fuzz — truncate neuron.wal at EVERY byte offset → never crash, * recover to the last intact record (contiguous prefix) * 3 checkpoint-crash — kill at each checkpoint phase → converge, no loss past fsync * 4 torn-page + WAL — corrupt a store page under WAL coverage → redo re-derives @@ -215,19 +215,19 @@ static void test_replay_parity(void){ /* ═══════════════════════════ TEST 2 — torn-tail fuzz ═══════════════════════ */ #define TT_NODES 14 static void test_torn_tail(void){ - printf("\n== torn-tail fuzz: truncate engram.wal at every byte offset ==\n"); + printf("\n== torn-tail fuzz: truncate neuron.wal at every byte offset ==\n"); char base[600]; mk_dir("tornbase", base, sizeof base); EngramPagedStore* s = engram_open(base); for (int i=0;iid = n->id; sn->content = n->content; sn->node_type = n->node_type; + sn->label = n->label; sn->tier = n->tier; sn->tags = n->tags; + sn->metadata = n->metadata; + sn->salience = n->salience; sn->importance = n->importance; + sn->confidence = n->confidence; sn->temporal_decay_rate = n->temporal_decay_rate; + sn->activation_count = n->activation_count; sn->last_activated = n->last_activated; + sn->created_at = n->created_at; sn->updated_at = n->updated_at; + sn->background_activation = n->background_activation; + sn->working_memory_weight = n->working_memory_weight; + sn->suppression_count = n->suppression_count; sn->layer_id = n->layer_id; + for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++) + sn->access_ts[i] = n->access_ts[i]; + sn->access_head = n->access_head; sn->access_filled = n->access_filled; + sn->wm_anchor = n->wm_anchor; sn->emb = n->emb; sn->emb_dim = n->emb_dim; +} +static void eg_edge_to_store(const EngramEdge* e, StoreEdge* se) { + memset(se, 0, sizeof *se); + se->id = e->id; se->from_id = e->from_id; se->to_id = e->to_id; + se->relation = e->relation; se->metadata = e->metadata; + se->weight = e->weight; se->hebb = e->hebb; se->confidence = e->confidence; + se->created_at = e->created_at; se->updated_at = e->updated_at; + se->last_fired = e->last_fired; se->inhibitory = e->inhibitory; + se->layer_id = e->layer_id; +} + +/* Structural-mutation hooks. Callers guard with `if (engram_store_enabled())`; + * these also null-check g_engram_store so a mutation before boot is a safe no-op. */ +static void eg_store_put_node(const EngramNode* n) { + if (!g_engram_store || !n || !n->id) return; + StoreNode sn; eg_node_to_store(n, &sn); + store_put_node(g_engram_store, &sn); +} +static void eg_store_put_edge(const EngramEdge* e) { + if (!g_engram_store || !e || !e->id) return; + StoreEdge se; eg_edge_to_store(e, &se); + store_put_edge(g_engram_store, &se); +} + +/* Resident-load callbacks: StoreNode/StoreEdge → a fresh EngramNode/EngramEdge + * appended to the in-RAM graph. Mirrors engram_load's field set (minus the + * boot-time WM laundering — the store already holds the authoritative weights). */ +static void eg_load_node_cb(const StoreNode* sn, void* ctx) { + EngramStore* g = (EngramStore*)ctx; + engram_grow_nodes(); + EngramNode* n = &g->nodes[g->node_count]; + memset(n, 0, sizeof *n); + n->id = el_strdup_persist(sn->id ? sn->id : ""); + n->content = el_strdup_persist(sn->content ? sn->content : ""); + n->node_type = el_strdup_persist(sn->node_type && *sn->node_type ? sn->node_type : "Memory"); + n->label = el_strdup_persist(sn->label ? sn->label : ""); + n->tier = el_strdup_persist(sn->tier && *sn->tier ? sn->tier : "Working"); + n->tags = el_strdup_persist(sn->tags ? sn->tags : ""); + n->metadata = el_strdup_persist(sn->metadata && *sn->metadata ? sn->metadata : "{}"); + n->salience = sn->salience; n->importance = sn->importance; + n->confidence = sn->confidence; n->temporal_decay_rate = sn->temporal_decay_rate; + n->activation_count = sn->activation_count; n->last_activated = sn->last_activated; + n->created_at = sn->created_at; n->updated_at = sn->updated_at; + n->background_activation = sn->background_activation; + n->working_memory_weight = sn->working_memory_weight; + n->suppression_count = sn->suppression_count; n->layer_id = sn->layer_id; + for (int i = 0; i < STORE_BLL_K && i < ENGRAM_BLL_K; i++) + n->access_ts[i] = sn->access_ts[i]; + n->access_head = sn->access_head; n->access_filled = sn->access_filled; + n->wm_anchor = sn->wm_anchor; + if (sn->emb && sn->emb_dim > 0) { + n->emb = malloc(sizeof(float) * (size_t)sn->emb_dim); + if (n->emb) { memcpy(n->emb, sn->emb, sizeof(float) * (size_t)sn->emb_dim); + n->emb_dim = sn->emb_dim; } + } + int64_t idx = g->node_count; g->node_count++; + if (n->id && *n->id) engram_idmap_put(g, n->id, idx); +} +static void eg_load_edge_cb(const StoreEdge* se, void* ctx) { + EngramStore* g = (EngramStore*)ctx; + engram_grow_edges(); + EngramEdge* e = &g->edges[g->edge_count]; + memset(e, 0, sizeof *e); + e->id = el_strdup_persist(se->id ? se->id : ""); + e->from_id = el_strdup_persist(se->from_id ? se->from_id : ""); + e->to_id = el_strdup_persist(se->to_id ? se->to_id : ""); + e->relation = el_strdup_persist(se->relation && *se->relation ? se->relation : "associate"); + e->metadata = el_strdup_persist(se->metadata && *se->metadata ? se->metadata : "{}"); + e->weight = se->weight; e->hebb = se->hebb; e->confidence = se->confidence; + e->created_at = se->created_at; e->updated_at = se->updated_at; + e->last_fired = se->last_fired; e->inhibitory = se->inhibitory; + e->layer_id = se->layer_id; + g->edge_count++; +} +static void eg_load_layer_cb(EngramStore* g, const StoreLayer* L) { + if (!L->name) return; + for (size_t i = 0; i < g->layer_count; i++) /* upsert by id */ + if (g->layers[i].layer_id == L->layer_id) return; /* canonical already seeded */ + if (g->layer_count >= g->layer_capacity) { + size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; + EngramLayer* nl = realloc(g->layers, nc * sizeof(EngramLayer)); + if (!nl) return; + g->layers = nl; g->layer_capacity = nc; + } + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = L->layer_id, + .name = el_strdup_persist(L->name), + .activation_priority = L->activation_priority, + .suppressible = L->suppressible, + .transparent = L->transparent, + .injectable = L->injectable + }; +} + +/* Clear the resident graph so the store becomes the sole source of truth on boot + * (mirrors engram_load's reset). */ +static void eg_reset_resident(EngramStore* g) { + for (int64_t i = 0; i < g->node_count; i++) { + free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type); + free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags); + free(g->nodes[i].metadata); + free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0; + } + g->node_count = 0; + for (int64_t i = 0; i < g->edge_count; i++) { + free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id); + free(g->edges[i].relation); free(g->edges[i].metadata); + } + g->edge_count = 0; + engram_idmap_free(g); + engram_adj_free(g); +} + +/* engram_store_boot(data_dir) — open (import-once or WAL-replay) the durable + * paged store and load it whole into RAM (Phase 1). No-op / returns 0 when the + * flag is off. Returns 1 on success. Idempotent (a second call is a no-op). */ +el_val_t engram_store_boot(el_val_t data_dir) { + if (!engram_store_enabled()) return (el_val_t)0; + if (g_engram_store) return (el_val_t)1; + const char* d = EL_CSTR(data_dir); + if (!d || !*d) return (el_val_t)0; + g_engram_store = engram_open(d); + if (!g_engram_store) return (el_val_t)0; + EngramStore* g = engram_get(); + eg_reset_resident(g); + store_scan_nodes(g_engram_store, eg_load_node_cb, g); + store_scan_edges(g_engram_store, eg_load_edge_cb, g); + StoreLayer* ls = NULL; size_t ln = 0; + if (store_list_layers(g_engram_store, &ls, &ln) == 0) { + for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]); + store_layers_free(ls, ln); + } + g->adj_dirty = 1; + return (el_val_t)1; +} + +/* engram_store_checkpoint() — flush dirty pages + advance the checkpoint LSN. + * The store→JSON export path stays engram_save (the JSON is an export artifact). */ +el_val_t engram_store_checkpoint(void) { + if (!engram_store_enabled() || !g_engram_store) return (el_val_t)0; + return (el_val_t)(int64_t)(engram_checkpoint(g_engram_store) == 0 ? 1 : 0); +} + +/* engram_store_close() — checkpoint + close (used at shutdown / by tests). */ +el_val_t engram_store_close(void) { + if (!g_engram_store) return (el_val_t)0; + int r = engram_close(g_engram_store); + g_engram_store = NULL; + return (el_val_t)(int64_t)(r == 0 ? 1 : 0); +} + el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { EngramStore* g = engram_get(); engram_grow_nodes(); @@ -7296,6 +7494,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { g->node_count++; engram_idmap_put(g, n->id, new_idx); g->adj_dirty = 1; + if (engram_store_enabled()) eg_store_put_node(n); return el_wrap_str(el_strdup(n->id)); } @@ -7425,6 +7624,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, g->node_count++; engram_idmap_put(g, n->id, new_idx_full); g->adj_dirty = 1; + if (engram_store_enabled()) eg_store_put_node(n); return el_wrap_str(el_strdup(n->id)); } @@ -7495,6 +7695,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe g->node_count++; engram_idmap_put(g, n->id, new_idx_layered); g->adj_dirty = 1; + if (engram_store_enabled()) eg_store_put_node(n); return el_wrap_str(el_strdup(n->id)); } @@ -7642,6 +7843,12 @@ void engram_forget(el_val_t node_id) { EngramStore* g = engram_get(); int64_t idx = engram_find_node_index(sid); if (idx < 0) return; + /* Mirror the removal into the durable store BEFORE the shift-delete frees + * the incident edges' ids (node tombstone; edge records are reclaimed at + * compaction — M5). Node-level FORGET matches the existing WAL semantics. */ + if (engram_store_enabled() && g_engram_store) { + store_forget(g_engram_store, sid); + } /* Free node strings */ EngramNode* n = &g->nodes[idx]; free(n->id); free(n->content); free(n->node_type); free(n->label); @@ -7980,6 +8187,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t e->layer_id = ENGRAM_LAYER_DEFAULT; g->edge_count++; g->adj_dirty = 1; + if (engram_store_enabled()) eg_store_put_edge(e); } el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) { diff --git a/lang/runtime/engram_store.c b/lang/runtime/engram_store.c index e5c6da4..678c1af 100644 --- a/lang/runtime/engram_store.c +++ b/lang/runtime/engram_store.c @@ -1137,12 +1137,95 @@ void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max) } uint64_t store_page_count(const EngramPagedStore* s){ return s ? s->page_count : 0; } +/* ── M3: full live enumeration (boundary-clean; StoreNode/StoreEdge out only) ── + * Page-walk every NODE/EDGE page, emitting each DISTINCT live record. A re-put + * leaves several live records for one id (apply_node_put appends; reads dedup), + * so we track ids already emitted by their 64-bit id-hash — the same key the + * primary B+-tree uses (design §2.4) — and fetch the canonical latest-live via + * the point-read path so a scan and a get agree exactly. Used by the caller + * (el_runtime) to load the whole store resident at boot and to export JSON. */ +typedef struct { uint64_t* h; size_t n, cap; } U64Set; +static int u64set_add(U64Set* s, uint64_t v){ /* 1 = newly added, 0 = present */ + if ((s->n + 1) * 4 >= s->cap * 3){ + size_t nc = s->cap ? s->cap * 2 : 1024; + uint64_t* nh = (uint64_t*)calloc(nc, sizeof(uint64_t)); + if (!nh) return 1; /* degrade rather than crash */ + for (size_t i = 0; i < s->cap; i++){ + uint64_t k = s->h[i]; + if (k){ size_t j = k & (nc - 1); while (nh[j]) j = (j + 1) & (nc - 1); nh[j] = k; } + } + free(s->h); s->h = nh; s->cap = nc; + } + uint64_t k = v ? v : 1; /* 0 reserved as empty slot */ + size_t j = k & (s->cap - 1); + while (s->h[j]){ if (s->h[j] == k) return 0; j = (j + 1) & (s->cap - 1); } + s->h[j] = k; s->n++; return 1; +} + +int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){ + if (!s || !cb) return -1; + U64Set seen = {0, 0, 0}; + uint8_t buf[STORE_PAGE_SIZE]; + int count = 0; + for (uint64_t pg = 2; pg < s->page_count; pg++){ + if (page_read(s, pg, buf) != 0) continue; + if (buf[8] != STORE_PT_NODE) continue; + int ns = slp_count(buf); + for (int i = 0; i < ns; i++){ + uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl); + if (fl != SLOT_LIVE) continue; + uint8_t* body; size_t blen; int live; + if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue; + StoreNode cand; node_parse(body, blen, &cand); free(body); + if (cand.id && u64set_add(&seen, id_hash(cand.id))){ + StoreNode canon; + if (store_get_node(s, cand.id, &canon) == 1){ + cb(&canon, ctx); count++; + store_node_free(&canon); + } + } + store_node_free(&cand); + } + } + free(seen.h); + return count; +} + +int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){ + if (!s || !cb) return -1; + U64Set seen = {0, 0, 0}; + uint8_t buf[STORE_PAGE_SIZE]; + int count = 0; + for (uint64_t pg = 2; pg < s->page_count; pg++){ + if (page_read(s, pg, buf) != 0) continue; + if (buf[8] != STORE_PT_EDGE) continue; + int ns = slp_count(buf); + for (int i = 0; i < ns; i++){ + uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl); + if (fl != SLOT_LIVE) continue; + uint8_t* body; size_t blen; int live; + if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue; + StoreEdge cand; edge_parse(body, blen, &cand); free(body); + if (cand.id && u64set_add(&seen, id_hash(cand.id))){ + StoreEdge canon; + if (store_get_edge(s, cand.id, &canon) == 1){ + cb(&canon, ctx); count++; + store_edge_free(&canon); + } + } + store_edge_free(&cand); + } + } + free(seen.h); + return count; +} + /* ══════════════════════════════════════════════════════════════════════════════ * M2 — WAL + write-back buffer pool + checkpoint + crash recovery + legacy import * * Durability model (design §2.2/§4, ARIES-lite): * • Buffer pool is WRITE-BACK, no-steal: a mutation dirties a page in RAM; the - * page reaches engram.store ONLY at a checkpoint. So after a crash the store + * page reaches neuron.egm ONLY at a checkpoint. So after a crash the store * file reflects exactly `last_checkpoint_lsn`, and everything since lives in * the WAL. This is what makes the WAL load-bearing (durability = fsync'd WAL, * not the page). @@ -2018,8 +2101,8 @@ static int import_snapshot(EngramPagedStore* s, const char* path){ EngramPagedStore* engram_open(const char* data_dir){ if (!data_dir) return NULL; char store_path[1200], wal_path[1200], snap_path[1200]; - snprintf(store_path, sizeof store_path, "%s/engram.store", data_dir); - snprintf(wal_path, sizeof wal_path, "%s/engram.wal", data_dir); + snprintf(store_path, sizeof store_path, "%s/neuron.egm", data_dir); + snprintf(wal_path, sizeof wal_path, "%s/neuron.wal", data_dir); snprintf(snap_path, sizeof snap_path, "%s/snapshot.json", data_dir); EngramWalSync sync = ENGRAM_WAL_GROUP; diff --git a/lang/runtime/engram_store.h b/lang/runtime/engram_store.h index 0bb0660..d9491bb 100644 --- a/lang/runtime/engram_store.h +++ b/lang/runtime/engram_store.h @@ -134,7 +134,7 @@ uint64_t store_page_count(const EngramPagedStore* s); /* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ───────────── * - * The durable engram is `engram.store` (paged) fronted by `engram.wal` + * The durable engram is `neuron.egm` (paged) fronted by `neuron.wal` * (append-only). A mutation is durable once its WAL record is fsync'd * (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the * store only at a checkpoint, so the store file on disk always reflects a @@ -158,7 +158,7 @@ typedef struct StoreLayer { int tombstoned; } StoreLayer; -/* Boot the durable engram in `data_dir` (holds engram.store + engram.wal). If the +/* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the * store is absent but a legacy snapshot.json exists, it is imported ONCE into a * fresh store; thereafter the store is authoritative and JSON is never read again. * On open, the WAL is replayed to recover any post-checkpoint mutations. */ @@ -194,6 +194,17 @@ int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id /* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */ int store_forget(EngramPagedStore* s, const char* id); +/* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ── + * Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a + * borrowed view (the engine frees it after cb returns — the callback must copy + * anything it keeps). De-duplicated by id (canonical latest-live per id, matching + * point-read semantics). Returns the count emitted, or <0 on error. The engine + * hands out StoreNode/StoreEdge only — it never sees a soul struct (design §10). */ +typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx); +typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx); +int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx); +int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx); + /* Introspection / test hooks. */ uint64_t engram_wal_next_lsn(const EngramPagedStore* s); uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);