diff --git a/engram/test/run_m35_hebb_persist.sh b/engram/test/run_m35_hebb_persist.sh new file mode 100755 index 0000000..1ce7dd3 --- /dev/null +++ b/engram/test/run_m35_hebb_persist.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# M3.5 PRE-FLIP GATE. Pure C harness (NOT elb/elc): links the real el_runtime.c +# native engram builtins + engram_store.c and proves activation-time field +# mutations (edge hebb, node activation_count, WM weight) persist through a +# checkpoint and survive a reboot from neuron.egm with snapshot.json DELETED. +# Writes ONLY under a throwaway /tmp dir with a throwaway HOME. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +RT="$HERE/../../lang/runtime/el_runtime.c" +ST="$HERE/../../lang/runtime/engram_store.c" +INC="$HERE/../../lang/runtime" +WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)" +BIN="$WORK/m35" +export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron +export ENGRAM_WAL_SYNC=always +unset ENGRAM_STORE +fail=0 + +echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) ==" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi + +echo +echo "== 0) flag-OFF: seed+activate+checkpoint must NOT touch the store ==" +DOFF="$WORK/off"; mkdir -p "$DOFF" +( unset ENGRAM_STORE; "$BIN" offcheck "$DOFF" ) +[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; } +[ -e "$DOFF/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \ + || echo " ok: no neuron.egm created with flag OFF" + +echo +echo "== 1) POSITIVE: ENGRAM_STORE=1 seed -> activate -> checkpoint(field-persist) -> close ==" +DPOS="$WORK/pos"; mkdir -p "$DPOS" +ENGRAM_STORE=1 "$BIN" pos_seed "$DPOS" || { echo "FAIL: pos_seed"; fail=1; } +[ -e "$DPOS/neuron.egm" ] && echo " ok: neuron.egm created" || { echo "FAIL: neuron.egm missing"; fail=1; } + +echo +echo "== 2) reboot from neuron.egm with snapshot.json DELETED (must never read JSON) ==" +rm -f "$DPOS/snapshot.json" +ENGRAM_STORE=1 "$BIN" pos_reboot "$DPOS" || { echo "FAIL: pos_reboot"; fail=1; } + +echo +echo "== 3) NEGATIVE CONTROL: seed -> activate -> close WITHOUT the field-persist checkpoint ==" +DNEG="$WORK/neg"; mkdir -p "$DNEG" +ENGRAM_STORE=1 "$BIN" neg_seed "$DNEG" || { echo "FAIL: neg_seed"; fail=1; } +rm -f "$DNEG/snapshot.json" +ENGRAM_STORE=1 "$BIN" neg_reboot "$DNEG" || { echo "FAIL: neg_reboot"; fail=1; } + +echo +echo "== 4) assertions (python over the JSON exports) ==" +python3 - "$DPOS" "$DNEG" <<'PY' +import json, sys, os +WM_FLOOR = 0.05 +HEBB_MIN = 1e-6 + +def load(d, name): + with open(os.path.join(d, name)) as f: return json.load(f) + +def node_by_label(g, label): + for n in g["nodes"]: + if n.get("label") == label: return n + return None + +def edge_between(g, a_id, b_id): + for e in g["edges"]: + if e.get("from_id") == a_id and e.get("to_id") == b_id: + return e + return None + +rc = 0 +def check(cond, msg): + global rc + if cond: print(f" PASS: {msg}") + else: print(f" FAIL: {msg}"); rc = 1 + +dpos, dneg = sys.argv[1], sys.argv[2] +pre = load(dpos, "pre_reboot.json") +rebt = load(dpos, "reboot.json") + +pa, pb = node_by_label(pre, "hebb-a"), node_by_label(pre, "hebb-b") +ra = node_by_label(rebt, "hebb-a") +assert pa and pb and ra, "target nodes missing" +pe = edge_between(pre, pa["id"], pb["id"]) +re = edge_between(rebt, pa["id"], pb["id"]) +assert pe and re, "target edge missing" + +pre_hebb = pe.get("hebb", 0.0) +rebt_hebb = re.get("hebb", 0.0) +pre_ac = pa.get("activation_count", 0) +rebt_ac = ra.get("activation_count", 0) +pre_wm = pa.get("working_memory_weight", 0.0) +rebt_wm = ra.get("working_memory_weight", 0.0) + +print(f" edge hebb-a->hebb-b : pre={pre_hebb!r} reboot={rebt_hebb!r}") +print(f" node hebb-a act_cnt : pre={pre_ac!r} reboot={rebt_ac!r}") +print(f" node hebb-a wm : pre={pre_wm!r} reboot={rebt_wm!r} (halved+floored expected)") + +# --- learning actually happened this run (else the test proves nothing) --- +check(pre_hebb > HEBB_MIN, f"activation raised edge hebb above 0 (pre={pre_hebb})") +check(pre_ac >= 1, f"activation reinforced node activation_count (pre={pre_ac})") +check(pre_wm > 0.0, f"activation promoted node to working memory (pre_wm={pre_wm})") + +# --- the load-bearing survival assertions after a real delete-JSON reboot --- +check(abs(rebt_hebb - pre_hebb) < 1e-12, + f"edge hebb SURVIVED reboot unchanged ({rebt_hebb} == {pre_hebb})") +check(rebt_ac == pre_ac, + f"node activation_count SURVIVED reboot unchanged ({rebt_ac} == {pre_ac})") + +# --- WM weight: must equal the JSON path's boot transform exactly (halve+floor) --- +expected_wm = pre_wm * 0.5 +if expected_wm < WM_FLOOR: expected_wm = 0.0 +check(abs(rebt_wm - expected_wm) < 1e-9, + f"node WM weight SURVIVED with the SAME boot transform as JSON path " + f"(reboot={rebt_wm} == halve+floor(pre)={expected_wm})") +check(expected_wm > 0.0, + f"WM survival is observable (halved weight stays above floor: {expected_wm} > {WM_FLOOR})") + +# --- NEGATIVE CONTROL: without the field-persist step the learning is LOST --- +npre = load(dneg, "neg_pre.json") +nrebt = load(dneg, "neg_reboot.json") +na_pre = node_by_label(npre, "hebb-a") +na_rebt = node_by_label(nrebt, "hebb-a") +ne_pre = edge_between(npre, na_pre["id"], node_by_label(npre, "hebb-b")["id"]) +ne_rebt = edge_between(nrebt, na_rebt["id"], node_by_label(nrebt, "hebb-b")["id"]) +print(f" [neg] edge hebb : pre={ne_pre.get('hebb',0.0)!r} reboot={ne_rebt.get('hebb',0.0)!r}") +print(f" [neg] node act_cnt : pre={na_pre.get('activation_count',0)!r} reboot={na_rebt.get('activation_count',0)!r}") +check(ne_pre.get("hebb", 0.0) > HEBB_MIN, + f"[neg] activation DID raise hebb in RAM (pre={ne_pre.get('hebb',0.0)})") +check(ne_rebt.get("hebb", 0.0) == 0.0, + "[neg] WITHOUT checkpoint field-persist, edge hebb is LOST on reboot (==0) — fix is load-bearing") +check(na_rebt.get("activation_count", 0) == 0, + "[neg] WITHOUT checkpoint field-persist, activation_count is LOST on reboot (==0)") + +sys.exit(rc) +PY +[ $? -ne 0 ] && fail=1 + +echo +echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) ==" +SANBIN="$WORK/m35.san" +gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ + -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" +if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else + export ASAN_OPTIONS=detect_leaks=0 + DSAN="$WORK/san"; mkdir -p "$DSAN" + ENGRAM_STORE=1 "$SANBIN" pos_seed "$DSAN" >/dev/null 2>"$WORK/san_run.log" && \ + { rm -f "$DSAN/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" pos_reboot "$DSAN" >/dev/null 2>>"$WORK/san_run.log"; } + if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then + echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1 + else + echo " ok: ASan+UBSan clean across pos_seed/checkpoint/reboot (field-persist, boot laundering)" + fi +fi + +echo +if [ "$fail" -eq 0 ]; then echo "================ M3.5 HEBB-PERSIST GATE: PASS ================"; else echo "================ M3.5 HEBB-PERSIST GATE: FAIL ================"; fi +rm -rf "$WORK" +exit $fail diff --git a/engram/test/test_m35_hebb_persist.c b/engram/test/test_m35_hebb_persist.c new file mode 100644 index 0000000..4aaf9d3 --- /dev/null +++ b/engram/test/test_m35_hebb_persist.c @@ -0,0 +1,130 @@ +/* test_m35_hebb_persist.c — M3.5 PRE-FLIP GATE. + * + * Proves that in-place field mutations made during spreading activation — edge + * `hebb` (+ last_fired), node `activation_count`, node working-memory weight — + * PERSIST to the paged store and survive a restart from neuron.egm with + * snapshot.json deleted. This is the "hebb-survives-restart" fix that gates the + * live cutover. + * + * Same style as test_m3_parity.c: a REAL el-level harness linking the actual + * el_runtime.c native engram builtins + engram_store.c, driving engram_node_full + * / engram_connect / engram_activate_json / engram_save / engram_store_boot / + * engram_store_checkpoint / engram_store_close directly from C. No EL interpreter. + * + * Modes (argv[1]), data dir (argv[2]): + * pos_seed — ENGRAM_STORE=1: fresh store, seed a graph tuned so activation + * co-activates a connected pair (edge hebb 0 -> ETA) and reinforces + * nodes (activation_count 0 -> >=1, WM weight -> >0). Export the + * post-activation resident graph to pre_reboot.json, then CHECKPOINT + * (the M3.5 field-persist), then close. + * pos_reboot— ENGRAM_STORE=1, snapshot.json deleted by runner: boot from + * neuron.egm (WAL replay), export reboot.json, close. The values in + * reboot.json are what actually survived the round-trip. + * neg_seed — identical to pos_seed but WITHOUT the checkpoint field-persist + * (negative control): activation mutations never reach the store. + * neg_reboot— boot from neuron.egm, export neg_reboot.json, close. + * offcheck — ENGRAM_STORE unset: seed+activate+checkpoint must NOT touch the + * store (no neuron.egm, checkpoint returns 0). + * + * The pass/fail assertions live in run_m35_hebb_persist.sh (python over the JSON + * exports): reboot.json must carry the learned hebb / activation_count and the + * JSON-identical halved WM weight; neg_reboot.json must have LOST them. + */ +#include "el_runtime.h" +#include +#include +#include + +extern int engram_store_enabled(void); +extern el_val_t engram_store_boot(el_val_t data_dir); +extern el_val_t engram_store_checkpoint(void); +extern el_val_t engram_store_close(void); + +static el_val_t S(const char* s){ return EL_STR(s); } +static el_val_t F(double d){ return el_from_float(d); } + +/* Two nodes with DISTINCT content (so the redundancy-suppression pass cannot + * dedup one of them away) that both match the query strongly, wired by one + * "associate" edge. A handful of weakly-related distractors make it a real + * graph. On activation both A and B promote to working memory and co-activate, + * so their edge's hebb rises from 0 to ENGRAM_HEBB_ETA. */ +static void build_seed(void){ + el_val_t a = engram_node_full(S("hebbian potentiation strengthens co-active memory links"), + S("Concept"), S("hebb-a"), F(0.9), F(0.85), F(1.0), S("Semantic"), + S("hebbian,memory,activation")); + el_val_t b = engram_node_full(S("co-active memory links accrue hebbian associative weight"), + S("Concept"), S("hebb-b"), F(0.9), F(0.85), F(1.0), S("Semantic"), + S("hebbian,memory,weight")); + el_val_t c = engram_node_full(S("unrelated culinary recipe for sourdough bread"), + S("Fact"), S("distractor-1"), F(0.4), F(0.4), F(1.0), S("Semantic"), + S("food")); + el_val_t d = engram_node_full(S("the weather forecast predicts rain tomorrow afternoon"), + S("Fact"), S("distractor-2"), F(0.4), F(0.4), F(1.0), S("Semantic"), + S("weather")); + engram_connect(a, b, F(0.8), S("associate")); /* the edge under test */ + engram_connect(a, c, F(0.3), S("associate")); + engram_connect(b, d, F(0.3), S("associate")); +} + +static const char* QUERY = + "hebbian potentiation co-active memory links associative weight"; + +static void export_graph(const char* dir, const char* name){ + char p[1024]; + snprintf(p, sizeof p, "%s/%s", dir, name); + if (!engram_save(S(p))){ fprintf(stderr, "save %s failed\n", name); exit(2); } +} + +int main(int argc, char** argv){ + if (argc < 3){ + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + const char* mode = argv[1]; + const char* dir = argv[2]; + + if (!strcmp(mode, "pos_seed") || !strcmp(mode, "neg_seed")){ + int persist = !strcmp(mode, "pos_seed"); + if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; } + if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; } + build_seed(); + el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3); + (void)act; + /* Capture the post-activation resident state BEFORE persisting/closing. */ + export_graph(dir, persist ? "pre_reboot.json" : "neg_pre.json"); + printf("[%s] nodes=%lld edges=%lld\n", mode, + (long long)(int64_t)engram_node_count(), + (long long)(int64_t)engram_edge_count()); + if (persist){ + if (!engram_store_checkpoint()){ fprintf(stderr, "checkpoint failed\n"); return 2; } + } + /* neg mode: NO field-persist checkpoint. engram_store_close still flushes + * pages, but no store_put_* ran post-creation, so the store keeps the + * pristine creation-time field values (hebb=0, activation_count=0). */ + engram_store_close(); + return 0; + } + if (!strcmp(mode, "pos_reboot") || !strcmp(mode, "neg_reboot")){ + if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; } + /* snapshot.json deleted by the runner — boot MUST come from neuron.egm. */ + if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; } + export_graph(dir, !strcmp(mode, "pos_reboot") ? "reboot.json" : "neg_reboot.json"); + printf("[%s] nodes=%lld edges=%lld\n", mode, + (long long)(int64_t)engram_node_count(), + (long long)(int64_t)engram_edge_count()); + engram_store_close(); + return 0; + } + if (!strcmp(mode, "offcheck")){ + int en = engram_store_enabled(); + el_val_t boot = engram_store_boot(S(dir)); /* no-op with flag off */ + build_seed(); + engram_activate_json(S(QUERY), (el_val_t)3); + el_val_t ck = engram_store_checkpoint(); /* must be a no-op */ + printf("[offcheck] enabled=%d boot=%lld checkpoint=%lld\n", + en, (long long)(int64_t)boot, (long long)(int64_t)ck); + return (en == 0 && (int64_t)boot == 0 && (int64_t)ck == 0) ? 0 : 1; + } + fprintf(stderr, "unknown mode %s\n", mode); + return 2; +} diff --git a/engram/test/test_m3_parity.c b/engram/test/test_m3_parity.c index bd474ba..da3db49 100644 --- a/engram/test/test_m3_parity.c +++ b/engram/test/test_m3_parity.c @@ -115,12 +115,19 @@ int main(int argc, char** argv){ if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; } snprintf(p, sizeof p, "%s/on_graph.json", dir); engram_save(S(p)); /* export resident (== store) */ + /* Checkpoint the freshly-imported (pristine) graph — this is the state + * the reboot comparison expects to round-trip. Under M3.5 a checkpoint + * persists the resident graph's CURRENT field state, so it must run + * BEFORE activation mutates fields in place; activation itself is + * exercised below only for the activation-result-set parity check. The + * M3.5 gate (test_m35_hebb_persist) separately proves that a checkpoint + * taken AFTER activation durably carries the learned hebb/WM state. */ + engram_store_checkpoint(); el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3); snprintf(p, sizeof p, "%s/on_act.json", dir); write_file(p, EL_CSTR(act)); printf("[on] nodes=%lld edges=%lld\n", (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count()); - engram_store_checkpoint(); engram_store_close(); return 0; } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index b9ef5de..ed89f69 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -7339,8 +7339,10 @@ static void eg_store_put_edge(const EngramEdge* e) { } /* Resident-load callbacks: StoreNode/StoreEdge → a fresh EngramNode/EngramEdge - * appended to the in-RAM graph. Mirrors engram_load's field set (minus the - * boot-time WM laundering — the store already holds the authoritative weights). */ + * appended to the in-RAM graph. Mirrors engram_load's field set. The boot-time + * WM laundering (halve + floor + global cap) that engram_load applies is done + * once, after all nodes are loaded, in engram_store_boot — see the block there — + * so the store-on boot behaves byte-identically to the JSON path (M3.5 parity). */ static void eg_load_node_cb(const StoreNode* sn, void* ctx) { EngramStore* g = (EngramStore*)ctx; engram_grow_nodes(); @@ -7427,6 +7429,10 @@ static void eg_reset_resident(EngramStore* g) { engram_adj_free(g); } +/* Defined later with engram_load; declared here for the boot-time WM laundering + * that keeps the store-on boot byte-identical to the JSON path (M3.5 parity). */ +static void eg_enforce_wm_cap_on_load(EngramStore* g); + /* engram_store_boot(data_dir) — open (import-once or WAL-replay) the durable * paged store and load it whole into RAM (Phase 1). No-op / returns 0 when the * flag is off. Returns 1 on success. Idempotent (a second call is a no-op). */ @@ -7446,14 +7452,60 @@ el_val_t engram_store_boot(el_val_t data_dir) { for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]); store_layers_free(ls, ln); } + /* Boot-time WM laundering — MUST match engram_load exactly (M3.5 parity). + * The JSON load path halves every persisted working_memory_weight on boot + * (stale pinned weights decay out over successive restarts; genuine WM state + * keeps continuity) and floors sub-ENGRAM_WM_FLOOR residue to zero, then + * enforces the global WM cap. The store now persists post-activation WM + * weights, so the store-on boot must apply the identical transform or the two + * persistence paths would diverge on the very first restart. */ + for (int64_t i = 0; i < g->node_count; i++) { + g->nodes[i].working_memory_weight *= 0.5; + if (g->nodes[i].working_memory_weight < ENGRAM_WM_FLOOR) + g->nodes[i].working_memory_weight = 0.0; + } + eg_enforce_wm_cap_on_load(g); g->adj_dirty = 1; return (el_val_t)1; } -/* engram_store_checkpoint() — flush dirty pages + advance the checkpoint LSN. +/* engram_store_checkpoint() — M3.5 PRE-FLIP GATE. + * + * Spreading activation mutates fields IN PLACE on the resident graph — edge + * `hebb`/`last_fired` (potentiation + homeostatic scaling), node + * `activation_count`/`last_activated`/`working_memory_weight`/`wm_anchor` + * (reinforcement + WM caps) — and also FORMS brand-new `hebbian-associate` + * edges. M3 only mirrored node/edge *creates* and *forgets*; none of those + * in-place mutations or activation-formed edges reached the paged store, so + * learned associations were lost on every restart. This is the fix that makes + * "learned edges survive a restart" true, and it gates the live cutover. + * + * On the soul's save/checkpoint path we push the resident graph's current field + * state through the store's WAL-logged API, then checkpoint: + * - store_put_node(n) for every node → persists WM weight, activation_count, + * last_activated, wm_anchor, the base-level access ring, etc. + * - store_put_edge(e) for every edge → persists hebb + last_fired AND creates + * any activation-formed edges. (store_hebb_batch is deliberately NOT used: it + * is a delta-only op that skips edge ids not already resident in the store — + * see apply_hebb_batch — so it cannot persist the newly-formed hebbian edges + * that are the whole point of this milestone. store_put_edge is the idempotent + * upsert that subsumes the hebb delta.) + * then engram_checkpoint flushes dirty pages + advances the checkpoint LSN. Every + * push is a WAL record, so a graceful restart restores them. + * + * Approach: a FULL WALK of the resident graph (not a dirty-set). At Phase-1 + * ~64 MB resident this is a cheap linear pass on an already-in-RAM array, run at + * the soul's save cadence (seconds-to-minutes), and it is trivially complete — + * no mutation site can be missed and no separate new-edge tracking is needed. An + * inter-checkpoint crash loses only the most-recent unsaved learning, exactly the + * same durability envelope as today's JSON-snapshot cadence. + * * The store→JSON export path stays engram_save (the JSON is an export artifact). */ el_val_t engram_store_checkpoint(void) { if (!engram_store_enabled() || !g_engram_store) return (el_val_t)0; + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) eg_store_put_node(&g->nodes[i]); + for (int64_t i = 0; i < g->edge_count; i++) eg_store_put_edge(&g->edges[i]); return (el_val_t)(int64_t)(engram_checkpoint(g_engram_store) == 0 ? 1 : 0); }