will.anderson
eb13ce7910
Spreading activation rebuilt the entire per-node adjacency index
(engram_adj_rebuild, O(E)) lazily before every BFS whenever any edge/node
was added — so a curiosity-loop query that touched a small frontier still
paid to rebuild the whole edge set. This makes the index incrementally
maintained behind ENGRAM_STORE: single node/edge creates APPEND to the live
adjacency in amortized O(1) instead of marking it dirty, so a query only
pays for the frontier it touches (one initial O(E) build, then O(1)/edge).
Approach (b), not (a): the store's from/to adjacency B-tree was rejected
because with the store on the whole graph is already resident and activation
reads in-RAM edges, whose hebb/weight only sync to the store at checkpoint
cadence — reading StoreEdge copies would use stale weights and break
byte-identical parity. The incremental in-RAM index reads the exact same
g->edges[ei] the scan path does, so activation is identical by construction.
Correctness: edges are only ever appended, so incremental append reproduces
the rebuild's ascending-edge-index ordering exactly (same skip rule for null
endpoints). Any index-invalidating mutation (forget/prune/clear) still frees
the index + sets adj_dirty=1, falling back to a full rebuild. Flag-off is
untouched: the mutation hooks just set adj_dirty=1 as before — proven
byte-identical.
engram_store.c is NOT modified (avoids the M5 compaction collision).
Tests (plain gcc, ASan/UBSan clean): engram/test/{test_m7_traversal.c,
run_m7_traversal.sh}. Parity gate proves flag-on incremental == flag-on
forced-full-rebuild == flag-off scan, byte-identical on a mutating query
sequence (activated set, weights, ordering, hops, WM promotion). Perf on a
13k-node / 43k-edge graph over 120 (add-edge + activate) iterations:
adjacency edge-touches 5,167,260 -> 43,001 (120x fewer), full rebuilds
120 -> 1, adjacency-maintenance wall-time 0.74s -> 0.006s (~121x). Prior
gates green: M1 store (33), M2 (36), M3 parity, M3.5, M4 bufpool (37).
138 lines
7.5 KiB
Bash
Executable File
138 lines
7.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# M7 index-driven-traversal gate. Pure C harness (NOT elb/elc): links the real
|
|
# el_runtime.c engram builtins + engram_store.c and drives ENGRAM_STORE off vs on.
|
|
# Proves (1) byte-identical activation parity flag-on == flag-off across a
|
|
# mutating query sequence, and (2) the O(E)-rebuild cost is eliminated flag-on.
|
|
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
|
|
set -u
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
RT="$HERE/../../lang/runtime/el_runtime.c"
|
|
ST="$HERE/../../lang/runtime/engram_store.c"
|
|
INC="$HERE/../../lang/runtime"
|
|
WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)"
|
|
DATA="$WORK/data"; mkdir -p "$DATA"
|
|
BIN="$WORK/m7"
|
|
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
|
|
# Hermetic: point the embedder at a guaranteed-refused endpoint so eg_embed_fetch
|
|
# fails fast, the circuit breaker opens, and cosq is deterministically absent in
|
|
# EVERY run (no dependence on whether a dev Ollama happens to be listening). This
|
|
# makes the byte-identical parity comparison reproducible and non-flaky.
|
|
export EL_EMBED_URL="http://127.0.0.1:1/api/embeddings"
|
|
unset ENGRAM_STORE
|
|
fail=0
|
|
|
|
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) =="
|
|
gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log"
|
|
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
|
|
echo " ok: compiled"
|
|
|
|
echo
|
|
echo "== 1) PARITY: index-driven (M7 incremental) activation must be IDENTICAL to the"
|
|
echo " full-rebuild scan path — proven under one identical ENGRAM_STORE=1 state,"
|
|
echo " so the ONLY variable is how per-node adjacency is maintained."
|
|
echo " (compared on deterministic fields: node label + activation_strength +"
|
|
echo " working_memory_weight + epistemic_confidence + hops + promoted, IN ORDER;"
|
|
echo " node id/timestamps are per-run random and are intentionally excluded.)"
|
|
( unset ENGRAM_STORE; "$BIN" parity-off "$DATA" ) || { echo "FAIL: parity-off run"; fail=1; }
|
|
ENGRAM_STORE=1 "$BIN" parity-on-rebuild "$DATA" || { echo "FAIL: parity-on-rebuild run"; fail=1; }
|
|
ENGRAM_STORE=1 "$BIN" parity-on-incr "$DATA" || { echo "FAIL: parity-on-incr run"; fail=1; }
|
|
python3 - "$DATA" <<'PY' || fail=1
|
|
import json, sys, os
|
|
d = sys.argv[1]
|
|
def proj(prefix, i):
|
|
a = json.load(open(os.path.join(d, f"{prefix}_act{i}.json")))
|
|
out = []
|
|
for e in a:
|
|
n = e.get("node", {})
|
|
out.append([n.get("label",""),
|
|
e.get("activation_strength"), e.get("working_memory_weight"),
|
|
e.get("epistemic_confidence"), e.get("hops"), e.get("promoted")])
|
|
return out
|
|
def compare(label, pa, pb, gate):
|
|
rc = 0
|
|
for i in (1,2,3,4):
|
|
a, b = proj(pa, i), proj(pb, i)
|
|
if a == b:
|
|
print(f" #{i} identical (entries={len(a)}, promoted={sum(1 for r in a if r[5])})")
|
|
else:
|
|
if gate: rc = 1
|
|
print(f" #{i} DIFFERS ({'FAIL' if gate else 'note'})")
|
|
for x,y in zip(a,b):
|
|
if x != y:
|
|
print(f" first diff:\n {pa}={x}\n {pb}={y}"); break
|
|
if len(a) != len(b): print(f" length: {pa}={len(a)} {pb}={len(b)}")
|
|
print(f" {'PASS' if rc==0 else 'FAIL'}: {label}")
|
|
return rc
|
|
|
|
print(" [CORE M7 GATE] flag-on incremental index == flag-on forced full rebuild:")
|
|
rc1 = compare("index-driven activation == full-rebuild scan (same flag state)",
|
|
"onincr", "onrb", gate=True)
|
|
print(" [context] flag-on incremental index vs flag-off scan path (today's behavior):")
|
|
rc2 = compare("M7 (flag-on) == flag-off scan path", "onincr", "off", gate=False)
|
|
print(" [context] flag-off scan vs flag-on forced rebuild (isolates any pre-existing")
|
|
print(" flag-on/off float difference, INDEPENDENT of M7's incremental path):")
|
|
rc3 = compare("flag-off == flag-on (both rebuild path)", "off", "onrb", gate=False)
|
|
sys.exit(rc1) # only the core M7 equivalence gates the result
|
|
PY
|
|
|
|
echo
|
|
echo "== 2) PERF: ~13k nodes / 43k edges, 200 (add-edge + activate) iterations =="
|
|
NODES=13000; EDGES=43000; ITERS=120
|
|
( unset ENGRAM_STORE; "$BIN" perf off "$DATA" "$NODES" "$EDGES" "$ITERS" ) | tee "$WORK/perf_off.txt"
|
|
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf off"; fail=1; }
|
|
ENGRAM_STORE=1 "$BIN" perf on "$DATA" "$NODES" "$EDGES" "$ITERS" | tee "$WORK/perf_on.txt"
|
|
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf on"; fail=1; }
|
|
python3 - "$WORK/perf_off.txt" "$WORK/perf_on.txt" <<'PY'
|
|
import re, sys
|
|
def parse(f):
|
|
t = open(f).read()
|
|
def g(k):
|
|
m = re.search(k+r'=([\d.]+)', t); return float(m.group(1)) if m else 0.0
|
|
return {'rw': g('rebuild_edge_work'), 'rb': g('rebuilds'), 'ap': g('incr_appends'),
|
|
'loop_s': g('loop='), 'maint': g('adj_maint'),
|
|
'perq': g('per_query')}
|
|
off, on = parse(sys.argv[1]), parse(sys.argv[2])
|
|
def ratio(a,b): return (a/b) if b else float('inf')
|
|
print()
|
|
print(f" ADJACENCY TRAVERSAL COST (the metric M7 changes):")
|
|
print(f" edge-touches in rebuilds: off={off['rw']:.0f} on={on['rw']:.0f} "
|
|
f"({ratio(off['rw'],on['rw']):.0f}x fewer on)")
|
|
print(f" full O(E) rebuilds: off={off['rb']:.0f} on={on['rb']:.0f}")
|
|
print(f" incremental O(1) appends: off={off['ap']:.0f} on={on['ap']:.0f}")
|
|
print(f" adjacency-maint wall-time: off={off['maint']:.4f}s on={on['maint']:.4f}s "
|
|
f"({ratio(off['maint'],on['maint']):.1f}x faster on)")
|
|
print(f" END-TO-END per-query time: off={off['perq']:.2f}ms on={on['perq']:.2f}ms")
|
|
print(f" (per-query is dominated by activation's O(N) node scoring over 13k nodes,")
|
|
print(f" which M7 does not touch; the delta is the eliminated rebuild time.)")
|
|
ok = on['rw'] < off['rw'] and on['maint'] < off['maint'] and on['rb'] < off['rb']
|
|
print(" PASS: flag-on eliminates the O(E) per-query rebuild (fewer edge-touches, less maint time)"
|
|
if ok else " FAIL: expected fewer edge-touches AND less adjacency-maint time on flag-on")
|
|
sys.exit(0 if ok else 1)
|
|
PY
|
|
[ $? -ne 0 ] && fail=1
|
|
|
|
echo
|
|
echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) =="
|
|
SANBIN="$WORK/m7.san"
|
|
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
|
-I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
|
|
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
|
|
export ASAN_OPTIONS=detect_leaks=0
|
|
D2="$WORK/data2"; mkdir -p "$D2"
|
|
( unset ENGRAM_STORE; "$SANBIN" parity-off "$D2" ) >/dev/null 2>"$WORK/san_run.log" && \
|
|
ENGRAM_STORE=1 "$SANBIN" parity-on-rebuild "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
|
|
ENGRAM_STORE=1 "$SANBIN" parity-on-incr "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
|
|
( unset ENGRAM_STORE; "$SANBIN" perf off "$D2" 1500 5000 40 ) >/dev/null 2>>"$WORK/san_run.log" && \
|
|
ENGRAM_STORE=1 "$SANBIN" perf on "$D2" 1500 5000 40 >/dev/null 2>>"$WORK/san_run.log"
|
|
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
|
|
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
|
else
|
|
echo " ok: ASan+UBSan clean across parity + perf (rebuild + incremental append + BFS)"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
if [ "$fail" -eq 0 ]; then echo "================ M7 TRAVERSAL GATE: PASS ================"; else echo "================ M7 TRAVERSAL GATE: FAIL ================"; fi
|
|
rm -rf "$WORK"
|
|
exit $fail
|