neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests 4 Actions Packages Projects Releases 5 Wiki Activity

engram M7: index-driven activation traversal (incremental adjacency, flag-gated)

Browse Source 
Spreading activation rebuilt the entire per-node adjacency index
(engram_adj_rebuild, O(E)) lazily before every BFS whenever any edge/node
was added — so a curiosity-loop query that touched a small frontier still
paid to rebuild the whole edge set. This makes the index incrementally
maintained behind ENGRAM_STORE: single node/edge creates APPEND to the live
adjacency in amortized O(1) instead of marking it dirty, so a query only
pays for the frontier it touches (one initial O(E) build, then O(1)/edge).

Approach (b), not (a): the store's from/to adjacency B-tree was rejected
because with the store on the whole graph is already resident and activation
reads in-RAM edges, whose hebb/weight only sync to the store at checkpoint
cadence — reading StoreEdge copies would use stale weights and break
byte-identical parity. The incremental in-RAM index reads the exact same
g->edges[ei] the scan path does, so activation is identical by construction.

Correctness: edges are only ever appended, so incremental append reproduces
the rebuild's ascending-edge-index ordering exactly (same skip rule for null
endpoints). Any index-invalidating mutation (forget/prune/clear) still frees
the index + sets adj_dirty=1, falling back to a full rebuild. Flag-off is
untouched: the mutation hooks just set adj_dirty=1 as before — proven
byte-identical.

engram_store.c is NOT modified (avoids the M5 compaction collision).

Tests (plain gcc, ASan/UBSan clean): engram/test/{test_m7_traversal.c,
run_m7_traversal.sh}. Parity gate proves flag-on incremental == flag-on
forced-full-rebuild == flag-off scan, byte-identical on a mutating query
sequence (activated set, weights, ordering, hops, WM promotion). Perf on a
13k-node / 43k-edge graph over 120 (add-edge + activate) iterations:
adjacency edge-touches 5,167,260 -> 43,001 (120x fewer), full rebuilds
120 -> 1, adjacency-maintenance wall-time 0.74s -> 0.006s (~121x). Prior
gates green: M1 store (33), M2 (36), M3 parity, M3.5, M4 bufpool (37).
This commit is contained in:
Will Anderson
2026-08-12 16:23:33 -05:00
parent 02dc12d785
commit eb13ce7910
3 changed files with 513 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/test/run_m7_traversal.sh
Executable
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# M7 index-driven-traversal gate. Pure C harness (NOT elb/elc): links the real
# el_runtime.c engram builtins + engram_store.c and drives ENGRAM_STORE off vs on.
# Proves (1) byte-identical activation parity flag-on == flag-off across a
# mutating query sequence, and (2) the O(E)-rebuild cost is eliminated flag-on.
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)"
DATA="$WORK/data"; mkdir -p "$DATA"
BIN="$WORK/m7"
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
# Hermetic: point the embedder at a guaranteed-refused endpoint so eg_embed_fetch
# fails fast, the circuit breaker opens, and cosq is deterministically absent in
# EVERY run (no dependence on whether a dev Ollama happens to be listening). This
# makes the byte-identical parity comparison reproducible and non-flaky.
export EL_EMBED_URL="http://127.0.0.1:1/api/embeddings"
unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) =="
gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
echo " ok: compiled"
echo
echo "== 1) PARITY: index-driven (M7 incremental) activation must be IDENTICAL to the"
echo " full-rebuild scan path — proven under one identical ENGRAM_STORE=1 state,"
echo " so the ONLY variable is how per-node adjacency is maintained."
echo " (compared on deterministic fields: node label + activation_strength +"
echo " working_memory_weight + epistemic_confidence + hops + promoted, IN ORDER;"
echo " node id/timestamps are per-run random and are intentionally excluded.)"
( unset ENGRAM_STORE; "$BIN" parity-off "$DATA" ) || { echo "FAIL: parity-off run"; fail=1; }
ENGRAM_STORE=1 "$BIN" parity-on-rebuild "$DATA" || { echo "FAIL: parity-on-rebuild run"; fail=1; }
ENGRAM_STORE=1 "$BIN" parity-on-incr "$DATA" || { echo "FAIL: parity-on-incr run"; fail=1; }
python3 - "$DATA" <<'PY' || fail=1
import json, sys, os
d = sys.argv[1]
def proj(prefix, i):
a = json.load(open(os.path.join(d, f"{prefix}_act{i}.json")))
out = []
for e in a:
n = e.get("node", {})
out.append([n.get("label",""),
e.get("activation_strength"), e.get("working_memory_weight"),
e.get("epistemic_confidence"), e.get("hops"), e.get("promoted")])
return out
def compare(label, pa, pb, gate):
rc = 0
for i in (1,2,3,4):
a, b = proj(pa, i), proj(pb, i)
if a == b:
print(f" #{i} identical (entries={len(a)}, promoted={sum(1 for r in a if r[5])})")
else:
if gate: rc = 1
print(f" #{i} DIFFERS ({'FAIL' if gate else 'note'})")
for x,y in zip(a,b):
if x != y:
print(f" first diff:\n {pa}={x}\n {pb}={y}"); break
if len(a) != len(b): print(f" length: {pa}={len(a)} {pb}={len(b)}")
print(f" {'PASS' if rc==0 else 'FAIL'}: {label}")
return rc
print(" [CORE M7 GATE] flag-on incremental index == flag-on forced full rebuild:")
rc1 = compare("index-driven activation == full-rebuild scan (same flag state)",
"onincr", "onrb", gate=True)
print(" [context] flag-on incremental index vs flag-off scan path (today's behavior):")
rc2 = compare("M7 (flag-on) == flag-off scan path", "onincr", "off", gate=False)
print(" [context] flag-off scan vs flag-on forced rebuild (isolates any pre-existing")
print(" flag-on/off float difference, INDEPENDENT of M7's incremental path):")
rc3 = compare("flag-off == flag-on (both rebuild path)", "off", "onrb", gate=False)
sys.exit(rc1) # only the core M7 equivalence gates the result
PY
echo
echo "== 2) PERF: ~13k nodes / 43k edges, 200 (add-edge + activate) iterations =="
NODES=13000; EDGES=43000; ITERS=120
( unset ENGRAM_STORE; "$BIN" perf off "$DATA" "$NODES" "$EDGES" "$ITERS" ) | tee "$WORK/perf_off.txt"
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf off"; fail=1; }
ENGRAM_STORE=1 "$BIN" perf on "$DATA" "$NODES" "$EDGES" "$ITERS" | tee "$WORK/perf_on.txt"
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf on"; fail=1; }
python3 - "$WORK/perf_off.txt" "$WORK/perf_on.txt" <<'PY'
import re, sys
def parse(f):
t = open(f).read()
def g(k):
m = re.search(k+r'=([\d.]+)', t); return float(m.group(1)) if m else 0.0
return {'rw': g('rebuild_edge_work'), 'rb': g('rebuilds'), 'ap': g('incr_appends'),
'loop_s': g('loop='), 'maint': g('adj_maint'),
'perq': g('per_query')}
off, on = parse(sys.argv[1]), parse(sys.argv[2])
def ratio(a,b): return (a/b) if b else float('inf')
print()
print(f" ADJACENCY TRAVERSAL COST (the metric M7 changes):")
print(f" edge-touches in rebuilds: off={off['rw']:.0f} on={on['rw']:.0f} "
f"({ratio(off['rw'],on['rw']):.0f}x fewer on)")
print(f" full O(E) rebuilds: off={off['rb']:.0f} on={on['rb']:.0f}")
print(f" incremental O(1) appends: off={off['ap']:.0f} on={on['ap']:.0f}")
print(f" adjacency-maint wall-time: off={off['maint']:.4f}s on={on['maint']:.4f}s "
f"({ratio(off['maint'],on['maint']):.1f}x faster on)")
print(f" END-TO-END per-query time: off={off['perq']:.2f}ms on={on['perq']:.2f}ms")
print(f" (per-query is dominated by activation's O(N) node scoring over 13k nodes,")
print(f" which M7 does not touch; the delta is the eliminated rebuild time.)")
ok = on['rw'] < off['rw'] and on['maint'] < off['maint'] and on['rb'] < off['rb']
print(" PASS: flag-on eliminates the O(E) per-query rebuild (fewer edge-touches, less maint time)"
if ok else " FAIL: expected fewer edge-touches AND less adjacency-maint time on flag-on")
sys.exit(0 if ok else 1)
PY
[ $? -ne 0 ] && fail=1
echo
echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m7.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
D2="$WORK/data2"; mkdir -p "$D2"
( unset ENGRAM_STORE; "$SANBIN" parity-off "$D2" ) >/dev/null 2>"$WORK/san_run.log" && \
ENGRAM_STORE=1 "$SANBIN" parity-on-rebuild "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
ENGRAM_STORE=1 "$SANBIN" parity-on-incr "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
( unset ENGRAM_STORE; "$SANBIN" perf off "$D2" 1500 5000 40 ) >/dev/null 2>>"$WORK/san_run.log" && \
ENGRAM_STORE=1 "$SANBIN" perf on "$D2" 1500 5000 40 >/dev/null 2>>"$WORK/san_run.log"
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
else
echo " ok: ASan+UBSan clean across parity + perf (rebuild + incremental append + BFS)"
fi
fi
echo
if [ "$fail" -eq 0 ]; then echo "================ M7 TRAVERSAL GATE: PASS ================"; else echo "================ M7 TRAVERSAL GATE: FAIL ================"; fi
rm -rf "$WORK"
exit $fail
engram/test/test_m7_traversal.c
+208
View File
@@ -0,0 +1,208 @@
/* test_m7_traversal.c — M7 index-driven activation traversal.
*
* Milestone 7 replaces the O(E) full adjacency rebuild that spreading activation
* paid before every BFS with an incrementally-maintained per-node index, behind
* the ENGRAM_STORE flag (flag-off = unchanged behavior). This harness links the
* REAL el_runtime.c engram builtins (+ engram_store.c) and drives activation
* directly — no EL interpreter, no store boot (the index optimization is a pure
* in-RAM concern; the flag is read from the environment).
*
* Modes (argv[1]):
* parity-off <dir> — ENGRAM_STORE unset: build a fixed graph, run a scripted
* sequence of activations WITH mid-sequence edge/node
* inserts, dump each activation's JSON to <dir>/off_actN.json.
* parity-on <dir> — ENGRAM_STORE=1: identical graph + identical sequence,
* dump to <dir>/on_actN.json. The runner asserts the off/on
* files are BYTE-IDENTICAL (same activated set, weights,
* ordering, hops, WM promotion).
* perf <off|on> <dir> <nodes> <edges> <iters>
* — build a large graph, then loop `iters` times doing
* (add 1 edge + activate). Prints wall-time and the M7
* instrumentation counters (rebuild calls / rebuild
* edge-work / incremental appends).
*
* Writes ONLY under the caller-provided throwaway dir.
*/
#include "el_runtime.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
/* M7 instrumentation getters (test-only; defined in el_runtime.c). */
extern int64_t engram_adj_rebuild_calls(void);
extern int64_t engram_adj_rebuild_edge_work(void);
extern int64_t engram_adj_incr_appends(void);
extern double engram_adj_maint_seconds(void);
extern void engram_adj_test_force_dirty(void);
extern int engram_store_enabled(void);
static el_val_t S(const char* s){ return EL_STR(s); }
static el_val_t F(double d){ return el_from_float(d); }
/* Deterministic LCG so off/on processes build byte-identical graphs. */
static uint64_t g_rng = 0x9E3779B97F4A7C15ULL;
static void rng_seed(uint64_t s){ g_rng = s ? s : 1; }
static uint64_t rng_next(void){ g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL; return g_rng >> 17; }
static el_val_t* g_handles = NULL; /* node id handles from engram_node_full */
static int64_t g_nnodes = 0;
static void write_file(const char* path, const char* content){
FILE* f = fopen(path, "wb");
if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
if (content) fwrite(content, 1, strlen(content), f);
fclose(f);
}
/* Build `n` nodes whose content carries query-matchable tokens, then `m`
* deterministic edges among them. Handles are retained for later connect. */
static void build_graph(int64_t n, int64_t m){
g_handles = malloc((size_t)n * sizeof(el_val_t));
g_nnodes = n;
static const char* topics[] = {
"storage engine durable log", "spreading activation graph traversal",
"hebbian potentiation memory", "buffer pool paging checkpoint",
"adjacency index edge lookup", "working memory promotion",
"b-tree primary index", "embeddings nearest neighbour" };
for (int64_t i = 0; i < n; i++){
char content[256];
snprintf(content, sizeof content,
"node %lld about %s and storage engine activation index",
(long long)i, topics[(size_t)(i % 8)]);
char label[32]; snprintf(label, sizeof label, "n%lld", (long long)i);
g_handles[i] = engram_node_full(S(content), S("Concept"), S(label),
F(0.7), F(0.6), F(1.0), S("Semantic"), S("storage,graph,index"));
}
for (int64_t k = 0; k < m; k++){
int64_t a = (int64_t)(rng_next() % (uint64_t)n);
int64_t b = (int64_t)(rng_next() % (uint64_t)n);
if (a == b) b = (b + 1) % n;
engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
}
}
static const char* Q1 = "storage engine activation and the durable log";
static const char* Q2 = "adjacency index graph traversal";
/* One scripted activation with an optional forced full-rebuild first. */
static el_val_t act(const char* q, int depth, int force_rebuild){
if (force_rebuild) engram_adj_test_force_dirty();
return engram_activate_json(S(q), (el_val_t)depth);
}
/* Run the scripted parity sequence and dump each activation JSON. `tag` names
* the output set. When force_rebuild is set, every activation first forces the
* O(E) full-rebuild path (the pre-M7 "scan" behavior); otherwise the M7
* incremental index is used. The graph build + query sequence are byte-for-byte
* deterministic, so any difference between two runs is attributable solely to
* the difference in adjacency maintenance (and/or the ENGRAM_STORE flag). */
static int run_parity(const char* dir, const char* tag, int force_rebuild){
char p[1024];
rng_seed(0xC0FFEE123ULL);
build_graph(60, 140);
el_val_t a1 = act(Q1, 3, force_rebuild);
snprintf(p, sizeof p, "%s/%s_act1.json", dir, tag); write_file(p, EL_CSTR(a1));
/* Mutate the graph BETWEEN activations: this is exactly where the M7 path
* appends incrementally while the rebuild path marks dirty + fully rebuilds.
* Parity must hold across this divergence in HOW the index is maintained. */
engram_connect(g_handles[0], g_handles[7], F(0.8), S("depends-on"));
engram_connect(g_handles[7], g_handles[23], F(0.7), S("enables"));
engram_connect(g_handles[23], g_handles[41],F(0.5), S("uses"));
el_val_t hnew = engram_node_full(S("freshly minted storage index node about activation"),
S("Concept"), S("nnew"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("storage,index"));
engram_connect(g_handles[0], hnew, F(0.9), S("about"));
el_val_t a2 = act(Q1, 3, force_rebuild);
snprintf(p, sizeof p, "%s/%s_act2.json", dir, tag); write_file(p, EL_CSTR(a2));
el_val_t a3 = act(Q2, 2, force_rebuild);
snprintf(p, sizeof p, "%s/%s_act3.json", dir, tag); write_file(p, EL_CSTR(a3));
el_val_t a4 = act(Q1, 3, force_rebuild);
snprintf(p, sizeof p, "%s/%s_act4.json", dir, tag); write_file(p, EL_CSTR(a4));
printf("[parity-%s] enabled=%d force_rebuild=%d nodes=%lld edges=%lld "
"rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
tag, engram_store_enabled(), force_rebuild,
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
(long long)engram_adj_rebuild_calls(), (long long)engram_adj_rebuild_edge_work(),
(long long)engram_adj_incr_appends());
return 0;
}
static double now_sec(void){
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
}
static int run_perf(const char* dir, const char* tag, int64_t n, int64_t m, int64_t iters){
(void)dir;
rng_seed(0xBEEF7777ULL);
double t_build0 = now_sec();
build_graph(n, m);
double t_build = now_sec() - t_build0;
int64_t rb0 = engram_adj_rebuild_calls();
int64_t rw0 = engram_adj_rebuild_edge_work();
int64_t ap0 = engram_adj_incr_appends();
double mt0 = engram_adj_maint_seconds();
double t0 = now_sec();
for (int64_t it = 0; it < iters; it++){
/* One structural mutation per query — the curiosity-loop cadence that
* makes the OLD path rebuild the whole adjacency before every BFS. */
int64_t a = (int64_t)(rng_next() % (uint64_t)n);
int64_t b = (int64_t)(rng_next() % (uint64_t)n);
if (a == b) b = (b + 1) % n;
engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
el_val_t r = engram_activate_json(S(Q1), (el_val_t)2);
(void)r;
}
double elapsed = now_sec() - t0;
double maint = engram_adj_maint_seconds() - mt0;
printf("[perf-%s] flag=%d nodes=%lld edges=%lld iters=%lld build=%.3fs "
"loop=%.3fs per_query=%.3fms adj_maint=%.4fs adj_maint_per_query=%.4fms | "
"rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
tag, engram_store_enabled(),
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
(long long)iters, t_build, elapsed, (elapsed / (double)iters) * 1e3,
maint, (maint / (double)iters) * 1e3,
(long long)(engram_adj_rebuild_calls() - rb0),
(long long)(engram_adj_rebuild_edge_work() - rw0),
(long long)(engram_adj_incr_appends() - ap0));
return 0;
}
int main(int argc, char** argv){
if (argc < 3){ fprintf(stderr, "usage: %s <parity-off|parity-on|perf> ...\n", argv[0]); return 2; }
const char* mode = argv[1];
if (!strcmp(mode, "parity-off")){
/* flag-off, rebuild path = today's scan behavior (the baseline). */
if (engram_store_enabled()){ fprintf(stderr, "parity-off requires ENGRAM_STORE unset\n"); return 2; }
return run_parity(argv[2], "off", 0);
}
if (!strcmp(mode, "parity-on-rebuild")){
/* flag-on, but force the O(E) rebuild before each activation. */
if (!engram_store_enabled()){ fprintf(stderr, "parity-on-rebuild requires ENGRAM_STORE=1\n"); return 2; }
return run_parity(argv[2], "onrb", 1);
}
if (!strcmp(mode, "parity-on-incr")){
/* flag-on, M7 incremental index (the code path under test). */
if (!engram_store_enabled()){ fprintf(stderr, "parity-on-incr requires ENGRAM_STORE=1\n"); return 2; }
return run_parity(argv[2], "onincr", 0);
}
if (!strcmp(mode, "perf")){
/* perf <off|on> <dir> <nodes> <edges> <iters> */
if (argc < 7){ fprintf(stderr, "usage: %s perf <off|on> <dir> <nodes> <edges> <iters>\n", argv[0]); return 2; }
const char* tag = argv[2];
int64_t n = strtoll(argv[4], NULL, 10);
int64_t m = strtoll(argv[5], NULL, 10);
int64_t iters = strtoll(argv[6], NULL, 10);
return run_perf(argv[3], tag, n, m, iters);
}
fprintf(stderr, "unknown mode %s\n", mode);
return 2;
}
lang/runtime/el_runtime.c
+168 -13
View File
@@ -6872,8 +6872,19 @@ typedef struct EngramStore {
int* adj_from_len;
int** adj_to;
int* adj_to_len;
/* M7 (index-driven traversal, ENGRAM_STORE only): per-list allocated
* capacity so single-edge/node mutations can APPEND to the adjacency in
* amortized O(1) instead of forcing an O(E) full rebuild before the next
* BFS. Flag-off never touches these (rebuild sets cap==len and no append
* path runs), so flag-off behavior is byte-identical to before. */
int* adj_from_cap;
int* adj_to_cap;
int adj_dirty; /* 1 = rebuild needed before next BFS */
int64_t adj_node_count; /* node_count at time of last adj_rebuild */
/* Number of node slots currently ALLOCATED in the adjacency arrays (== the
* length of adj_from/adj_to/…). Invariant while the index is live
* (adj_dirty==0 && adj_from!=NULL): adj_node_count >= node_count, and every
* slot in [0,adj_node_count) is a valid (possibly NULL) list. */
int64_t adj_node_count;
} EngramStore;
static EngramStore* engram_global = NULL;
@@ -7150,11 +7161,38 @@ static void engram_idmap_rebuild(EngramStore* g) {
}
}
/* ── M7 traversal instrumentation ────────────────────────────────────────────
* Cumulative, process-lifetime counters that quantify the index-driven-traversal
* win. Purely observational: they never influence activation. `edge_work` sums
* the O(E) cost paid by full adjacency rebuilds; `incr_appends` counts the O(1)
* amortized single-edge appends that replace those rebuilds when ENGRAM_STORE is
* on. Exposed to test harnesses via the getters below. */
int64_t _eg_adj_rebuild_calls = 0;
int64_t _eg_adj_rebuild_edge_work = 0;
int64_t _eg_adj_incr_appends = 0;
double _eg_adj_maint_ns = 0.0; /* wall-time in adjacency maintenance */
int64_t engram_adj_rebuild_calls(void) { return _eg_adj_rebuild_calls; }
int64_t engram_adj_rebuild_edge_work(void) { return _eg_adj_rebuild_edge_work; }
int64_t engram_adj_incr_appends(void) { return _eg_adj_incr_appends; }
double engram_adj_maint_seconds(void) { return _eg_adj_maint_ns * 1e-9; }
/* Test-only: force the next activation to fall back to a full O(E) rebuild,
* so a harness can prove the incremental-index BFS is identical to the
* rebuild-index BFS under one identical flag state. No production caller. */
void engram_adj_test_force_dirty(void) { engram_get()->adj_dirty = 1; }
static double _eg_adj_now_ns(void) {
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1e9 + (double)ts.tv_nsec;
}
/* ── Adjacency index helpers ─────────────────────────────────────────────────
* Per-node adjacency lists: adj_from[i] holds edge indices where
* g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side.
* BFS uses these instead of scanning all edges on every hop.
* Called once per activation call when adj_dirty != 0.
* Full rebuild runs once per activation call when adj_dirty != 0. When
* ENGRAM_STORE is on, single node/edge mutations instead APPEND to the live
* index (engram_adj_on_node_added / engram_adj_on_edge_added) so the common
* curiosity-loop cadence (add a few edges, then query) never pays the O(E)
* rebuild — the index-driven-traversal milestone (M7).
*/
static void engram_adj_free(EngramStore* g) {
int64_t old_nc = g->adj_node_count;
@@ -7162,17 +7200,20 @@ static void engram_adj_free(EngramStore* g) {
for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]);
free(g->adj_from); g->adj_from = NULL;
free(g->adj_from_len); g->adj_from_len = NULL;
free(g->adj_from_cap); g->adj_from_cap = NULL;
}
if (g->adj_to) {
for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]);
free(g->adj_to); g->adj_to = NULL;
free(g->adj_to_len); g->adj_to_len = NULL;
free(g->adj_to_cap); g->adj_to_cap = NULL;
}
g->adj_node_count = 0;
g->adj_dirty = 1;
}
static void engram_adj_rebuild(EngramStore* g) {
double _t0 = _eg_adj_now_ns();
/* Free old adjacency arrays */
if (g->adj_from) {
/* Use adj_node_count (count at build time) not current node_count —
@@ -7182,11 +7223,11 @@ static void engram_adj_rebuild(EngramStore* g) {
for (int64_t i = 0; i < old_nc; i++) {
free(g->adj_from[i]); free(g->adj_to[i]);
}
free(g->adj_from); free(g->adj_from_len);
free(g->adj_to); free(g->adj_to_len);
free(g->adj_from); free(g->adj_from_len); free(g->adj_from_cap);
free(g->adj_to); free(g->adj_to_len); free(g->adj_to_cap);
}
g->adj_from = NULL; g->adj_from_len = NULL;
g->adj_to = NULL; g->adj_to_len = NULL;
g->adj_from = NULL; g->adj_from_len = NULL; g->adj_from_cap = NULL;
g->adj_to = NULL; g->adj_to_len = NULL; g->adj_to_cap = NULL;
g->adj_node_count = 0;
if (g->node_count == 0) { g->adj_dirty = 0; return; }
@@ -7205,14 +7246,19 @@ static void engram_adj_rebuild(EngramStore* g) {
/* Allocate per-node arrays */
g->adj_from = calloc((size_t)g->node_count, sizeof(int*));
g->adj_from_len = calloc((size_t)g->node_count, sizeof(int));
g->adj_from_cap = calloc((size_t)g->node_count, sizeof(int));
g->adj_to = calloc((size_t)g->node_count, sizeof(int*));
g->adj_to_len = calloc((size_t)g->node_count, sizeof(int));
if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) {
g->adj_to_cap = calloc((size_t)g->node_count, sizeof(int));
if (!g->adj_from || !g->adj_from_len || !g->adj_from_cap ||
!g->adj_to || !g->adj_to_len || !g->adj_to_cap) {
free(from_cnt); free(to_cnt);
free(g->adj_from); g->adj_from = NULL;
free(g->adj_from_len); g->adj_from_len = NULL;
free(g->adj_from_cap); g->adj_from_cap = NULL;
free(g->adj_to); g->adj_to = NULL;
free(g->adj_to_len); g->adj_to_len = NULL;
free(g->adj_to_cap); g->adj_to_cap = NULL;
return;
}
for (int64_t i = 0; i < g->node_count; i++) {
@@ -7237,14 +7283,21 @@ static void engram_adj_rebuild(EngramStore* g) {
if (ti >= 0 && g->adj_to[ti])
g->adj_to[ti][to_pos[ti]++] = (int)ei;
}
/* Copy counts */
/* Copy counts. cap == len after a fresh rebuild: the arrays are exactly
* sized, so the first incremental append to any list will grow it. */
for (int64_t i = 0; i < g->node_count; i++) {
g->adj_from_len[i] = from_cnt[i];
g->adj_to_len[i] = to_cnt[i];
g->adj_from_cap[i] = from_cnt[i];
g->adj_to_cap[i] = to_cnt[i];
}
free(from_cnt); free(to_cnt); free(from_pos); free(to_pos);
g->adj_node_count = g->node_count;
g->adj_dirty = 0;
/* M7 instrumentation (test-only counters; no behavioral effect). */
_eg_adj_rebuild_calls++;
_eg_adj_rebuild_edge_work += g->edge_count;
_eg_adj_maint_ns += _eg_adj_now_ns() - _t0;
}
static int64_t engram_find_node_index(const char* id) {
@@ -7388,6 +7441,108 @@ int engram_store_enabled(void) {
strcmp(f, "true") == 0)) ? 1 : 0;
}
/* ── M7: incremental adjacency maintenance (index-driven traversal) ───────────
*
* When ENGRAM_STORE is on, a single node/edge create keeps the already-built
* per-node adjacency index live by APPENDING to it, instead of marking it dirty
* and forcing the next activation to rebuild all O(E) adjacency lists from
* scratch. The result the BFS consumes is byte-identical to a full rebuild:
* - Edges are only ever appended to g->edges[], so their indices increase
* monotonically; appending in creation order reproduces the exact ascending
* edge-index ordering a rebuild's ei-ascending scan produces.
* - The same skip rule as rebuild applies: an edge with a NULL endpoint id
* contributes to NEITHER list.
* - Deletes/shifts (forget, prune, clear) still free the index and set
* adj_dirty=1, so any index-invalidating mutation falls back to a full
* rebuild. The append path only runs while the index is live and clean.
* Flag-off never reaches these helpers: the mutation sites call
* engram_adj_on_{node,edge}_added, which for flag-off simply set adj_dirty=1 —
* exactly the previous behavior, byte-for-byte. */
/* Grow the adjacency arrays so index `need`-1 is addressable. Preserves all
* existing lists; new slots are zeroed (NULL list, len 0, cap 0). Sets
* adj_node_count to the new allocated length so engram_adj_free frees exactly
* the slots that exist. Returns 0 on OOM (caller falls back to a full rebuild
* by setting adj_dirty). */
static int engram_adj_grow_slots(EngramStore* g, int64_t need) {
if (need <= g->adj_node_count) return 1;
int64_t nc = g->adj_node_count ? g->adj_node_count : 8;
while (nc < need) nc *= 2;
int** nf = realloc(g->adj_from, (size_t)nc * sizeof(int*));
int* nfl = realloc(g->adj_from_len, (size_t)nc * sizeof(int));
int* nfc = realloc(g->adj_from_cap, (size_t)nc * sizeof(int));
int** nt = realloc(g->adj_to, (size_t)nc * sizeof(int*));
int* ntl = realloc(g->adj_to_len, (size_t)nc * sizeof(int));
int* ntc = realloc(g->adj_to_cap, (size_t)nc * sizeof(int));
if (nf) g->adj_from = nf;
if (nfl) g->adj_from_len = nfl;
if (nfc) g->adj_from_cap = nfc;
if (nt) g->adj_to = nt;
if (ntl) g->adj_to_len = ntl;
if (ntc) g->adj_to_cap = ntc;
if (!nf || !nfl || !nfc || !nt || !ntl || !ntc) return 0;
for (int64_t i = g->adj_node_count; i < nc; i++) {
g->adj_from[i] = NULL; g->adj_from_len[i] = 0; g->adj_from_cap[i] = 0;
g->adj_to[i] = NULL; g->adj_to_len[i] = 0; g->adj_to_cap[i] = 0;
}
g->adj_node_count = nc;
return 1;
}
/* Append edge index `ei` to the list at (*arr,*len,*cap), doubling capacity as
* needed. Returns 0 on OOM. */
static int engram_adj_list_push(int** arr, int* len, int* cap, int ei) {
if (*len >= *cap) {
int ncap = *cap ? *cap * 2 : 2;
int* na = realloc(*arr, (size_t)ncap * sizeof(int));
if (!na) return 0;
*arr = na; *cap = ncap;
}
(*arr)[(*len)++] = ei;
return 1;
}
/* Append the freshly-created edge g->edges[ei] to the live adjacency index.
* Mirrors engram_adj_rebuild's per-edge classification exactly. Returns 0 on
* OOM (caller forces a rebuild). */
static int engram_adj_add_edge(EngramStore* g, int64_t ei) {
if (ei < 0 || ei >= g->edge_count) return 1;
EngramEdge* e = &g->edges[ei];
if (!e->from_id || !e->to_id) return 1; /* same skip rule as rebuild */
double _t0 = _eg_adj_now_ns();
int64_t fi = engram_idmap_get(g, e->from_id);
int64_t ti = engram_idmap_get(g, e->to_id);
int64_t hi = (fi > ti) ? fi : ti;
if (hi >= 0 && !engram_adj_grow_slots(g, hi + 1)) return 0;
if (fi >= 0 && !engram_adj_list_push(&g->adj_from[fi], &g->adj_from_len[fi],
&g->adj_from_cap[fi], (int)ei)) return 0;
if (ti >= 0 && !engram_adj_list_push(&g->adj_to[ti], &g->adj_to_len[ti],
&g->adj_to_cap[ti], (int)ei)) return 0;
_eg_adj_incr_appends++;
_eg_adj_maint_ns += _eg_adj_now_ns() - _t0;
return 1;
}
/* Mutation-site hook for a newly-appended node at the current top index. When
* the index is live and clean under ENGRAM_STORE, reserve its (empty) adjacency
* slot so a later BFS that seeds this node can index adj_*_len[idx] safely
* without a rebuild. Otherwise defer to the lazy full rebuild (flag-off path,
* or index not yet built / already dirty). */
static void engram_adj_on_node_added(EngramStore* g) {
if (engram_store_enabled() && g->adj_from && !g->adj_dirty) {
if (engram_adj_grow_slots(g, g->node_count)) return;
}
g->adj_dirty = 1; /* flag-off, or OOM/not-built: fall back to rebuild */
}
/* Mutation-site hook for the newly-appended edge at index g->edge_count-1. */
static void engram_adj_on_edge_added(EngramStore* g, int64_t ei) {
if (engram_store_enabled() && g->adj_from && !g->adj_dirty) {
if (engram_adj_add_edge(g, ei)) return;
}
g->adj_dirty = 1; /* flag-off, or OOM/not-built: fall back to rebuild */
}
/* EngramNode → borrowed StoreNode view (no ownership transfer; the store copies
* every field it persists, so shared string pointers are safe). */
static void eg_node_to_store(const EngramNode* n, StoreNode* sn) {
@@ -7637,7 +7792,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
int64_t new_idx = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7767,7 +7922,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
int64_t new_idx_full = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx_full);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7838,7 +7993,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
int64_t new_idx_layered = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx_layered);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -8343,7 +8498,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
e->last_fired = 0;
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
engram_adj_on_edge_added(g, g->edge_count - 1);
if (engram_store_enabled()) eg_store_put_edge(e);
}
@@ -9923,7 +10078,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
ne->last_fired = now_ms;
ne->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
engram_adj_on_edge_added(g, g->edge_count - 1);
_eg_hebb_links_formed++;
hebb_edge_total++;
formed++;