neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests 4 Actions Packages Projects Releases 5 Wiki Activity

integrate M7 index-driven traversal into tiered trunk

Browse Source
This commit is contained in:
Will Anderson
2026-08-12 16:43:19 -05:00
parent ce0d33ba93 eb13ce7910
commit 89589864cd
3 changed files with 513 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/el_runtime.c
+168 -13
View File
@@ -6872,8 +6872,19 @@ typedef struct EngramStore {
int* adj_from_len;
int** adj_to;
int* adj_to_len;
/* M7 (index-driven traversal, ENGRAM_STORE only): per-list allocated
* capacity so single-edge/node mutations can APPEND to the adjacency in
* amortized O(1) instead of forcing an O(E) full rebuild before the next
* BFS. Flag-off never touches these (rebuild sets cap==len and no append
* path runs), so flag-off behavior is byte-identical to before. */
int* adj_from_cap;
int* adj_to_cap;
int adj_dirty; /* 1 = rebuild needed before next BFS */
int64_t adj_node_count; /* node_count at time of last adj_rebuild */
/* Number of node slots currently ALLOCATED in the adjacency arrays (== the
* length of adj_from/adj_to/…). Invariant while the index is live
* (adj_dirty==0 && adj_from!=NULL): adj_node_count >= node_count, and every
* slot in [0,adj_node_count) is a valid (possibly NULL) list. */
int64_t adj_node_count;
} EngramStore;
static EngramStore* engram_global = NULL;
@@ -7150,11 +7161,38 @@ static void engram_idmap_rebuild(EngramStore* g) {
}
}
/* ── M7 traversal instrumentation ────────────────────────────────────────────
* Cumulative, process-lifetime counters that quantify the index-driven-traversal
* win. Purely observational: they never influence activation. `edge_work` sums
* the O(E) cost paid by full adjacency rebuilds; `incr_appends` counts the O(1)
* amortized single-edge appends that replace those rebuilds when ENGRAM_STORE is
* on. Exposed to test harnesses via the getters below. */
int64_t _eg_adj_rebuild_calls = 0;
int64_t _eg_adj_rebuild_edge_work = 0;
int64_t _eg_adj_incr_appends = 0;
double _eg_adj_maint_ns = 0.0; /* wall-time in adjacency maintenance */
int64_t engram_adj_rebuild_calls(void) { return _eg_adj_rebuild_calls; }
int64_t engram_adj_rebuild_edge_work(void) { return _eg_adj_rebuild_edge_work; }
int64_t engram_adj_incr_appends(void) { return _eg_adj_incr_appends; }
double engram_adj_maint_seconds(void) { return _eg_adj_maint_ns * 1e-9; }
/* Test-only: force the next activation to fall back to a full O(E) rebuild,
* so a harness can prove the incremental-index BFS is identical to the
* rebuild-index BFS under one identical flag state. No production caller. */
void engram_adj_test_force_dirty(void) { engram_get()->adj_dirty = 1; }
static double _eg_adj_now_ns(void) {
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1e9 + (double)ts.tv_nsec;
}
/* ── Adjacency index helpers ─────────────────────────────────────────────────
* Per-node adjacency lists: adj_from[i] holds edge indices where
* g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side.
* BFS uses these instead of scanning all edges on every hop.
* Called once per activation call when adj_dirty != 0.
* Full rebuild runs once per activation call when adj_dirty != 0. When
* ENGRAM_STORE is on, single node/edge mutations instead APPEND to the live
* index (engram_adj_on_node_added / engram_adj_on_edge_added) so the common
* curiosity-loop cadence (add a few edges, then query) never pays the O(E)
* rebuild — the index-driven-traversal milestone (M7).
*/
static void engram_adj_free(EngramStore* g) {
int64_t old_nc = g->adj_node_count;
@@ -7162,17 +7200,20 @@ static void engram_adj_free(EngramStore* g) {
for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]);
free(g->adj_from); g->adj_from = NULL;
free(g->adj_from_len); g->adj_from_len = NULL;
free(g->adj_from_cap); g->adj_from_cap = NULL;
}
if (g->adj_to) {
for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]);
free(g->adj_to); g->adj_to = NULL;
free(g->adj_to_len); g->adj_to_len = NULL;
free(g->adj_to_cap); g->adj_to_cap = NULL;
}
g->adj_node_count = 0;
g->adj_dirty = 1;
}
static void engram_adj_rebuild(EngramStore* g) {
double _t0 = _eg_adj_now_ns();
/* Free old adjacency arrays */
if (g->adj_from) {
/* Use adj_node_count (count at build time) not current node_count —
@@ -7182,11 +7223,11 @@ static void engram_adj_rebuild(EngramStore* g) {
for (int64_t i = 0; i < old_nc; i++) {
free(g->adj_from[i]); free(g->adj_to[i]);
}
free(g->adj_from); free(g->adj_from_len);
free(g->adj_to); free(g->adj_to_len);
free(g->adj_from); free(g->adj_from_len); free(g->adj_from_cap);
free(g->adj_to); free(g->adj_to_len); free(g->adj_to_cap);
}
g->adj_from = NULL; g->adj_from_len = NULL;
g->adj_to = NULL; g->adj_to_len = NULL;
g->adj_from = NULL; g->adj_from_len = NULL; g->adj_from_cap = NULL;
g->adj_to = NULL; g->adj_to_len = NULL; g->adj_to_cap = NULL;
g->adj_node_count = 0;
if (g->node_count == 0) { g->adj_dirty = 0; return; }
@@ -7205,14 +7246,19 @@ static void engram_adj_rebuild(EngramStore* g) {
/* Allocate per-node arrays */
g->adj_from = calloc((size_t)g->node_count, sizeof(int*));
g->adj_from_len = calloc((size_t)g->node_count, sizeof(int));
g->adj_from_cap = calloc((size_t)g->node_count, sizeof(int));
g->adj_to = calloc((size_t)g->node_count, sizeof(int*));
g->adj_to_len = calloc((size_t)g->node_count, sizeof(int));
if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) {
g->adj_to_cap = calloc((size_t)g->node_count, sizeof(int));
if (!g->adj_from || !g->adj_from_len || !g->adj_from_cap ||
!g->adj_to || !g->adj_to_len || !g->adj_to_cap) {
free(from_cnt); free(to_cnt);
free(g->adj_from); g->adj_from = NULL;
free(g->adj_from_len); g->adj_from_len = NULL;
free(g->adj_from_cap); g->adj_from_cap = NULL;
free(g->adj_to); g->adj_to = NULL;
free(g->adj_to_len); g->adj_to_len = NULL;
free(g->adj_to_cap); g->adj_to_cap = NULL;
return;
}
for (int64_t i = 0; i < g->node_count; i++) {
@@ -7237,14 +7283,21 @@ static void engram_adj_rebuild(EngramStore* g) {
if (ti >= 0 && g->adj_to[ti])
g->adj_to[ti][to_pos[ti]++] = (int)ei;
}
/* Copy counts */
/* Copy counts. cap == len after a fresh rebuild: the arrays are exactly
* sized, so the first incremental append to any list will grow it. */
for (int64_t i = 0; i < g->node_count; i++) {
g->adj_from_len[i] = from_cnt[i];
g->adj_to_len[i] = to_cnt[i];
g->adj_from_cap[i] = from_cnt[i];
g->adj_to_cap[i] = to_cnt[i];
}
free(from_cnt); free(to_cnt); free(from_pos); free(to_pos);
g->adj_node_count = g->node_count;
g->adj_dirty = 0;
/* M7 instrumentation (test-only counters; no behavioral effect). */
_eg_adj_rebuild_calls++;
_eg_adj_rebuild_edge_work += g->edge_count;
_eg_adj_maint_ns += _eg_adj_now_ns() - _t0;
}
static int64_t engram_find_node_index(const char* id) {
@@ -7388,6 +7441,108 @@ int engram_store_enabled(void) {
strcmp(f, "true") == 0)) ? 1 : 0;
}
/* ── M7: incremental adjacency maintenance (index-driven traversal) ───────────
*
* When ENGRAM_STORE is on, a single node/edge create keeps the already-built
* per-node adjacency index live by APPENDING to it, instead of marking it dirty
* and forcing the next activation to rebuild all O(E) adjacency lists from
* scratch. The result the BFS consumes is byte-identical to a full rebuild:
* - Edges are only ever appended to g->edges[], so their indices increase
* monotonically; appending in creation order reproduces the exact ascending
* edge-index ordering a rebuild's ei-ascending scan produces.
* - The same skip rule as rebuild applies: an edge with a NULL endpoint id
* contributes to NEITHER list.
* - Deletes/shifts (forget, prune, clear) still free the index and set
* adj_dirty=1, so any index-invalidating mutation falls back to a full
* rebuild. The append path only runs while the index is live and clean.
* Flag-off never reaches these helpers: the mutation sites call
* engram_adj_on_{node,edge}_added, which for flag-off simply set adj_dirty=1 —
* exactly the previous behavior, byte-for-byte. */
/* Grow the adjacency arrays so index `need`-1 is addressable. Preserves all
* existing lists; new slots are zeroed (NULL list, len 0, cap 0). Sets
* adj_node_count to the new allocated length so engram_adj_free frees exactly
* the slots that exist. Returns 0 on OOM (caller falls back to a full rebuild
* by setting adj_dirty). */
static int engram_adj_grow_slots(EngramStore* g, int64_t need) {
if (need <= g->adj_node_count) return 1;
int64_t nc = g->adj_node_count ? g->adj_node_count : 8;
while (nc < need) nc *= 2;
int** nf = realloc(g->adj_from, (size_t)nc * sizeof(int*));
int* nfl = realloc(g->adj_from_len, (size_t)nc * sizeof(int));
int* nfc = realloc(g->adj_from_cap, (size_t)nc * sizeof(int));
int** nt = realloc(g->adj_to, (size_t)nc * sizeof(int*));
int* ntl = realloc(g->adj_to_len, (size_t)nc * sizeof(int));
int* ntc = realloc(g->adj_to_cap, (size_t)nc * sizeof(int));
if (nf) g->adj_from = nf;
if (nfl) g->adj_from_len = nfl;
if (nfc) g->adj_from_cap = nfc;
if (nt) g->adj_to = nt;
if (ntl) g->adj_to_len = ntl;
if (ntc) g->adj_to_cap = ntc;
if (!nf || !nfl || !nfc || !nt || !ntl || !ntc) return 0;
for (int64_t i = g->adj_node_count; i < nc; i++) {
g->adj_from[i] = NULL; g->adj_from_len[i] = 0; g->adj_from_cap[i] = 0;
g->adj_to[i] = NULL; g->adj_to_len[i] = 0; g->adj_to_cap[i] = 0;
}
g->adj_node_count = nc;
return 1;
}
/* Append edge index `ei` to the list at (*arr,*len,*cap), doubling capacity as
* needed. Returns 0 on OOM. */
static int engram_adj_list_push(int** arr, int* len, int* cap, int ei) {
if (*len >= *cap) {
int ncap = *cap ? *cap * 2 : 2;
int* na = realloc(*arr, (size_t)ncap * sizeof(int));
if (!na) return 0;
*arr = na; *cap = ncap;
}
(*arr)[(*len)++] = ei;
return 1;
}
/* Append the freshly-created edge g->edges[ei] to the live adjacency index.
* Mirrors engram_adj_rebuild's per-edge classification exactly. Returns 0 on
* OOM (caller forces a rebuild). */
static int engram_adj_add_edge(EngramStore* g, int64_t ei) {
if (ei < 0 || ei >= g->edge_count) return 1;
EngramEdge* e = &g->edges[ei];
if (!e->from_id || !e->to_id) return 1; /* same skip rule as rebuild */
double _t0 = _eg_adj_now_ns();
int64_t fi = engram_idmap_get(g, e->from_id);
int64_t ti = engram_idmap_get(g, e->to_id);
int64_t hi = (fi > ti) ? fi : ti;
if (hi >= 0 && !engram_adj_grow_slots(g, hi + 1)) return 0;
if (fi >= 0 && !engram_adj_list_push(&g->adj_from[fi], &g->adj_from_len[fi],
&g->adj_from_cap[fi], (int)ei)) return 0;
if (ti >= 0 && !engram_adj_list_push(&g->adj_to[ti], &g->adj_to_len[ti],
&g->adj_to_cap[ti], (int)ei)) return 0;
_eg_adj_incr_appends++;
_eg_adj_maint_ns += _eg_adj_now_ns() - _t0;
return 1;
}
/* Mutation-site hook for a newly-appended node at the current top index. When
* the index is live and clean under ENGRAM_STORE, reserve its (empty) adjacency
* slot so a later BFS that seeds this node can index adj_*_len[idx] safely
* without a rebuild. Otherwise defer to the lazy full rebuild (flag-off path,
* or index not yet built / already dirty). */
static void engram_adj_on_node_added(EngramStore* g) {
if (engram_store_enabled() && g->adj_from && !g->adj_dirty) {
if (engram_adj_grow_slots(g, g->node_count)) return;
}
g->adj_dirty = 1; /* flag-off, or OOM/not-built: fall back to rebuild */
}
/* Mutation-site hook for the newly-appended edge at index g->edge_count-1. */
static void engram_adj_on_edge_added(EngramStore* g, int64_t ei) {
if (engram_store_enabled() && g->adj_from && !g->adj_dirty) {
if (engram_adj_add_edge(g, ei)) return;
}
g->adj_dirty = 1; /* flag-off, or OOM/not-built: fall back to rebuild */
}
/* EngramNode → borrowed StoreNode view (no ownership transfer; the store copies
* every field it persists, so shared string pointers are safe). */
static void eg_node_to_store(const EngramNode* n, StoreNode* sn) {
@@ -7637,7 +7792,7 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
int64_t new_idx = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7767,7 +7922,7 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
int64_t new_idx_full = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx_full);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -7838,7 +7993,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
int64_t new_idx_layered = g->node_count;
g->node_count++;
engram_idmap_put(g, n->id, new_idx_layered);
g->adj_dirty = 1;
engram_adj_on_node_added(g);
if (engram_store_enabled()) eg_store_put_node(n);
return el_wrap_str(el_strdup(n->id));
}
@@ -8343,7 +8498,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
e->last_fired = 0;
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
engram_adj_on_edge_added(g, g->edge_count - 1);
if (engram_store_enabled()) eg_store_put_edge(e);
}
@@ -9923,7 +10078,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
ne->last_fired = now_ms;
ne->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
g->adj_dirty = 1;
engram_adj_on_edge_added(g, g->edge_count - 1);
_eg_hebb_links_formed++;
hebb_edge_total++;
formed++;