neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests 4 Actions Packages Projects Releases 5 Wiki Activity

M8: wire engram_vindex ANN into engram_activate seed selection (O(n)->ANN, O(n) fallback preserved)

Browse Source 
Persistent process-lifetime HNSW index (engram_vindex) over resident node
embeddings, node_id == resident g->nodes[] index. Built lazily on first
activation, grown incrementally as appended nodes get embedded, rebuilt on
emb-dim change or resident-array shrink.

Seed selection queries the ANN for the nearest embedded nodes to the effective
query vector, replacing the O(K*N) exact argmax scan as the DISCOVERY step.
Each ANN candidate is admitted through the identical gate the exact scan uses
(exact cosine >= SEED_MIN via cosq, reached/dup skips, content dedup, same
decay/dampen shaping), so scoring/dynamics are unchanged. The exact argmax
scan is preserved verbatim and tops up any unfilled seed slot, and runs in
full when the index is empty/too-small/unavailable -> pre-M8 behaviour exactly.

cosq (O(N*dim) cosine fill) is intentionally retained: it still feeds the
propagation qgate and the Pass-2 WM term (activation dynamics, out of M8
scope). ANN accelerates SELECTION only.
This commit is contained in:
Will Anderson
2026-08-12 17:13:06 -05:00
parent e0b55c3080
commit 1507614dbf
1 changed files with 127 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/el_runtime.c
+127 -3
View File
@@ -7432,6 +7432,7 @@ static char* engram_first_n_chars(const char* s, size_t n) {
* WAL-logged API so neuron.egm/neuron.wal stay authoritative.
* ══════════════════════════════════════════════════════════════════════════ */
#include "engram_store.h"
#include "engram_vindex.h" /* M8: ANN (HNSW) index for activation seed selection */
static EngramPagedStore* g_engram_store = NULL;
@@ -8963,6 +8964,58 @@ static double engram_goal_bias(const EngramNode* n, const char* query) {
return bias;
}
/* ── M8 wiring: persistent ANN over resident node embeddings ────────────────
* A single process-lifetime HNSW index (engram_vindex) accelerates activation
* seed SELECTION (see engram_activate). The index node_id IS the resident
* g->nodes[] index, so a search result maps back with zero lookup. Built lazily
* on first use from every resident node carrying an emb of the query dim; grown
* incrementally as newly-appended nodes get embedded (cheap tail scan — the
* activation backfill loop mints embeddings newest-first, so fresh content is
* indexed within a scan cycle); fully rebuilt only when the emb dim changes or
* the resident array SHRINKS (a compaction/reorder that could invalidate cached
* indices).
*
* STALENESS (honest tradeoff): an embedding minted on an OLDER node (index below
* the last-built count) by the backfill loop is not indexed until the next full
* rebuild (process restart, dim change, or a shrink). This can only LOWER recall
* for those nodes — it can NEVER mis-seed — because in engram_activate every ANN
* candidate is re-validated against the EXACT cosine (cosq[bi] ≥ SEED_MIN) and
* the exact O(n) argmax scan tops up any seed slot the ANN leaves unfilled.
* Single-threaded, matching the adjacent query-embedding cache (no lock).
* Returns NULL when no index is available → caller falls back to the O(n) scan. */
static VIndex* _eg_vindex = NULL;
static int32_t _eg_vindex_dim = 0;
static int64_t _eg_vindex_built_nc = 0; /* g->node_count at last (re)build */
static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) {
if (!g || dim <= 0) return _eg_vindex;
/* Drop a stale index: embedder dim changed, or the resident array shrank
* (indices may have been reused/reordered → cached node_ids unsafe). */
if (_eg_vindex && (_eg_vindex_dim != dim || g->node_count < _eg_vindex_built_nc)) {
vindex_free(_eg_vindex);
_eg_vindex = NULL; _eg_vindex_dim = 0; _eg_vindex_built_nc = 0;
}
if (!_eg_vindex) {
VIndex* idx = vindex_create((int)dim, 0, 0);
if (!idx) return NULL;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
if (n->emb && n->emb_dim == dim)
(void)vindex_insert(idx, (uint64_t)i, n->emb);
}
_eg_vindex = idx; _eg_vindex_dim = dim; _eg_vindex_built_nc = g->node_count;
} else if (g->node_count > _eg_vindex_built_nc) {
/* Incremental: index newly-appended nodes that already carry an emb. */
for (int64_t i = _eg_vindex_built_nc; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
if (n->emb && n->emb_dim == dim)
(void)vindex_insert(_eg_vindex, (uint64_t)i, n->emb);
}
_eg_vindex_built_nc = g->node_count;
}
return _eg_vindex;
}
el_val_t engram_activate(el_val_t query, el_val_t depth) {
EngramStore* g = engram_get();
const char* q = EL_CSTR(query);
@@ -9070,14 +9123,17 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
}
}
}
free(e_eff); e_eff = NULL; /* only needed to fill cosq */
/* NOTE (M8): e_eff is NOT freed here anymore — the effective query vector
* (query ⊕ context centroid) is reused below as the ANN query for seed
* selection so the ANN searches the SAME direction cosq was scored against.
* It is freed right after the semantic-seed-supplement block. */
/* Per-node layer-1 tracking. */
double* best_bg = calloc((size_t)g->node_count, sizeof(double));
int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t));
int* reached = calloc((size_t)g->node_count, sizeof(int));
if (!best_bg || !best_hops || !reached) {
free(best_bg); free(best_hops); free(reached); free(cosq); return out;
free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out;
}
/* ── LAYER 1: broad fan-out (background activation) ─────────────────
@@ -9088,7 +9144,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry));
int64_t seed_count = 0;
if (!seeds) {
free(best_bg); free(best_hops); free(reached); free(cosq); return out;
free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out;
}
/* Tokenize once: a node seeds if it matches ANY query token, and its seed
* activation is scaled by token coverage (fraction of distinct query
@@ -9150,6 +9206,73 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
int64_t sel[ENGRAM_EMBED_SEED_K];
uint64_t selkey[ENGRAM_EMBED_SEED_K];
int nsel = 0;
/* ── M8: ANN-accelerated seed candidates (engram_vindex) ────────────
* Ask the persistent HNSW index for the nearest embedded nodes to the
* effective query vector in ~O(log N), replacing the O(K·N) exact
* argmax scan below as the seed DISCOVERY mechanism. Every ANN
* candidate is then admitted through the IDENTICAL gate the exact scan
* uses — real cosine threshold (cosq[bi] > SEED_MIN, which also rejects
* the -2.0 "no/!=dim emb" sentinel), reached[]/seed_dup[] skips, content
* dedup, and the same decay/dampen shaping — so the ANN changes only
* WHICH nodes are discovered, never how a discovered node is scored or
* seeded. The exact scan below is preserved verbatim and tops up any
* slot the ANN leaves unfilled (recall < 1, or index stale/absent), so
* seed quality can never regress and the pre-M8 behaviour is recovered
* exactly when the index is unavailable. Request K*8 candidates — the
* same budget as the exact scan's retry `guard` — so dedup/threshold
* rejects still leave enough distinct seeds. */
{
VIndex* vx = eg_vindex_sync(g, q_dim);
if (vx && (int64_t)vindex_size(vx) >= ENGRAM_EMBED_SEED_K) {
const float* seed_qv = e_eff ? e_eff : q_emb;
int kreq = ENGRAM_EMBED_SEED_K * 8;
uint64_t* aid = malloc((size_t)kreq * sizeof(uint64_t));
float* ad = malloc((size_t)kreq * sizeof(float));
if (seed_qv && aid && ad) {
int got = vindex_search(vx, seed_qv, kreq,
VINDEX_DEFAULT_EF_SEARCH, aid, ad);
for (int r = 0; r < got && nsel < ENGRAM_EMBED_SEED_K; r++) {
int64_t bi = (int64_t)aid[r];
if (bi < 0 || bi >= g->node_count) continue; /* stale id guard */
if (reached[bi]) continue; /* lexically seeded */
if (seed_dup && seed_dup[bi]) continue;
double bc = cosq[bi]; /* EXACT cosine — parity gate */
if (!(bc > ENGRAM_EMBED_SEED_MIN)) continue;
EngramNode* n = &g->nodes[bi];
uint64_t key = eg_content_key(n);
int dup = 0;
for (int s = 0; s < nsel; s++) {
if (eg_same_content(n, &g->nodes[sel[s]], key, selkey[s])) {
dup = 1; break;
}
}
if (dup) {
_eg_act_dup_seeds++;
if (seed_dup) seed_dup[bi] = 1;
continue;
}
double tdecay = engram_temporal_decay(n, now_ms);
double dampen = engram_activation_dampen(n);
double act = bc * tdecay * dampen;
seeds[seed_count].idx = bi;
seeds[seed_count].act = act;
seeds[seed_count].created_at = n->created_at;
seed_count++;
best_bg[bi] = act;
best_hops[bi] = 0;
reached[bi] = 1;
sel[nsel] = bi; selkey[nsel] = key; nsel++;
}
}
free(aid); free(ad);
}
}
/* Exact O(n) argmax fallback / top-up (pre-M8 selection, verbatim).
* Runs only for seed slots the ANN did not fill: when the index is
* unavailable it fills all K (identical to pre-M8); when the ANN filled
* all K the `nsel < K` guard makes this a no-op (no O(n) scan). */
int guard = ENGRAM_EMBED_SEED_K * 8;
while (nsel < ENGRAM_EMBED_SEED_K && guard-- > 0) {
int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN;
@@ -9186,6 +9309,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
}
free(seed_dup);
}
free(e_eff); e_eff = NULL; /* M8: no longer needed past seed selection */
/* Compute mean seed created_at for temporal proximity bonus.
* Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 —
* which is NOT the arithmetic mean: it exponentially over-weights the