engram: fix lazy-embed index gap (#20) and make activate's cosine scan lazy; extract vindex harvest primitive with a bench/oracle harness

Adds an O(1) "seen" bitmap so lazily-embedded older nodes get picked up
incrementally instead of only on a full rebuild (embed-gap #20).

Replaces engram_activate's O(N*D) cosine prescan with a lazy-memoized
cosine cache (eg_cosq_at), proven bit-identical to the old path.

Extracts a clean vindex_harvest_from_store primitive (read-only vector
harvest, careful malloc/ownership/error-path handling) reused by both
index-build and the new vindex_bench.c — a read-only proof harness
comparing brute-force vs HNSW recall/latency on both the real store and
synthetic data.

.nsbx-env intentionally excluded — local sandbox config (ports, paths,
dev-only placeholder key), not checked in.

lang/AGENTS.md

@@ -4,6 +4,37 @@ El is a self-hosting, statically-typed language that compiles to C. This file or
 
 ---
 
+## Current work in this worktree — the API reshape / decorated seam (IN PROGRESS, 2026-08-14)
+
+This is the `api-reshape` worktree. The build here reshapes Neuron's external
+surface and how it is *declared* — proven on isolated dev-port clones only; **live
+prod engram `:8742` is untouched and nothing is promoted.** Full framing lives in
+`neuron/docs/architecture/06-cognitive-architecture.md` (Update — 2026-08-14 deep
+night) and `02-components.md §5`.
+
+- **Surface collapse.** The ~90 noun-organized CRUD MCP tools collapse to a few
+  **geometry ops** — `read` (the *vantage-read*: re-origin + salience/recency +
+  an **aperture** → a bounded slice, curing the whole-self dump), `write`,
+  `relate`, `supersede` (evolve/tombstone/promote, never a hard delete) — plus the
+  agentic primitives `think`/`attend`/`learn`/`ground`/`assert`. The old noun is a
+  `type` parameter. Implemented in `tools/api-reshape/surface.el` with a parity
+  harness (`parity.sh`); aperture proven to bound output. **Not yet:** compiled
+  into the MCP server, hot-swap, all-alias dispatch.
+- **Decorated seam.** `@route(path,method,…)` makes codegen synthesize
+  `el_route_dispatch` (replacing the hand-written `handle_request` if-else) —
+  proven decorate→serve on `:8951`. `@manager`/`@engine`/`@accessor` are **parsed
+  but structurally inert** in the shipped compiler today; the `@route` codegen
+  lives on the **unmerged branch `feat/el-route-decorators`**. Telemetry-emit and
+  dharma-bus auto-wiring at the boundary are **staged, not shipped**. In-process,
+  an `@accessor` reaches the engram via **`engram_*` builtins**, not `http_get`.
+
+**Do not edit** the protected build sources while this is in flight:
+`el-compiler/src/codegen.el`, `el-compiler/runtime/el_seed.c` (and the archived
+`legacy/el_runtime.c`), the `runtime/engram_*.c` boot files, and `surface.el`
+(when present in the reshape tree) — these are owned by the build agents.
+
+---
+
 ## What El Is
 
 El compiles `.el` source → C → native binary. Every El value is `el_val_t` (int64_t). Strings are heap pointers cast through int64_t. The compiler is written in El (self-hosting).

lang/runtime/el_runtime.c

@@ -9352,6 +9352,21 @@ static double engram_goal_bias(const EngramNode* n, const char* query) {
 static VIndex*  _eg_vindex          = NULL;
 static int32_t  _eg_vindex_dim      = 0;
 static int64_t  _eg_vindex_built_nc = 0;   /* g->node_count at last (re)build */
+static uint8_t* _eg_vindex_seen     = NULL;/* per-ordinal: 1 iff inserted into _eg_vindex */
+static int64_t  _eg_vindex_seen_cap = 0;
+
+/* Grow the per-ordinal "indexed" bitmap to hold at least `need` entries, zeroing
+ * the new tail. Returns 0 on success, -1 on OOM (caller keeps the old map). */
+static int eg_vindex_seen_ensure(int64_t need) {
+    if (need <= _eg_vindex_seen_cap) return 0;
+    int64_t nc = _eg_vindex_seen_cap ? _eg_vindex_seen_cap : 1024;
+    while (nc < need) nc *= 2;
+    uint8_t* ns = (uint8_t*)realloc(_eg_vindex_seen, (size_t)nc);
+    if (!ns) return -1;
+    memset(ns + _eg_vindex_seen_cap, 0, (size_t)(nc - _eg_vindex_seen_cap));
+    _eg_vindex_seen = ns; _eg_vindex_seen_cap = nc;
+    return 0;
+}
 
 static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) {
     if (!g || dim <= 0) return _eg_vindex;
@@ -9360,22 +9375,32 @@ static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) {
     if (_eg_vindex && (_eg_vindex_dim != dim || g->node_count < _eg_vindex_built_nc)) {
         vindex_free(_eg_vindex);
         _eg_vindex = NULL; _eg_vindex_dim = 0; _eg_vindex_built_nc = 0;
+        free(_eg_vindex_seen); _eg_vindex_seen = NULL; _eg_vindex_seen_cap = 0;
     }
     if (!_eg_vindex) {
         VIndex* idx = vindex_create((int)dim, 0, 0);
         if (!idx) return NULL;
+        if (eg_vindex_seen_ensure(g->node_count)) { vindex_free(idx); return NULL; }
         for (int64_t i = 0; i < g->node_count; i++) {
             EngramNode* n = &g->nodes[i];
-            if (n->emb && n->emb_dim == dim)
-                (void)vindex_insert(idx, (uint64_t)i, n->emb);
+            if (n->emb && n->emb_dim == dim && vindex_insert(idx, (uint64_t)i, n->emb) == 0)
+                _eg_vindex_seen[i] = 1;
         }
         _eg_vindex = idx; _eg_vindex_dim = dim; _eg_vindex_built_nc = g->node_count;
-    } else if (g->node_count > _eg_vindex_built_nc) {
-        /* Incremental: index newly-appended nodes that already carry an emb. */
-        for (int64_t i = _eg_vindex_built_nc; i < g->node_count; i++) {
+    } else if (eg_vindex_seen_ensure(g->node_count) == 0) {
+        /* Incremental (embed-gap #20 fix): index EVERY node that now carries an emb
+         * but is not yet in the index — whether newly APPENDED or lazily EMBEDDED on
+         * an OLDER ordinal by the backfill loop. The previous tail-only scan left a
+         * lazily-embedded older node invisible until the next full rebuild; that node
+         * was silently absent from route_nearest / autoconnect (which have NO exact
+         * top-up, unlike engram_activate) and lowered activation recall. This O(node_count)
+         * presence check carries no D factor, so it is negligible beside the cosq scan on
+         * the same path. On seen-map OOM we fall through unchanged (index simply not grown). */
+        for (int64_t i = 0; i < g->node_count; i++) {
+            if (_eg_vindex_seen[i]) continue;
             EngramNode* n = &g->nodes[i];
-            if (n->emb && n->emb_dim == dim)
-                (void)vindex_insert(_eg_vindex, (uint64_t)i, n->emb);
+            if (n->emb && n->emb_dim == dim && vindex_insert(_eg_vindex, (uint64_t)i, n->emb) == 0)
+                _eg_vindex_seen[i] = 1;
         }
         _eg_vindex_built_nc = g->node_count;
     }
@@ -9438,6 +9463,25 @@ static int eg_geo_prime_max(void){      /* cap primed members added to the front
     if (v < 0) v = 0; if (v > 256) v = 256; return v;
 }
 
+/* Lazy per-node cosine vs the effective query (M8.1 activate-latency fix,
+ * 2026-08-14). Computes cos(node_i, query) ON DEMAND and memoizes it, replacing
+ * the full O(N·D) prescan that dominated live activate latency (~330ms + CPU peg
+ * + restart thrash). Returns exactly the value the prescan produced for any node
+ * it is asked about; nodes never asked about are never computed — and the prescan
+ * version never READ them either (Pass-3 skips !reached[i]; the exact top-up only
+ * runs when the ANN underfills), so engram_activate stays BIT-IDENTICAL while
+ * touching only ANN candidates + propagation-reached nodes. Degrades to the -2.0
+ * "no/!=dim embedding" sentinel exactly as the prescan did. Callers guard with
+ * `if (cosq)`, so cosq/cos_done are non-NULL here. */
+static inline double eg_cosq_at(EngramStore* g, double* cosq, unsigned char* cos_done,
+                                const float* qv, int32_t dim, int64_t i) {
+    if (cos_done[i]) return cosq[i];
+    EngramNode* n = &g->nodes[i];
+    cosq[i] = (n->emb && n->emb_dim == dim && qv) ? eg_cosine(n->emb, qv, dim) : -2.0;
+    cos_done[i] = 1;
+    return cosq[i];
+}
+
 el_val_t engram_activate(el_val_t query, el_val_t depth) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -9537,15 +9581,26 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
      * twice, coherently — HippoRAG). cosq stays NULL when the embedder is
      * unavailable; every consumer degrades to pure lexical behavior. */
     double* cosq = NULL;
+    unsigned char* cos_done = NULL;   /* M8.1: which cosq[i] have been computed (lazy) */
+    float* cosq_qv = NULL;            /* M8.1: OWNED copy of the effective query vector,
+                                       * kept alive past the e_eff free (9889) for the
+                                       * lazy cosq reads in Pass-2/Pass-3 propagation. */
     if (q_emb) {
         const float* qv = e_eff ? e_eff : q_emb;
-        cosq = calloc((size_t)g->node_count, sizeof(double));
-        if (cosq) {
-            for (int64_t i = 0; i < g->node_count; i++) {
-                EngramNode* n = &g->nodes[i];
-                cosq[i] = (n->emb && n->emb_dim == q_dim)
-                          ? eg_cosine(n->emb, qv, q_dim) : -2.0;
-            }
+        cosq     = calloc((size_t)g->node_count, sizeof(double));
+        cos_done = calloc((size_t)g->node_count, 1);
+        if (cosq && cos_done && qv) {
+            cosq_qv = malloc((size_t)q_dim * sizeof(float));
+            if (cosq_qv) memcpy(cosq_qv, qv, (size_t)q_dim * sizeof(float));
+        }
+        /* M8.1 activate-latency fix: NO full O(N·D) prescan here. cosq is filled
+         * lazily via eg_cosq_at() only for ANN candidates + propagation-reached
+         * nodes — bit-identical to the prescan (see eg_cosq_at). If any alloc
+         * failed, degrade to embedder-down behaviour: every `if (cosq)` consumer
+         * skips and activation falls back to pure lexical, exactly as before. */
+        if (!cosq || !cos_done || !cosq_qv) {
+            free(cosq); free(cos_done); free(cosq_qv);
+            cosq = NULL; cos_done = NULL; cosq_qv = NULL;
         }
     }
     /* NOTE (M8): e_eff is NOT freed here anymore — the effective query vector
@@ -9558,7 +9613,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     int64_t* best_hops  = calloc((size_t)g->node_count, sizeof(int64_t));
     int*     reached    = calloc((size_t)g->node_count, sizeof(int));
     if (!best_bg || !best_hops || !reached) {
-        free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out;
+        free(best_bg); free(best_hops); free(reached); free(cosq); free(cos_done); free(cosq_qv); free(e_eff); return out;
     }
 
     /* ── LAYER 1: broad fan-out (background activation) ─────────────────
@@ -9569,7 +9624,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry));
     int64_t seed_count = 0;
     if (!seeds) {
-        free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out;
+        free(best_bg); free(best_hops); free(reached); free(cosq); free(cos_done); free(cosq_qv); free(e_eff); return out;
     }
     /* Tokenize once: a node seeds if it matches ANY query token, and its seed
      * activation is scaled by token coverage (fraction of distinct query
@@ -9662,7 +9717,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
                         if (bi < 0 || bi >= g->node_count) continue; /* stale id guard */
                         if (reached[bi]) continue;              /* lexically seeded */
                         if (seed_dup && seed_dup[bi]) continue;
-                        double bc = cosq[bi];   /* EXACT cosine — parity gate */
+                        double bc = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, bi);   /* EXACT cosine — parity gate (lazy) */
                         if (!(bc > ENGRAM_EMBED_SEED_MIN)) continue;
                         EngramNode* n = &g->nodes[bi];
                         uint64_t key = eg_content_key(n);
@@ -9704,7 +9759,8 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
             for (int64_t i = 0; i < g->node_count; i++) {
                 if (reached[i]) continue;
                 if (seed_dup && seed_dup[i]) continue;
-                if (cosq[i] > bc) { bc = cosq[i]; bi = i; }
+                double ci = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, i);
+                if (ci > bc) { bc = ci; bi = i; }
             }
             if (bi < 0) break;
             EngramNode* n = &g->nodes[bi];
@@ -9880,7 +9936,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     typedef struct { int64_t idx; int64_t hops; double act; } Frontier;
     Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier));
     if (!fr) {
-        free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); return out;
+        free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); free(cos_done); free(cosq_qv); return out;
     }
     int64_t fhead = 0, ftail = 0;
     int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16);
@@ -9982,9 +10038,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
              * information, no penalty); cosq == NULL (embedder down) means
              * no gating at all — same graceful degradation as seeding. */
             double qgate = 1.0;
-            if (cosq && cosq[oi] > -1.5) {
-                double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0;
-                qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c;
+            if (cosq) {
+                double coi = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, oi);
+                if (coi > -1.5) {
+                    double c = coi > 0.0 ? coi : 0.0;
+                    qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c;
+                }
             }
             /* ── ACT-R fan effect (2026-08-11 self-review) ──
              * Symmetric degree normalization over the (source, target) pair.
@@ -10036,7 +10095,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     double* inhibition = calloc((size_t)g->node_count, sizeof(double));
     if (!inhibition) {
         free(best_bg); free(best_hops); free(reached); free(seeds); free(fr);
-        free(cosq); return out;
+        free(cosq); free(cos_done); free(cosq_qv); return out;
     }
     for (int64_t ei = 0; ei < g->edge_count; ei++) {
         EngramEdge* e = &g->edges[ei];
@@ -10061,7 +10120,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     double* wm_weights = calloc((size_t)g->node_count, sizeof(double));
     if (!wm_weights) {
         free(best_bg); free(best_hops); free(reached); free(seeds);
-        free(fr); free(inhibition); free(cosq); return out;
+        free(fr); free(inhibition); free(cosq); free(cos_done); free(cosq_qv); return out;
     }
     /* Per-call breakthrough budget (2026-08-02) — see ENGRAM_BREAKTHROUGH_BUDGET. */
     int64_t bt_budget = ENGRAM_BREAKTHROUGH_BUDGET;
@@ -10135,9 +10194,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
          * dragged in), not the semantic one (what the query actually
          * means); relevance to the current query is not stale merely
          * because the node was in WM a moment ago. */
-        if (cosq && cosq[i] > ENGRAM_EMBED_S0) {
-            raw_wm += ENGRAM_EMBED_WM_WEIGHT
-                    * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0);
+        if (cosq) {
+            double ci = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, i);
+            if (ci > ENGRAM_EMBED_S0) {
+                raw_wm += ENGRAM_EMBED_WM_WEIGHT
+                        * (ci - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0);
+            }
         }
         /* Threshold gate: must exceed per-type threshold to enter working
          * memory. Type threshold replaces the old flat 0.2 filter. */
@@ -10817,7 +10879,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     int64_t rcount = 0;
     if (!results) {
         free(best_bg); free(best_hops); free(reached); free(seeds);
-        free(fr); free(inhibition); free(wm_weights); free(cosq);
+        free(fr); free(inhibition); free(wm_weights); free(cosq); free(cos_done); free(cosq_qv);
         free(was_wm); return out;
     }
     for (int64_t i = 0; i < g->node_count; i++) {
@@ -10885,7 +10947,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     }
     free(best_bg); free(best_hops); free(reached);
     free(seeds); free(fr); free(inhibition); free(wm_weights); free(results);
-    free(cosq);
+    free(cosq); free(cos_done); free(cosq_qv);
     return out;
 }
 

lang/runtime/engram_vindex.c

@@ -538,19 +538,20 @@ static int strset_add(StrSet* s, const char* key){   /* 1 added, 0 dup, -1 err *
 }
 static void strset_free(StrSet* s){ for(size_t i=0;i<s->cap;i++) free(s->k[i]); free(s->k); }
 
-int vindex_build_from_store(VIndex* ix, const char* store_path,
-                            char*** ids_out, int* n_out){
-    if (!ix || !store_path) return -1;
+int vindex_harvest_from_store(const char* store_path, int dim,
+                              float** vecs_out, char*** ids_out, int* n_out){
+    if (!store_path || dim <= 0 || !vecs_out) return -1;
     int fd = open(store_path, O_RDONLY);
     if (fd < 0) return -1;
     struct stat st;
     if (fstat(fd, &st) != 0){ close(fd); return -1; }
     uint64_t npages = (uint64_t)st.st_size / VS_PAGE_SIZE;
 
-    char** ids = NULL; size_t ids_n = 0, ids_cap = 0;
+    float* vecs = NULL; size_t vn = 0, vcap = 0;   /* row-major float[vn*dim] */
+    char** ids  = NULL; size_t ids_n = 0, ids_cap = 0;
     StrSet seen = {0,0,0};
-    int inserted = 0;
     uint8_t page[VS_PAGE_SIZE];
+    int failed = 0;
 
     for (uint64_t pg = 2; pg < npages; pg++){       /* pages 0,1 = superblocks */
         if (vs_pread(fd, pg, page)) continue;
@@ -563,28 +564,67 @@ int vindex_build_from_store(VIndex* ix, const char* store_path,
             if ((size_t)off + VS_REC_HDR > VS_PAGE_SIZE) continue;
             uint8_t* body=NULL; size_t blen=0;
             if (vs_read_body(fd, page, off, len, &body, &blen)) continue;
-            char* id=NULL; float* emb=NULL; int dim=0;
-            vs_parse_node(body, blen, &id, &emb, &dim);
+            char* id=NULL; float* emb=NULL; int edim=0;
+            vs_parse_node(body, blen, &id, &emb, &edim);
             free(body);
-            if (!id || !emb || dim != ix->dim){ free(id); free(emb); continue; }
+            if (!id || !emb || edim != dim){ free(id); free(emb); continue; }
             int add = strset_add(&seen, id);
             if (add <= 0){ free(id); free(emb); continue; }   /* dup or err */
-            if (vindex_insert(ix, (uint64_t)inserted, emb) != 0){ free(id); free(emb); break; }
+            if (vn == vcap){
+                size_t nc = vcap ? vcap*2 : 1024;
+                float* nv = (float*)realloc(vecs, nc*(size_t)dim*sizeof(float));
+                if (!nv){ free(id); free(emb); failed = 1; goto out; }
+                vecs = nv; vcap = nc;
+            }
+            memcpy(vecs + vn*(size_t)dim, emb, (size_t)dim*sizeof(float));
             free(emb);
             if (ids_n == ids_cap){
-                size_t nc = ids_cap ? ids_cap*2 : 256;
+                size_t nc = ids_cap ? ids_cap*2 : 1024;
                 char** ni = (char**)realloc(ids, nc*sizeof(char*));
-                if (!ni){ free(id); break; }
+                if (!ni){ free(id); failed = 1; goto out; }
                 ids = ni; ids_cap = nc;
             }
             ids[ids_n++] = id;    /* transfers ownership */
-            inserted++;
+            vn++;
         }
     }
+out:
     close(fd);
     strset_free(&seen);
-    if (ids_out){ *ids_out = ids; if (n_out) *n_out = (int)ids_n; }
-    else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); if (n_out) *n_out=(int)ids_n; }
+    if (failed){
+        free(vecs);
+        for (size_t i=0;i<ids_n;i++) free(ids[i]);
+        free(ids);
+        return -1;
+    }
+    *vecs_out = vecs;
+    if (n_out) *n_out = (int)vn;
+    if (ids_out){ *ids_out = ids; }
+    else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); }
+    return (int)vn;
+}
+
+int vindex_build_from_store(VIndex* ix, const char* store_path,
+                            char*** ids_out, int* n_out){
+    if (!ix || !store_path) return -1;
+    float* vecs = NULL; char** ids = NULL; int n = 0;
+    int h = vindex_harvest_from_store(store_path, ix->dim, &vecs, &ids, &n);
+    if (h < 0) return -1;
+    int inserted = 0;
+    for (int i = 0; i < n; i++){
+        if (vindex_insert(ix, (uint64_t)inserted, vecs + (size_t)i*ix->dim) != 0) break;
+        inserted++;
+    }
+    free(vecs);
+    if (ids_out){
+        *ids_out = ids; if (n_out) *n_out = inserted;
+        /* free any ids beyond what we inserted (insert failure tail) */
+        for (int i = inserted; i < n; i++) free(ids[i]);
+    } else {
+        for (int i = 0; i < n; i++) free(ids[i]);
+        free(ids);
+        if (n_out) *n_out = inserted;
+    }
     return inserted;
 }
 

lang/runtime/engram_vindex.h

@@ -73,6 +73,18 @@ void vindex_free(VIndex* idx);
 int vindex_build_from_store(VIndex* idx, const char* store_path,
                             char*** ids_out, int* n_out);
 
+/* Read-only harvest of the raw (un-normalised) emb vectors from a paged store,
+ * applying the SAME filtering vindex_build_from_store does (live records only,
+ * deduped by store id, emb present with emb_dim == `dim`), in insertion order.
+ * On success sets *vecs_out to a malloc'd float[n*dim] (row i == the i-th kept
+ * vector) and *n_out to n; if `ids_out` is non-NULL, sets it to a malloc'd array
+ * of n strdup'd store ids (ids_out[i] == the id of row i). Caller frees *vecs_out,
+ * each id string, and the id array. Returns n, or <0 on error. Used both by
+ * vindex_build_from_store (which then inserts each row) and by benchmarks/oracles
+ * that need the same vector set the index holds. */
+int vindex_harvest_from_store(const char* store_path, int dim,
+                              float** vecs_out, char*** ids_out, int* n_out);
+
 /* Optional persistence (index is rebuildable from the store; provided for
  * convenience). vindex_save writes a self-describing snapshot; vindex_load
  * reconstructs an index from one. Return 0 / non-NULL on success. */

lang/runtime/vindex_bench.c

@@ -0,0 +1,238 @@
+/* vindex_bench.c — standalone proof harness for the engram HNSW ANN index.
+ *
+ * Measures brute-force cosine top-k (the correctness ORACLE) vs vindex_search
+ * (HNSW) on: (a) the REAL paged store harvested read-only, and (b) synthetic
+ * clustered data at several sizes to trace the scaling curve. Reports build time,
+ * per-query latency (brute vs HNSW), and recall@k (HNSW top-k vs brute top-k).
+ *
+ * Read-only: never opens a socket, never writes the store. Safe on an nsbx clone.
+ *
+ * Build:  cc -O2 -std=c11 vindex_bench.c engram_vindex.c -lm -o vindex_bench
+ * Usage:  vindex_bench store <neuron.egm> <dim> [nqueries] [k] [ef_csv]
+ *         vindex_bench synth <N> [dim] [clusters] [nqueries] [k] [ef_csv]
+ */
+#include "engram_vindex.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <stdint.h>
+#include <time.h>
+
+/* ── deterministic PRNG (splitmix64) so runs are reproducible ─────────────── */
+static uint64_t g_seed = 0xD1B54A32D192ED03ULL;
+static uint64_t sm(void){
+    uint64_t z = (g_seed += 0x9E3779B97F4A7C15ULL);
+    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+    z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+    return z ^ (z >> 31);
+}
+static double urand(void){ return (double)((sm() >> 11) + 1) * (1.0/9007199254740993.0); }
+static double grand(void){ /* Box-Muller */
+    double u1 = urand(), u2 = urand();
+    return sqrt(-2.0*log(u1)) * cos(2.0*M_PI*u2);
+}
+
+static double now_s(void){
+    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (double)ts.tv_sec + (double)ts.tv_nsec*1e-9;
+}
+
+/* L2-normalise a row in place. */
+static void l2norm(float* v, int dim){
+    double ss = 0; for (int i=0;i<dim;i++) ss += (double)v[i]*v[i];
+    if (ss > 0){ float inv = (float)(1.0/sqrt(ss)); for (int i=0;i<dim;i++) v[i]*=inv; }
+}
+
+/* Brute-force top-k by cosine distance (1 - dot on normalised vecs).
+ * data is n*dim, already L2-normalised. Writes k node ids (row indices) into
+ * out_ids ascending by distance. Returns nothing; assumes k<=n. */
+static void brute_topk(const float* data, int n, int dim, const float* q,
+                       int k, int* out_ids, float* out_d){
+    /* maintain a small sorted array of the k best (ascending distance). */
+    for (int i=0;i<k;i++){ out_ids[i]=-1; out_d[i]=2.0f+1.0f; }
+    for (int i=0;i<n;i++){
+        const float* r = data + (size_t)i*dim;
+        float s0=0,s1=0,s2=0,s3=0; int j=0;
+        for (; j+4<=dim; j+=4){ s0+=q[j]*r[j]; s1+=q[j+1]*r[j+1]; s2+=q[j+2]*r[j+2]; s3+=q[j+3]*r[j+3]; }
+        float dot=(s0+s1)+(s2+s3); for (; j<dim; j++) dot+=q[j]*r[j];
+        float d = 1.0f - dot;
+        if (d >= out_d[k-1]) continue;
+        int p = k-1;
+        while (p>0 && out_d[p-1] > d){ out_d[p]=out_d[p-1]; out_ids[p]=out_ids[p-1]; p--; }
+        out_d[p]=d; out_ids[p]=i;
+    }
+}
+
+/* recall@k: |brute_topk ∩ hnsw_topk| / k. Both are id arrays of length k. */
+static double recall_at_k(const int* gt, const uint64_t* ann, int nann, int k){
+    int hit = 0;
+    for (int i=0;i<k;i++){
+        if (gt[i] < 0) continue;
+        for (int j=0;j<nann;j++){ if ((int)ann[j] == gt[i]){ hit++; break; } }
+    }
+    return (double)hit / (double)k;
+}
+
+/* Parse "64,128,256" into an int array; returns count. */
+static int parse_csv(const char* s, int* out, int maxo){
+    int n=0; if(!s||!*s) return 0;
+    const char* p=s;
+    while(*p && n<maxo){ out[n++]=atoi(p); while(*p && *p!=',') p++; if(*p==',') p++; }
+    return n;
+}
+
+/* Generate n unit vectors on a LOW-DIMENSIONAL MANIFOLD, the property that makes
+ * real text embeddings tractable for ANN: each vector is a fixed random linear map
+ * A (dim × LATENT) applied to a latent gaussian z ∈ R^LATENT, plus small ambient
+ * noise, then L2-normalised. Points therefore lie near a `latent`-dim subspace, so
+ * every point has a well-defined tight neighbourhood (high recall) and the HNSW
+ * graph is cheap to build — unlike near-isotropic 768-d gaussians, where the curse
+ * of dimensionality makes all points near-equidistant (no structure → slow build,
+ * low recall) and unlike tight clusters (near-duplicates → artificial top-k ties).
+ * `sigma` is the ambient-noise scale. This reproduces the intrinsic-dimensionality
+ * regime of nomic embeddings, so the scaling curve reflects real-corpus behaviour. */
+#define SYNTH_LATENT 48
+static void gen_synth(float* data, int n, int dim, int clusters, double sigma){
+    (void)clusters;
+    float* A = malloc((size_t)dim*SYNTH_LATENT*sizeof(float));   /* fixed random basis */
+    for (size_t i=0;i<(size_t)dim*SYNTH_LATENT;i++) A[i]=(float)grand();
+    float z[SYNTH_LATENT];
+    for (int i=0;i<n;i++){
+        for (int l=0;l<SYNTH_LATENT;l++) z[l]=(float)grand();
+        float* v = data+(size_t)i*dim;
+        for (int j=0;j<dim;j++){
+            float acc = (float)(sigma*grand());
+            const float* row = A + (size_t)j*SYNTH_LATENT;
+            for (int l=0;l<SYNTH_LATENT;l++) acc += row[l]*z[l];
+            v[j]=acc;
+        }
+        l2norm(v, dim);
+    }
+    free(A);
+}
+
+/* Build M / ef_construction come from env (VIDX_M / VIDX_EFC) so the scaling
+ * sweep can trade build cost against graph quality without a recompile. 0 = default. */
+static int env_int(const char* k, int dflt){ const char* s=getenv(k); return (s&&*s)?atoi(s):dflt; }
+
+/* Run the full brute-vs-HNSW comparison over an already-normalised dataset. */
+static void run_bench(const char* label, float* data, int n, int dim,
+                      int nq, int k, int* efs, int nef, double build_s){
+    (void)build_s;
+    int bM = env_int("VIDX_M", 0), bEFC = env_int("VIDX_EFC", 0);
+    printf("\n=== %s : N=%d  dim=%d  k=%d  queries=%d ===\n", label, n, dim, k, nq);
+
+    /* build the index once (shared across ef settings). */
+    double t0 = now_s();
+    VIndex* ix = vindex_create(dim, bM, bEFC);
+    for (int i=0;i<n;i++) vindex_insert(ix, (uint64_t)i, data + (size_t)i*dim);
+    double bt = now_s()-t0;
+    printf("HNSW build: M=%d ef_construction=%d  ->  %.3f s  (%.1f k nodes/s)\n",
+           bM?bM:VINDEX_DEFAULT_M, bEFC?bEFC:VINDEX_DEFAULT_EF_CONSTRUCTION, bt, n/1000.0/bt);
+
+    /* choose query vectors: perturb random dataset rows (near-but-not-identical). */
+    int* qidx = malloc((size_t)nq*sizeof(int));
+    float* qv = malloc((size_t)nq*dim*sizeof(float));
+    for (int i=0;i<nq;i++){
+        int r = (int)(sm() % (uint64_t)n);
+        qidx[i]=r;
+        float* dst = qv+(size_t)i*dim; const float* src = data+(size_t)r*dim;
+        for (int j=0;j<dim;j++) dst[j] = src[j] + (float)(0.01*grand());
+        l2norm(dst, dim);
+    }
+
+    /* ground truth: brute-force top-k for every query (also the oracle latency). */
+    int* gt = malloc((size_t)nq*k*sizeof(int));
+    float* gd = malloc((size_t)k*sizeof(float));
+    double tb0 = now_s();
+    for (int i=0;i<nq;i++) brute_topk(data, n, dim, qv+(size_t)i*dim, k, gt+(size_t)i*k, gd);
+    double brute_ms = (now_s()-tb0)*1000.0/nq;
+    printf("BRUTE-FORCE  : %8.3f ms/query   (oracle; O(N*D))\n", brute_ms);
+
+    /* HNSW at each ef. */
+    uint64_t* aid = malloc((size_t)k*sizeof(uint64_t));
+    float* ad = malloc((size_t)k*sizeof(float));
+    printf("%-6s %14s %12s %10s\n", "ef", "HNSW ms/query", "speedup", "recall@k");
+    for (int e=0;e<nef;e++){
+        int ef = efs[e];
+        double th0 = now_s();
+        double rec_sum = 0;
+        for (int i=0;i<nq;i++){
+            int m = vindex_search(ix, qv+(size_t)i*dim, k, ef, aid, ad);
+            rec_sum += recall_at_k(gt+(size_t)i*k, aid, m, k);
+        }
+        double hnsw_ms = (now_s()-th0)*1000.0/nq;
+        printf("%-6d %14.4f %11.1fx %10.4f\n", ef, hnsw_ms, brute_ms/hnsw_ms, rec_sum/nq);
+    }
+
+    free(qidx); free(qv); free(gt); free(gd); free(aid); free(ad);
+    vindex_free(ix);
+}
+
+int main(int argc, char** argv){
+    setvbuf(stdout, NULL, _IOLBF, 0);   /* line-buffered so progress streams to a log */
+    if (argc < 2){ fprintf(stderr,"usage: %s store <path> <dim> [nq] [k] [ef_csv] | synth <N> [dim] [clusters] [nq] [k] [ef_csv] | sweep <dim> <N_csv> [nq] [k] [ef_csv]\n", argv[0]); return 2; }
+    int defef[8]; int ndef;
+
+    if (strcmp(argv[1],"sweep")==0){
+        if (argc < 4){ fprintf(stderr,"sweep needs <dim> <N_csv>\n"); return 2; }
+        int dim = atoi(argv[2]);
+        int Ns[16]; int nN = parse_csv(argv[3], Ns, 16);
+        int nq = (argc>4)?atoi(argv[4]):200;
+        int k  = (argc>5)?atoi(argv[5]):10;
+        ndef = (argc>6)?parse_csv(argv[6],defef,8):parse_csv("64,128,200",defef,8);
+        for (int s=0;s<nN;s++){
+            int N = Ns[s];
+            float* data = malloc((size_t)N*dim*sizeof(float));
+            if (!data){ fprintf(stderr,"OOM at N=%d\n",N); continue; }
+            int clusters = N/100; if (clusters < 64) clusters = 64;
+            gen_synth(data, N, dim, clusters, 1.0);
+            char lbl[64]; snprintf(lbl,sizeof lbl,"SYNTH N=%d", N);
+            run_bench(lbl, data, N, dim, nq, k, defef, ndef, 0.0);
+            free(data);
+        }
+        return 0;
+    }
+
+    if (strcmp(argv[1],"store")==0){
+        if (argc < 4){ fprintf(stderr,"store needs <path> <dim>\n"); return 2; }
+        const char* path = argv[2]; int dim = atoi(argv[3]);
+        int nq = (argc>4)?atoi(argv[4]):500;
+        int k  = (argc>5)?atoi(argv[5]):10;
+        ndef = (argc>6)?parse_csv(argv[6],defef,8):parse_csv("32,64,128,200,400",defef,8);
+        printf("Harvesting emb vectors from %s (dim=%d) ...\n", path, dim);
+        float* data=NULL; int n=0;
+        double t0=now_s();
+        int h = vindex_harvest_from_store(path, dim, &data, NULL, &n);
+        double harvest_s = now_s()-t0;
+        if (h < 0 || n == 0){ fprintf(stderr,"harvest failed (h=%d n=%d) — wrong dim or path?\n", h, n); return 1; }
+        printf("Harvested %d live embedded nodes in %.2f s\n", n, harvest_s);
+        for (int i=0;i<n;i++) l2norm(data+(size_t)i*dim, dim);   /* oracle needs normalised */
+        if (nq > n) nq = n;
+        run_bench("REAL STORE", data, n, dim, nq, k, defef, ndef, 0.0);
+        free(data);
+        return 0;
+    }
+
+    if (strcmp(argv[1],"synth")==0){
+        if (argc < 3){ fprintf(stderr,"synth needs <N>\n"); return 2; }
+        int N = atoi(argv[2]);
+        int dim = (argc>3)?atoi(argv[3]):768;
+        int clusters = (argc>4)?atoi(argv[4]):200;
+        int nq = (argc>5)?atoi(argv[5]):500;
+        int k  = (argc>6)?atoi(argv[6]):10;
+        ndef = (argc>7)?parse_csv(argv[7],defef,8):parse_csv("64,128,200",defef,8);
+        printf("Generating %d synthetic clustered vectors (dim=%d clusters=%d) ...\n", N, dim, clusters);
+        float* data = malloc((size_t)N*dim*sizeof(float));
+        if (!data){ fprintf(stderr,"OOM allocating %zu bytes\n", (size_t)N*dim*sizeof(float)); return 1; }
+        gen_synth(data, N, dim, clusters, 0.35);
+        char lbl[64]; snprintf(lbl,sizeof lbl,"SYNTH");
+        run_bench(lbl, data, N, dim, nq, k, defef, ndef, 0.0);
+        free(data);
+        return 0;
+    }
+
+    fprintf(stderr,"unknown mode '%s'\n", argv[1]);
+    return 2;
+}