From 08cbcef5d9aee1b6d43010d9c323c769f66cfdeb Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 14:26:16 -0500 Subject: [PATCH] engram: fix lazy-embed index gap (#20) and make activate's cosine scan lazy; extract vindex harvest primitive with a bench/oracle harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an O(1) "seen" bitmap so lazily-embedded older nodes get picked up incrementally instead of only on a full rebuild (embed-gap #20). Replaces engram_activate's O(N*D) cosine prescan with a lazy-memoized cosine cache (eg_cosq_at), proven bit-identical to the old path. Extracts a clean vindex_harvest_from_store primitive (read-only vector harvest, careful malloc/ownership/error-path handling) reused by both index-build and the new vindex_bench.c — a read-only proof harness comparing brute-force vs HNSW recall/latency on both the real store and synthetic data. .nsbx-env intentionally excluded — local sandbox config (ports, paths, dev-only placeholder key), not checked in. --- lang/AGENTS.md | 31 +++++ lang/runtime/el_runtime.c | 120 +++++++++++++----- lang/runtime/engram_vindex.c | 68 +++++++--- lang/runtime/engram_vindex.h | 12 ++ lang/runtime/vindex_bench.c | 238 +++++++++++++++++++++++++++++++++++ 5 files changed, 426 insertions(+), 43 deletions(-) create mode 100644 lang/runtime/vindex_bench.c diff --git a/lang/AGENTS.md b/lang/AGENTS.md index ea93660..1471c3d 100644 --- a/lang/AGENTS.md +++ b/lang/AGENTS.md @@ -4,6 +4,37 @@ El is a self-hosting, statically-typed language that compiles to C. This file or --- +## Current work in this worktree — the API reshape / decorated seam (IN PROGRESS, 2026-08-14) + +This is the `api-reshape` worktree. The build here reshapes Neuron's external +surface and how it is *declared* — proven on isolated dev-port clones only; **live +prod engram `:8742` is untouched and nothing is promoted.** Full framing lives in +`neuron/docs/architecture/06-cognitive-architecture.md` (Update — 2026-08-14 deep +night) and `02-components.md §5`. + +- **Surface collapse.** The ~90 noun-organized CRUD MCP tools collapse to a few + **geometry ops** — `read` (the *vantage-read*: re-origin + salience/recency + + an **aperture** → a bounded slice, curing the whole-self dump), `write`, + `relate`, `supersede` (evolve/tombstone/promote, never a hard delete) — plus the + agentic primitives `think`/`attend`/`learn`/`ground`/`assert`. The old noun is a + `type` parameter. Implemented in `tools/api-reshape/surface.el` with a parity + harness (`parity.sh`); aperture proven to bound output. **Not yet:** compiled + into the MCP server, hot-swap, all-alias dispatch. +- **Decorated seam.** `@route(path,method,…)` makes codegen synthesize + `el_route_dispatch` (replacing the hand-written `handle_request` if-else) — + proven decorate→serve on `:8951`. `@manager`/`@engine`/`@accessor` are **parsed + but structurally inert** in the shipped compiler today; the `@route` codegen + lives on the **unmerged branch `feat/el-route-decorators`**. Telemetry-emit and + dharma-bus auto-wiring at the boundary are **staged, not shipped**. In-process, + an `@accessor` reaches the engram via **`engram_*` builtins**, not `http_get`. + +**Do not edit** the protected build sources while this is in flight: +`el-compiler/src/codegen.el`, `el-compiler/runtime/el_seed.c` (and the archived +`legacy/el_runtime.c`), the `runtime/engram_*.c` boot files, and `surface.el` +(when present in the reshape tree) — these are owned by the build agents. + +--- + ## What El Is El compiles `.el` source → C → native binary. Every El value is `el_val_t` (int64_t). Strings are heap pointers cast through int64_t. The compiler is written in El (self-hosting). diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 1a089d0..e9bffae 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -9352,6 +9352,21 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { static VIndex* _eg_vindex = NULL; static int32_t _eg_vindex_dim = 0; static int64_t _eg_vindex_built_nc = 0; /* g->node_count at last (re)build */ +static uint8_t* _eg_vindex_seen = NULL;/* per-ordinal: 1 iff inserted into _eg_vindex */ +static int64_t _eg_vindex_seen_cap = 0; + +/* Grow the per-ordinal "indexed" bitmap to hold at least `need` entries, zeroing + * the new tail. Returns 0 on success, -1 on OOM (caller keeps the old map). */ +static int eg_vindex_seen_ensure(int64_t need) { + if (need <= _eg_vindex_seen_cap) return 0; + int64_t nc = _eg_vindex_seen_cap ? _eg_vindex_seen_cap : 1024; + while (nc < need) nc *= 2; + uint8_t* ns = (uint8_t*)realloc(_eg_vindex_seen, (size_t)nc); + if (!ns) return -1; + memset(ns + _eg_vindex_seen_cap, 0, (size_t)(nc - _eg_vindex_seen_cap)); + _eg_vindex_seen = ns; _eg_vindex_seen_cap = nc; + return 0; +} static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) { if (!g || dim <= 0) return _eg_vindex; @@ -9360,22 +9375,32 @@ static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) { if (_eg_vindex && (_eg_vindex_dim != dim || g->node_count < _eg_vindex_built_nc)) { vindex_free(_eg_vindex); _eg_vindex = NULL; _eg_vindex_dim = 0; _eg_vindex_built_nc = 0; + free(_eg_vindex_seen); _eg_vindex_seen = NULL; _eg_vindex_seen_cap = 0; } if (!_eg_vindex) { VIndex* idx = vindex_create((int)dim, 0, 0); if (!idx) return NULL; + if (eg_vindex_seen_ensure(g->node_count)) { vindex_free(idx); return NULL; } for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (n->emb && n->emb_dim == dim) - (void)vindex_insert(idx, (uint64_t)i, n->emb); + if (n->emb && n->emb_dim == dim && vindex_insert(idx, (uint64_t)i, n->emb) == 0) + _eg_vindex_seen[i] = 1; } _eg_vindex = idx; _eg_vindex_dim = dim; _eg_vindex_built_nc = g->node_count; - } else if (g->node_count > _eg_vindex_built_nc) { - /* Incremental: index newly-appended nodes that already carry an emb. */ - for (int64_t i = _eg_vindex_built_nc; i < g->node_count; i++) { + } else if (eg_vindex_seen_ensure(g->node_count) == 0) { + /* Incremental (embed-gap #20 fix): index EVERY node that now carries an emb + * but is not yet in the index — whether newly APPENDED or lazily EMBEDDED on + * an OLDER ordinal by the backfill loop. The previous tail-only scan left a + * lazily-embedded older node invisible until the next full rebuild; that node + * was silently absent from route_nearest / autoconnect (which have NO exact + * top-up, unlike engram_activate) and lowered activation recall. This O(node_count) + * presence check carries no D factor, so it is negligible beside the cosq scan on + * the same path. On seen-map OOM we fall through unchanged (index simply not grown). */ + for (int64_t i = 0; i < g->node_count; i++) { + if (_eg_vindex_seen[i]) continue; EngramNode* n = &g->nodes[i]; - if (n->emb && n->emb_dim == dim) - (void)vindex_insert(_eg_vindex, (uint64_t)i, n->emb); + if (n->emb && n->emb_dim == dim && vindex_insert(_eg_vindex, (uint64_t)i, n->emb) == 0) + _eg_vindex_seen[i] = 1; } _eg_vindex_built_nc = g->node_count; } @@ -9438,6 +9463,25 @@ static int eg_geo_prime_max(void){ /* cap primed members added to the front if (v < 0) v = 0; if (v > 256) v = 256; return v; } +/* Lazy per-node cosine vs the effective query (M8.1 activate-latency fix, + * 2026-08-14). Computes cos(node_i, query) ON DEMAND and memoizes it, replacing + * the full O(N·D) prescan that dominated live activate latency (~330ms + CPU peg + * + restart thrash). Returns exactly the value the prescan produced for any node + * it is asked about; nodes never asked about are never computed — and the prescan + * version never READ them either (Pass-3 skips !reached[i]; the exact top-up only + * runs when the ANN underfills), so engram_activate stays BIT-IDENTICAL while + * touching only ANN candidates + propagation-reached nodes. Degrades to the -2.0 + * "no/!=dim embedding" sentinel exactly as the prescan did. Callers guard with + * `if (cosq)`, so cosq/cos_done are non-NULL here. */ +static inline double eg_cosq_at(EngramStore* g, double* cosq, unsigned char* cos_done, + const float* qv, int32_t dim, int64_t i) { + if (cos_done[i]) return cosq[i]; + EngramNode* n = &g->nodes[i]; + cosq[i] = (n->emb && n->emb_dim == dim && qv) ? eg_cosine(n->emb, qv, dim) : -2.0; + cos_done[i] = 1; + return cosq[i]; +} + el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -9537,15 +9581,26 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * twice, coherently — HippoRAG). cosq stays NULL when the embedder is * unavailable; every consumer degrades to pure lexical behavior. */ double* cosq = NULL; + unsigned char* cos_done = NULL; /* M8.1: which cosq[i] have been computed (lazy) */ + float* cosq_qv = NULL; /* M8.1: OWNED copy of the effective query vector, + * kept alive past the e_eff free (9889) for the + * lazy cosq reads in Pass-2/Pass-3 propagation. */ if (q_emb) { const float* qv = e_eff ? e_eff : q_emb; - cosq = calloc((size_t)g->node_count, sizeof(double)); - if (cosq) { - for (int64_t i = 0; i < g->node_count; i++) { - EngramNode* n = &g->nodes[i]; - cosq[i] = (n->emb && n->emb_dim == q_dim) - ? eg_cosine(n->emb, qv, q_dim) : -2.0; - } + cosq = calloc((size_t)g->node_count, sizeof(double)); + cos_done = calloc((size_t)g->node_count, 1); + if (cosq && cos_done && qv) { + cosq_qv = malloc((size_t)q_dim * sizeof(float)); + if (cosq_qv) memcpy(cosq_qv, qv, (size_t)q_dim * sizeof(float)); + } + /* M8.1 activate-latency fix: NO full O(N·D) prescan here. cosq is filled + * lazily via eg_cosq_at() only for ANN candidates + propagation-reached + * nodes — bit-identical to the prescan (see eg_cosq_at). If any alloc + * failed, degrade to embedder-down behaviour: every `if (cosq)` consumer + * skips and activation falls back to pure lexical, exactly as before. */ + if (!cosq || !cos_done || !cosq_qv) { + free(cosq); free(cos_done); free(cosq_qv); + cosq = NULL; cos_done = NULL; cosq_qv = NULL; } } /* NOTE (M8): e_eff is NOT freed here anymore — the effective query vector @@ -9558,7 +9613,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); int* reached = calloc((size_t)g->node_count, sizeof(int)); if (!best_bg || !best_hops || !reached) { - free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out; + free(best_bg); free(best_hops); free(reached); free(cosq); free(cos_done); free(cosq_qv); free(e_eff); return out; } /* ── LAYER 1: broad fan-out (background activation) ───────────────── @@ -9569,7 +9624,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry)); int64_t seed_count = 0; if (!seeds) { - free(best_bg); free(best_hops); free(reached); free(cosq); free(e_eff); return out; + free(best_bg); free(best_hops); free(reached); free(cosq); free(cos_done); free(cosq_qv); free(e_eff); return out; } /* Tokenize once: a node seeds if it matches ANY query token, and its seed * activation is scaled by token coverage (fraction of distinct query @@ -9662,7 +9717,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (bi < 0 || bi >= g->node_count) continue; /* stale id guard */ if (reached[bi]) continue; /* lexically seeded */ if (seed_dup && seed_dup[bi]) continue; - double bc = cosq[bi]; /* EXACT cosine — parity gate */ + double bc = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, bi); /* EXACT cosine — parity gate (lazy) */ if (!(bc > ENGRAM_EMBED_SEED_MIN)) continue; EngramNode* n = &g->nodes[bi]; uint64_t key = eg_content_key(n); @@ -9704,7 +9759,8 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { for (int64_t i = 0; i < g->node_count; i++) { if (reached[i]) continue; if (seed_dup && seed_dup[i]) continue; - if (cosq[i] > bc) { bc = cosq[i]; bi = i; } + double ci = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, i); + if (ci > bc) { bc = ci; bi = i; } } if (bi < 0) break; EngramNode* n = &g->nodes[bi]; @@ -9880,7 +9936,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); if (!fr) { - free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); return out; + free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); free(cos_done); free(cosq_qv); return out; } int64_t fhead = 0, ftail = 0; int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16); @@ -9982,9 +10038,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * information, no penalty); cosq == NULL (embedder down) means * no gating at all — same graceful degradation as seeding. */ double qgate = 1.0; - if (cosq && cosq[oi] > -1.5) { - double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0; - qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; + if (cosq) { + double coi = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, oi); + if (coi > -1.5) { + double c = coi > 0.0 ? coi : 0.0; + qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; + } } /* ── ACT-R fan effect (2026-08-11 self-review) ── * Symmetric degree normalization over the (source, target) pair. @@ -10036,7 +10095,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double* inhibition = calloc((size_t)g->node_count, sizeof(double)); if (!inhibition) { free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); - free(cosq); return out; + free(cosq); free(cos_done); free(cosq_qv); return out; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; @@ -10061,7 +10120,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); if (!wm_weights) { free(best_bg); free(best_hops); free(reached); free(seeds); - free(fr); free(inhibition); free(cosq); return out; + free(fr); free(inhibition); free(cosq); free(cos_done); free(cosq_qv); return out; } /* Per-call breakthrough budget (2026-08-02) — see ENGRAM_BREAKTHROUGH_BUDGET. */ int64_t bt_budget = ENGRAM_BREAKTHROUGH_BUDGET; @@ -10135,9 +10194,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * dragged in), not the semantic one (what the query actually * means); relevance to the current query is not stale merely * because the node was in WM a moment ago. */ - if (cosq && cosq[i] > ENGRAM_EMBED_S0) { - raw_wm += ENGRAM_EMBED_WM_WEIGHT - * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); + if (cosq) { + double ci = eg_cosq_at(g, cosq, cos_done, cosq_qv, q_dim, i); + if (ci > ENGRAM_EMBED_S0) { + raw_wm += ENGRAM_EMBED_WM_WEIGHT + * (ci - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); + } } /* Threshold gate: must exceed per-type threshold to enter working * memory. Type threshold replaces the old flat 0.2 filter. */ @@ -10817,7 +10879,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { int64_t rcount = 0; if (!results) { free(best_bg); free(best_hops); free(reached); free(seeds); - free(fr); free(inhibition); free(wm_weights); free(cosq); + free(fr); free(inhibition); free(wm_weights); free(cosq); free(cos_done); free(cosq_qv); free(was_wm); return out; } for (int64_t i = 0; i < g->node_count; i++) { @@ -10885,7 +10947,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); - free(cosq); + free(cosq); free(cos_done); free(cosq_qv); return out; } diff --git a/lang/runtime/engram_vindex.c b/lang/runtime/engram_vindex.c index d9c1571..bb80572 100644 --- a/lang/runtime/engram_vindex.c +++ b/lang/runtime/engram_vindex.c @@ -538,19 +538,20 @@ static int strset_add(StrSet* s, const char* key){ /* 1 added, 0 dup, -1 err * } static void strset_free(StrSet* s){ for(size_t i=0;icap;i++) free(s->k[i]); free(s->k); } -int vindex_build_from_store(VIndex* ix, const char* store_path, - char*** ids_out, int* n_out){ - if (!ix || !store_path) return -1; +int vindex_harvest_from_store(const char* store_path, int dim, + float** vecs_out, char*** ids_out, int* n_out){ + if (!store_path || dim <= 0 || !vecs_out) return -1; int fd = open(store_path, O_RDONLY); if (fd < 0) return -1; struct stat st; if (fstat(fd, &st) != 0){ close(fd); return -1; } uint64_t npages = (uint64_t)st.st_size / VS_PAGE_SIZE; - char** ids = NULL; size_t ids_n = 0, ids_cap = 0; + float* vecs = NULL; size_t vn = 0, vcap = 0; /* row-major float[vn*dim] */ + char** ids = NULL; size_t ids_n = 0, ids_cap = 0; StrSet seen = {0,0,0}; - int inserted = 0; uint8_t page[VS_PAGE_SIZE]; + int failed = 0; for (uint64_t pg = 2; pg < npages; pg++){ /* pages 0,1 = superblocks */ if (vs_pread(fd, pg, page)) continue; @@ -563,28 +564,67 @@ int vindex_build_from_store(VIndex* ix, const char* store_path, if ((size_t)off + VS_REC_HDR > VS_PAGE_SIZE) continue; uint8_t* body=NULL; size_t blen=0; if (vs_read_body(fd, page, off, len, &body, &blen)) continue; - char* id=NULL; float* emb=NULL; int dim=0; - vs_parse_node(body, blen, &id, &emb, &dim); + char* id=NULL; float* emb=NULL; int edim=0; + vs_parse_node(body, blen, &id, &emb, &edim); free(body); - if (!id || !emb || dim != ix->dim){ free(id); free(emb); continue; } + if (!id || !emb || edim != dim){ free(id); free(emb); continue; } int add = strset_add(&seen, id); if (add <= 0){ free(id); free(emb); continue; } /* dup or err */ - if (vindex_insert(ix, (uint64_t)inserted, emb) != 0){ free(id); free(emb); break; } + if (vn == vcap){ + size_t nc = vcap ? vcap*2 : 1024; + float* nv = (float*)realloc(vecs, nc*(size_t)dim*sizeof(float)); + if (!nv){ free(id); free(emb); failed = 1; goto out; } + vecs = nv; vcap = nc; + } + memcpy(vecs + vn*(size_t)dim, emb, (size_t)dim*sizeof(float)); free(emb); if (ids_n == ids_cap){ - size_t nc = ids_cap ? ids_cap*2 : 256; + size_t nc = ids_cap ? ids_cap*2 : 1024; char** ni = (char**)realloc(ids, nc*sizeof(char*)); - if (!ni){ free(id); break; } + if (!ni){ free(id); failed = 1; goto out; } ids = ni; ids_cap = nc; } ids[ids_n++] = id; /* transfers ownership */ - inserted++; + vn++; } } +out: close(fd); strset_free(&seen); - if (ids_out){ *ids_out = ids; if (n_out) *n_out = (int)ids_n; } - else { for (size_t i=0;idim, &vecs, &ids, &n); + if (h < 0) return -1; + int inserted = 0; + for (int i = 0; i < n; i++){ + if (vindex_insert(ix, (uint64_t)inserted, vecs + (size_t)i*ix->dim) != 0) break; + inserted++; + } + free(vecs); + if (ids_out){ + *ids_out = ids; if (n_out) *n_out = inserted; + /* free any ids beyond what we inserted (insert failure tail) */ + for (int i = inserted; i < n; i++) free(ids[i]); + } else { + for (int i = 0; i < n; i++) free(ids[i]); + free(ids); + if (n_out) *n_out = inserted; + } return inserted; } diff --git a/lang/runtime/engram_vindex.h b/lang/runtime/engram_vindex.h index 6303c84..911b191 100644 --- a/lang/runtime/engram_vindex.h +++ b/lang/runtime/engram_vindex.h @@ -73,6 +73,18 @@ void vindex_free(VIndex* idx); int vindex_build_from_store(VIndex* idx, const char* store_path, char*** ids_out, int* n_out); +/* Read-only harvest of the raw (un-normalised) emb vectors from a paged store, + * applying the SAME filtering vindex_build_from_store does (live records only, + * deduped by store id, emb present with emb_dim == `dim`), in insertion order. + * On success sets *vecs_out to a malloc'd float[n*dim] (row i == the i-th kept + * vector) and *n_out to n; if `ids_out` is non-NULL, sets it to a malloc'd array + * of n strdup'd store ids (ids_out[i] == the id of row i). Caller frees *vecs_out, + * each id string, and the id array. Returns n, or <0 on error. Used both by + * vindex_build_from_store (which then inserts each row) and by benchmarks/oracles + * that need the same vector set the index holds. */ +int vindex_harvest_from_store(const char* store_path, int dim, + float** vecs_out, char*** ids_out, int* n_out); + /* Optional persistence (index is rebuildable from the store; provided for * convenience). vindex_save writes a self-describing snapshot; vindex_load * reconstructs an index from one. Return 0 / non-NULL on success. */ diff --git a/lang/runtime/vindex_bench.c b/lang/runtime/vindex_bench.c new file mode 100644 index 0000000..ef0c482 --- /dev/null +++ b/lang/runtime/vindex_bench.c @@ -0,0 +1,238 @@ +/* vindex_bench.c — standalone proof harness for the engram HNSW ANN index. + * + * Measures brute-force cosine top-k (the correctness ORACLE) vs vindex_search + * (HNSW) on: (a) the REAL paged store harvested read-only, and (b) synthetic + * clustered data at several sizes to trace the scaling curve. Reports build time, + * per-query latency (brute vs HNSW), and recall@k (HNSW top-k vs brute top-k). + * + * Read-only: never opens a socket, never writes the store. Safe on an nsbx clone. + * + * Build: cc -O2 -std=c11 vindex_bench.c engram_vindex.c -lm -o vindex_bench + * Usage: vindex_bench store [nqueries] [k] [ef_csv] + * vindex_bench synth [dim] [clusters] [nqueries] [k] [ef_csv] + */ +#include "engram_vindex.h" +#include +#include +#include +#include +#include +#include + +/* ── deterministic PRNG (splitmix64) so runs are reproducible ─────────────── */ +static uint64_t g_seed = 0xD1B54A32D192ED03ULL; +static uint64_t sm(void){ + uint64_t z = (g_seed += 0x9E3779B97F4A7C15ULL); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + return z ^ (z >> 31); +} +static double urand(void){ return (double)((sm() >> 11) + 1) * (1.0/9007199254740993.0); } +static double grand(void){ /* Box-Muller */ + double u1 = urand(), u2 = urand(); + return sqrt(-2.0*log(u1)) * cos(2.0*M_PI*u2); +} + +static double now_s(void){ + struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec*1e-9; +} + +/* L2-normalise a row in place. */ +static void l2norm(float* v, int dim){ + double ss = 0; for (int i=0;i 0){ float inv = (float)(1.0/sqrt(ss)); for (int i=0;i= out_d[k-1]) continue; + int p = k-1; + while (p>0 && out_d[p-1] > d){ out_d[p]=out_d[p-1]; out_ids[p]=out_ids[p-1]; p--; } + out_d[p]=d; out_ids[p]=i; + } +} + +/* recall@k: |brute_topk ∩ hnsw_topk| / k. Both are id arrays of length k. */ +static double recall_at_k(const int* gt, const uint64_t* ann, int nann, int k){ + int hit = 0; + for (int i=0;i %.3f s (%.1f k nodes/s)\n", + bM?bM:VINDEX_DEFAULT_M, bEFC?bEFC:VINDEX_DEFAULT_EF_CONSTRUCTION, bt, n/1000.0/bt); + + /* choose query vectors: perturb random dataset rows (near-but-not-identical). */ + int* qidx = malloc((size_t)nq*sizeof(int)); + float* qv = malloc((size_t)nq*dim*sizeof(float)); + for (int i=0;i [nq] [k] [ef_csv] | synth [dim] [clusters] [nq] [k] [ef_csv] | sweep [nq] [k] [ef_csv]\n", argv[0]); return 2; } + int defef[8]; int ndef; + + if (strcmp(argv[1],"sweep")==0){ + if (argc < 4){ fprintf(stderr,"sweep needs \n"); return 2; } + int dim = atoi(argv[2]); + int Ns[16]; int nN = parse_csv(argv[3], Ns, 16); + int nq = (argc>4)?atoi(argv[4]):200; + int k = (argc>5)?atoi(argv[5]):10; + ndef = (argc>6)?parse_csv(argv[6],defef,8):parse_csv("64,128,200",defef,8); + for (int s=0;s \n"); return 2; } + const char* path = argv[2]; int dim = atoi(argv[3]); + int nq = (argc>4)?atoi(argv[4]):500; + int k = (argc>5)?atoi(argv[5]):10; + ndef = (argc>6)?parse_csv(argv[6],defef,8):parse_csv("32,64,128,200,400",defef,8); + printf("Harvesting emb vectors from %s (dim=%d) ...\n", path, dim); + float* data=NULL; int n=0; + double t0=now_s(); + int h = vindex_harvest_from_store(path, dim, &data, NULL, &n); + double harvest_s = now_s()-t0; + if (h < 0 || n == 0){ fprintf(stderr,"harvest failed (h=%d n=%d) — wrong dim or path?\n", h, n); return 1; } + printf("Harvested %d live embedded nodes in %.2f s\n", n, harvest_s); + for (int i=0;i n) nq = n; + run_bench("REAL STORE", data, n, dim, nq, k, defef, ndef, 0.0); + free(data); + return 0; + } + + if (strcmp(argv[1],"synth")==0){ + if (argc < 3){ fprintf(stderr,"synth needs \n"); return 2; } + int N = atoi(argv[2]); + int dim = (argc>3)?atoi(argv[3]):768; + int clusters = (argc>4)?atoi(argv[4]):200; + int nq = (argc>5)?atoi(argv[5]):500; + int k = (argc>6)?atoi(argv[6]):10; + ndef = (argc>7)?parse_csv(argv[7],defef,8):parse_csv("64,128,200",defef,8); + printf("Generating %d synthetic clustered vectors (dim=%d clusters=%d) ...\n", N, dim, clusters); + float* data = malloc((size_t)N*dim*sizeof(float)); + if (!data){ fprintf(stderr,"OOM allocating %zu bytes\n", (size_t)N*dim*sizeof(float)); return 1; } + gen_synth(data, N, dim, clusters, 0.35); + char lbl[64]; snprintf(lbl,sizeof lbl,"SYNTH"); + run_bench(lbl, data, N, dim, nq, k, defef, ndef, 0.0); + free(data); + return 0; + } + + fprintf(stderr,"unknown mode '%s'\n", argv[1]); + return 2; +}