neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests 4 Actions Packages Projects Releases 5 Wiki Activity

copy engram_vindex ANN module onto tiered trunk (staged->committed; not yet wired)

Browse Source
This commit is contained in:
Will Anderson
2026-08-12 16:59:30 -05:00
parent 89589864cd
commit 0d299ee0f1
4 changed files with 1068 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/test/run_vindex_tests.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
# Build + RUN the M8 HNSW vector-index tests. Pure C11 (gcc/cc), stdlib + libm
# only. This is a standalone C module — NOT folded through elb/elc.
#
# Two passes:
# 1. PERF — optimised (-O2, no sanitizer): the real recall@10 gate + speedup
# numbers at full size (N=5000 recall, N=5000/20000 speedup).
# 2. SAFETY — ASan + UBSan on the same suite at reduced size (VINDEX_QUICK=1);
# memory-safety is size-independent, so this stays fast.
set -e
HERE=$(cd "$(dirname "$0")" && pwd)
RT="$HERE/../../lang/runtime"
CC=${CC:-cc}
SRC="$HERE/test_vindex.c $RT/engram_vindex.c $RT/engram_store.c"
WARN="-std=c11 -Wall -Wextra"
TMP=$(mktemp -d)
echo "### PASS 1: PERF (optimised, un-sanitised) — recall gate + speedup"
$CC $WARN -O2 -I"$RT" $SRC -lm -o "$TMP/perf"
"$TMP/perf"
echo
echo "### PASS 2: SAFETY (ASan/UBSan, reduced size)"
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe"
VINDEX_QUICK=1 ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
engram/test/test_vindex.c
+312
View File
@@ -0,0 +1,312 @@
/* test_vindex.c — build + RUN gate for the M8 HNSW vector index.
*
* Covers: recall@10 vs brute-force oracle, brute-force-vs-index speedup,
* correctness edge cases (k>N, identical vectors, self-query, zero vector),
* determinism (seeded PRNG → identical graphs), and vindex_build_from_store
* over a real engram_store on-disk file.
*
* Pure C11; links engram_vindex.c + engram_store.c; -lm. ASan/UBSan clean.
*/
#include "engram_vindex.h"
#include "engram_store.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>
#include <time.h>
#include <unistd.h>
#define DIM 768
static int g_fail = 0;
/* VINDEX_QUICK=1 shrinks the two large builds so the ASan/UBSan pass (which runs
* ~5-10x slower) stays fast — memory-safety is size-independent. The perf numbers
* (recall gate + speedup) come from the un-sanitized, full-size pass. */
static int g_quick = 0;
static int envint(const char* k, int dflt){ const char* s=getenv(k); return s?atoi(s):dflt; }
#define CHECK(cond, msg) do{ if(!(cond)){ printf(" FAIL: %s\n", msg); g_fail=1; } else { printf(" ok: %s\n", msg); } }while(0)
/* deterministic test PRNG (splitmix64) */
static uint64_t rng_state = 0xABCDEF0123456789ULL;
static uint64_t xrng(uint64_t* s){
uint64_t z=(*s+=0x9E3779B97F4A7C15ULL);
z=(z^(z>>30))*0xBF58476D1CE4E5B9ULL; z=(z^(z>>27))*0x94D049BB133111EBULL;
return z^(z>>31);
}
static float frand(uint64_t* s){ return (float)((xrng(s)>>11)*(1.0/9007199254740992.0)) - 0.5f; }
static double now_s(void){
struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t);
return t.tv_sec + t.tv_nsec*1e-9;
}
/* fill vec[N*DIM]: mostly random, some clustered groups (center + small noise). */
static void gen_vectors(float* v, int N, uint64_t seed){
uint64_t s = seed;
int clustered = N/5; /* last fifth is clustered */
int ncenters = 20;
float* centers = (float*)malloc((size_t)ncenters*DIM*sizeof(float));
for (int c=0;c<ncenters;c++) for(int d=0;d<DIM;d++) centers[c*DIM+d]=frand(&s);
for (int i=0;i<N;i++){
if (i < N-clustered){
for (int d=0;d<DIM;d++) v[i*DIM+d]=frand(&s);
} else {
int c = (int)(xrng(&s)%ncenters);
for (int d=0;d<DIM;d++) v[i*DIM+d]=centers[c*DIM+d] + 0.05f*frand(&s);
}
}
free(centers);
}
static float cosdist(const float* a, const float* b){
double da=0,db=0,dot=0;
for(int i=0;i<DIM;i++){ da+=(double)a[i]*a[i]; db+=(double)b[i]*b[i]; dot+=(double)a[i]*b[i]; }
if (da<=0||db<=0) return 1.0f;
return (float)(1.0 - dot/(sqrt(da)*sqrt(db)));
}
/* brute-force top-k node ids into ids[k] (ascending distance). */
static void brute_topk(const float* v, int N, const float* q, int k, int* ids){
float* bd = (float*)malloc((size_t)k*sizeof(float));
for (int i=0;i<k;i++){ ids[i]=-1; bd[i]=1e30f; }
for (int i=0;i<N;i++){
float d = cosdist(q, v+(size_t)i*DIM);
if (d < bd[k-1]){
int p=k-1;
while (p>0 && bd[p-1]>d){ bd[p]=bd[p-1]; ids[p]=ids[p-1]; p--; }
bd[p]=d; ids[p]=i;
}
}
free(bd);
}
/* ── Test 1: recall@10 vs brute force + latency/recall tradeoff ────────────── */
static void test_recall(void){
int N=envint("VINDEX_N_RECALL", g_quick?1500:5000), Q=200, K=10;
printf("\n== Test 1: recall@10 vs brute force (N=%d, DIM=768) ==\n", N);
float* v = (float*)malloc((size_t)N*DIM*sizeof(float));
gen_vectors(v, N, 111);
double t0=now_s();
VIndex* ix = vindex_create(DIM, VINDEX_DEFAULT_M, VINDEX_DEFAULT_EF_CONSTRUCTION);
for (int i=0;i<N;i++) vindex_insert(ix, (uint64_t)i, v+(size_t)i*DIM);
double build_s = now_s()-t0;
printf(" build: %d vectors in %.2fs (M=%d, ef_construction=%d)\n",
N, build_s, VINDEX_DEFAULT_M, VINDEX_DEFAULT_EF_CONSTRUCTION);
/* queries: half random, half near a real vector (perturbed). */
float* qs = (float*)malloc((size_t)Q*DIM*sizeof(float));
uint64_t s=999;
for (int i=0;i<Q;i++){
if (i<Q/2) for(int d=0;d<DIM;d++) qs[i*DIM+d]=frand(&s);
else { int base=(int)(xrng(&s)%N); for(int d=0;d<DIM;d++) qs[i*DIM+d]=v[base*DIM+d]+0.03f*frand(&s); }
}
/* oracle */
int* oracle = (int*)malloc((size_t)Q*K*sizeof(int));
for (int i=0;i<Q;i++) brute_topk(v, N, qs+(size_t)i*DIM, K, oracle+(size_t)i*K);
int efs[] = { 10, 32, 64, 128 };
for (int e=0;e<4;e++){
int ef=efs[e];
uint64_t ids[64]; float dd[64];
int hits=0;
double qt0=now_s();
for (int i=0;i<Q;i++){
int n=vindex_search(ix, qs+(size_t)i*DIM, K, ef, ids, dd);
for (int a=0;a<n;a++) for(int b=0;b<K;b++) if((int)ids[a]==oracle[i*K+b]){ hits++; break; }
}
double qs_ms = (now_s()-qt0)*1000.0/Q;
double recall = (double)hits/(Q*K);
printf(" ef_search=%-4d recall@10=%.4f latency=%.3f ms/query\n", ef, recall, qs_ms);
if (ef==VINDEX_DEFAULT_EF_SEARCH && !g_quick)
CHECK(recall >= 0.90, "recall@10 >= 0.90 at default ef_search=128");
}
free(oracle); free(qs); free(v); vindex_free(ix);
}
/* ── Test 2: speedup vs brute force ───────────────────────────────────────── */
static void speedup_at(int N){
int Q=100, K=10;
float* v=(float*)malloc((size_t)N*DIM*sizeof(float));
gen_vectors(v,N,222);
VIndex* ix=vindex_create(DIM,16,200);
double bt0=now_s();
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)i,v+(size_t)i*DIM);
printf(" N=%d build=%.2fs\n", N, now_s()-bt0);
float* qs=(float*)malloc((size_t)Q*DIM*sizeof(float));
uint64_t s=333; for(int i=0;i<Q*DIM;i++) qs[i]=frand(&s);
/* brute force */
int scratch[16];
double b0=now_s();
for(int i=0;i<Q;i++) brute_topk(v,N,qs+(size_t)i*DIM,K,scratch);
double bf=(now_s()-b0)/Q;
/* index */
uint64_t ids[16]; float dd[16];
double i0=now_s();
for(int i=0;i<Q;i++) vindex_search(ix,qs+(size_t)i*DIM,K,64,ids,dd);
double iq=(now_s()-i0)/Q;
printf(" N=%d brute=%.4f ms/q index=%.4f ms/q speedup=%.1fx\n",
N, bf*1000, iq*1000, bf/iq);
CHECK(iq < bf, "index query faster than brute force");
free(qs); free(v); vindex_free(ix);
}
static void test_speedup(void){
printf("\n== Test 2: brute-force vs index speedup ==\n");
speedup_at(g_quick?2000:5000);
speedup_at(envint("VINDEX_N_BIG", g_quick?3000:20000));
}
/* ── Test 3: edge cases ───────────────────────────────────────────────────── */
static void test_edges(void){
printf("\n== Test 3: correctness edge cases ==\n");
/* k larger than node count */
{
VIndex* ix=vindex_create(DIM,16,200);
float vec[DIM]; uint64_t s=1;
for(int i=0;i<3;i++){ for(int d=0;d<DIM;d++) vec[d]=frand(&s); vindex_insert(ix,(uint64_t)i,vec); }
uint64_t ids[50]; float dd[50];
int n=vindex_search(ix, vec, 50, 64, ids, dd);
CHECK(n==3, "k > node count returns exactly node-count results");
vindex_free(ix);
}
/* duplicate / identical vectors */
{
VIndex* ix=vindex_create(DIM,16,200);
float a[DIM]; uint64_t s=2; for(int d=0;d<DIM;d++) a[d]=frand(&s);
for(int i=0;i<10;i++) vindex_insert(ix,(uint64_t)i,a); /* all identical */
float b[DIM]; for(int d=0;d<DIM;d++) b[d]=frand(&s);
vindex_insert(ix,100,b);
uint64_t ids[5]; float dd[5];
int n=vindex_search(ix,a,5,64,ids,dd);
CHECK(n==5, "identical-vector index returns k results");
CHECK(dd[0] < 1e-4f, "top-1 distance ~0 for a duplicated vector");
vindex_free(ix);
}
/* query equal to an indexed vector returns itself as top-1, dist ~0 */
{
VIndex* ix=vindex_create(DIM,16,200);
int N=500; float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,7);
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)(1000+i),v+(size_t)i*DIM);
int probe=137;
uint64_t ids[3]; float dd[3];
int n=vindex_search(ix, v+(size_t)probe*DIM, 3, 64, ids, dd);
CHECK(n>=1 && ids[0]==(uint64_t)(1000+probe), "self-query returns itself as top-1");
CHECK(dd[0] < 1e-4f, "self-query top-1 distance ~0");
free(v); vindex_free(ix);
}
/* zero vector: no NaN, handled */
{
VIndex* ix=vindex_create(DIM,16,200);
float z[DIM]; memset(z,0,sizeof z);
float a[DIM]; uint64_t s=3; for(int d=0;d<DIM;d++) a[d]=frand(&s);
vindex_insert(ix,0,z); vindex_insert(ix,1,a);
uint64_t ids[2]; float dd[2];
int n=vindex_search(ix, z, 2, 64, ids, dd); /* zero query */
int nan=0; for(int i=0;i<n;i++) if(isnan(dd[i])||isinf(dd[i])) nan=1;
CHECK(n>=1 && !nan, "zero vector query produces no NaN/Inf");
n=vindex_search(ix, a, 2, 64, ids, dd); /* zero indexed */
nan=0; for(int i=0;i<n;i++) if(isnan(dd[i])||isinf(dd[i])) nan=1;
CHECK(!nan, "indexed zero vector produces no NaN/Inf");
vindex_free(ix);
}
}
/* ── Test 4: determinism ──────────────────────────────────────────────────── */
static void test_determinism(void){
printf("\n== Test 4: determinism (seeded PRNG → identical results) ==\n");
int N=1500;
float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,55);
uint64_t ids1[10],ids2[10]; float d1[10],d2[10];
int identical=1;
for (int build=0; build<2; build++){
VIndex* ix=vindex_create(DIM,16,200);
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)i,v+(size_t)i*DIM);
/* probe several queries */
for (int q=0;q<20;q++){
uint64_t* ida = build? ids2 : ids1; float* da = build? d2 : d1;
vindex_search(ix, v+(size_t)(q*37%N)*DIM, 10, 64, ida, da);
if (build==1){
/* re-run build-0 query stored? simpler: compare within-run below */
}
}
vindex_free(ix);
}
/* Proper comparison: run two fresh builds, same single query. */
identical=1;
for (int q=0;q<25;q++){
int qi=(q*61)%N;
VIndex* a=vindex_create(DIM,16,200); for(int i=0;i<N;i++) vindex_insert(a,(uint64_t)i,v+(size_t)i*DIM);
VIndex* b=vindex_create(DIM,16,200); for(int i=0;i<N;i++) vindex_insert(b,(uint64_t)i,v+(size_t)i*DIM);
int na=vindex_search(a, v+(size_t)qi*DIM,10,64,ids1,d1);
int nb=vindex_search(b, v+(size_t)qi*DIM,10,64,ids2,d2);
if (na!=nb) identical=0;
for(int i=0;i<na;i++) if(ids1[i]!=ids2[i] || d1[i]!=d2[i]) identical=0;
vindex_free(a); vindex_free(b);
}
CHECK(identical, "two independent builds give byte-identical query results");
free(v);
}
/* ── Test 5: build_from_store ─────────────────────────────────────────────── */
static void test_build_from_store(void){
printf("\n== Test 5: vindex_build_from_store over a real engram_store ==\n");
char path[256];
snprintf(path,sizeof path,"/tmp/vindex_test_store_%d.engram",(int)getpid());
unlink(path);
EngramPagedStore* st = store_create(path);
if (!st){ printf(" FAIL: store_create\n"); g_fail=1; return; }
int N=300;
float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,88);
for (int i=0;i<N;i++){
StoreNode n; memset(&n,0,sizeof n);
char id[32]; snprintf(id,sizeof id,"node-%d",i);
n.id=id; n.content="x"; n.node_type="concept"; n.tier="Semantic";
n.emb = v+(size_t)i*DIM; n.emb_dim=DIM;
if (store_put_node(st,&n)!=0){ printf(" FAIL: put_node %d\n",i); g_fail=1; }
}
/* a node WITHOUT an emb — must be skipped by build_from_store. */
{ StoreNode n; memset(&n,0,sizeof n); n.id=(char*)"no-emb"; n.content="y"; n.node_type="concept"; n.tier="Semantic";
store_put_node(st,&n); }
store_close(st);
VIndex* ix = vindex_create(DIM,16,200);
char** ids=NULL; int nids=0;
int ins = vindex_build_from_store(ix, path, &ids, &nids);
printf(" build_from_store inserted %d vectors (expected %d; 1 emb-less skipped)\n", ins, N);
CHECK(ins==N, "build_from_store inserts exactly the emb'd nodes");
CHECK((size_t)ins==vindex_size(ix), "index size matches insert count");
/* query with a known vector → must return its own node id as top-1. */
int probe=42;
uint64_t rids[5]; float dd[5];
int n=vindex_search(ix, v+(size_t)probe*DIM, 5, 64, rids, dd);
int correct = (n>=1 && rids[0]<(uint64_t)nids && strcmp(ids[rids[0]], "node-42")==0);
printf(" query for node-42's vector → top-1 id=%s dist=%.5f\n",
(n>=1 && rids[0]<(uint64_t)nids)? ids[rids[0]] : "?", n?dd[0]:-1);
CHECK(correct, "build_from_store query resolves to the right node id");
CHECK(n>=1 && dd[0]<1e-4f, "top-1 distance ~0 for exact stored vector");
for (int i=0;i<nids;i++) free(ids[i]);
free(ids); free(v); vindex_free(ix); unlink(path);
}
int main(void){
(void)rng_state;
g_quick = envint("VINDEX_QUICK", 0);
printf("=== engram_vindex (HNSW) test suite ===%s\n", g_quick?" [QUICK]":"");
test_recall();
test_speedup();
test_edges();
test_determinism();
test_build_from_store();
printf("\n=== %s ===\n", g_fail? "FAILURES PRESENT" : "ALL TESTS PASSED");
return g_fail;
}
lang/runtime/engram_vindex.c
+649
View File
@@ -0,0 +1,649 @@
/* engram_vindex.c — HNSW ANN index over f32 embedding vectors (design §9 M8).
*
* Self-contained: plain C11, stdlib + libm (-lm for sqrtf/logf) only. No
* dependency on el_runtime; the store is read via its PERMANENT on-disk format
* (design §2.4), decoded read-only here so engram_store.{c,h} stay untouched.
*
* Algorithm: Malkov & Yashunin, "Efficient and robust approximate nearest
* neighbor search using Hierarchical Navigable Small World graphs" (2016).
* - multi-layer graph; level ~ Exp(1/ln M), assigned by a per-node seeded PRNG
* (deterministic: seed = FIXED_SEED ^ node_ordinal) so a rebuild is bit-for-
* bit reproducible regardless of wall-clock or global rand() state.
* - greedy descent through upper layers to an entry point, then an ef-bounded
* best-first search at each layer (Algorithm 2).
* - neighbour selection by the diversity heuristic (Algorithm 4), not plain
* k-nearest, with keep-pruned backfill for connectivity.
* - bidirectional links; a neighbour whose degree exceeds M (2M on layer 0) is
* re-pruned with the same heuristic.
*
* Metric: vectors are L2-normalised on entry, so cosine similarity == dot
* product; distance = 1 - dot (in [0,2], smaller == nearer). Deterministic tie-
* breaks are by element index so results are stable across identical builds.
*/
#include "engram_vindex.h"
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
/* Deterministic PRNG seed base (fixed constant — never wall-clock/rand). */
#define VINDEX_FIXED_SEED 0x9E3779B97F4A7C15ULL
/* ── deterministic PRNG (splitmix64) ──────────────────────────────────────── */
static inline uint64_t splitmix64(uint64_t* s){
uint64_t z = (*s += 0x9E3779B97F4A7C15ULL);
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
return z ^ (z >> 31);
}
/* Uniform double in (0,1]. */
static inline double sm_uniform(uint64_t* s){
/* 53-bit mantissa; +1 keeps it in (0,1] so log() never sees 0. */
return ((double)((splitmix64(s) >> 11) + 1)) * (1.0 / 9007199254740993.0);
}
/* ── element + index structures ───────────────────────────────────────────── */
typedef struct {
int count;
int cap;
int* ids; /* neighbour element indices */
} NeighList;
typedef struct {
uint64_t node_id;
int level; /* top layer this element appears on (>=0) */
float* vec; /* dim floats, L2-normalised */
NeighList* links; /* level+1 lists; links[l] = neighbours at layer l */
} Elem;
struct VIndex {
int dim;
int M; /* max neighbours per node, upper layers */
int M0; /* == 2*M, layer 0 */
int ef_construction;
double mL; /* level normaliser = 1/ln(M) */
Elem* elems;
size_t n;
size_t cap;
int entry; /* entry-point element index, -1 if empty */
int max_level; /* current top layer */
/* scratch: version-stamped visited set (O(1) reset). */
uint32_t* visited;
uint32_t visit_epoch;
size_t visited_cap;
};
/* ── small helpers ────────────────────────────────────────────────────────── */
static float* vec_normalise_copy(const float* v, int dim){
float* out = (float*)malloc((size_t)dim * sizeof(float));
if (!out) return NULL;
double ss = 0.0;
for (int i=0;i<dim;i++) ss += (double)v[i]*(double)v[i];
if (ss > 0.0){
float inv = (float)(1.0 / sqrt(ss));
for (int i=0;i<dim;i++) out[i] = v[i]*inv;
} else {
for (int i=0;i<dim;i++) out[i] = 0.0f; /* zero vector stays zero */
}
return out;
}
/* Cosine distance between two normalised vectors: 1 - dot. In [0,2].
* Float accumulation in 4 lanes so the compiler auto-vectorises the hot path
* (this is the dominant cost of both build and search). */
static float vdist(const VIndex* ix, const float* a, const float* b){
int dim = ix->dim;
float s0=0,s1=0,s2=0,s3=0;
int i=0;
for (; i+4<=dim; i+=4){
s0 += a[i]*b[i]; s1 += a[i+1]*b[i+1];
s2 += a[i+2]*b[i+2]; s3 += a[i+3]*b[i+3];
}
float dot = (s0+s1)+(s2+s3);
for (; i<dim; i++) dot += a[i]*b[i];
return 1.0f - dot;
}
static int nl_push(NeighList* nl, int id){
if (nl->count == nl->cap){
int nc = nl->cap ? nl->cap*2 : 4;
int* np = (int*)realloc(nl->ids, (size_t)nc*sizeof(int));
if (!np) return -1;
nl->ids = np; nl->cap = nc;
}
nl->ids[nl->count++] = id;
return 0;
}
/* ── binary heaps over (dist,elem) pairs ──────────────────────────────────── */
typedef struct { float d; int e; } Pair;
typedef struct { Pair* a; int n, cap; } Heap;
static int heap_reserve(Heap* h, int need){
if (need <= h->cap) return 0;
int nc = h->cap ? h->cap*2 : 16;
while (nc < need) nc *= 2;
Pair* na = (Pair*)realloc(h->a, (size_t)nc*sizeof(Pair));
if (!na) return -1;
h->a = na; h->cap = nc; return 0;
}
/* Order predicate: for a MAX-heap on distance, "higher priority" = larger dist;
* ties broken by larger element index (deterministic + stable). is_max selects. */
static inline int pair_before(Pair x, Pair y, int is_max){
if (x.d != y.d) return is_max ? (x.d > y.d) : (x.d < y.d);
return is_max ? (x.e > y.e) : (x.e < y.e);
}
static int heap_push(Heap* h, Pair v, int is_max){
if (heap_reserve(h, h->n+1)) return -1;
int i = h->n++;
h->a[i] = v;
while (i > 0){
int p = (i-1)/2;
if (pair_before(h->a[i], h->a[p], is_max)){
Pair t=h->a[i]; h->a[i]=h->a[p]; h->a[p]=t; i=p;
} else break;
}
return 0;
}
static Pair heap_pop(Heap* h, int is_max){
Pair top = h->a[0];
h->a[0] = h->a[--h->n];
int i = 0;
for (;;){
int l=2*i+1, r=2*i+2, best=i;
if (l<h->n && pair_before(h->a[l], h->a[best], is_max)) best=l;
if (r<h->n && pair_before(h->a[r], h->a[best], is_max)) best=r;
if (best==i) break;
Pair t=h->a[i]; h->a[i]=h->a[best]; h->a[best]=t; i=best;
}
return top;
}
/* ── visited set ──────────────────────────────────────────────────────────── */
static int visited_ensure(VIndex* ix){
if (ix->visited_cap >= ix->cap && ix->visited) return 0;
size_t nc = ix->cap ? ix->cap : 16;
uint32_t* nv = (uint32_t*)realloc(ix->visited, nc*sizeof(uint32_t));
if (!nv) return -1;
if (nc > ix->visited_cap) memset(nv + ix->visited_cap, 0, (nc-ix->visited_cap)*sizeof(uint32_t));
ix->visited = nv; ix->visited_cap = nc;
return 0;
}
static inline void visited_reset(VIndex* ix){
if (++ix->visit_epoch == 0){ /* wrapped: clear all */
memset(ix->visited, 0, ix->visited_cap*sizeof(uint32_t));
ix->visit_epoch = 1;
}
}
static inline int is_visited(VIndex* ix, int e){ return ix->visited[e]==ix->visit_epoch; }
static inline void mark_visited(VIndex* ix, int e){ ix->visited[e]=ix->visit_epoch; }
/* ── search one layer (Algorithm 2): best-first, ef-bounded ───────────────── */
/* Returns results as an unsorted Heap (max-heap on distance, size<=ef). Caller
* owns res->a. `q` is a normalised query. */
static int search_layer(VIndex* ix, const float* q, const int* eps, int neps,
int ef, int layer, Heap* res /*out, max-heap*/){
Heap cand = {0,0,0}; /* min-heap: nearest to expand */
res->a=NULL; res->n=0; res->cap=0;
visited_reset(ix);
for (int i=0;i<neps;i++){
int e = eps[i];
if (is_visited(ix,e)) continue;
mark_visited(ix,e);
float d = vdist(ix, q, ix->elems[e].vec);
Pair p = { d, e };
if (heap_push(&cand,p,0) || heap_push(res,p,1)){ free(cand.a); return -1; }
}
while (res->n > ef) heap_pop(res,1); /* trim to ef */
while (cand.n > 0){
Pair c = heap_pop(&cand,0);
float worst = res->a[0].d; /* farthest kept result */
if (res->n >= ef && c.d > worst) break;
Elem* ce = &ix->elems[c.e];
if (layer <= ce->level){
NeighList* nl = &ce->links[layer];
for (int i=0;i<nl->count;i++){
int e = nl->ids[i];
if (is_visited(ix,e)) continue;
mark_visited(ix,e);
float d = vdist(ix, q, ix->elems[e].vec);
if (res->n < ef || d < res->a[0].d){
Pair p = { d, e };
if (heap_push(&cand,p,0) || heap_push(res,p,1)){ free(cand.a); return -1; }
if (res->n > ef) heap_pop(res,1);
}
}
}
}
free(cand.a);
return 0;
}
/* ── neighbour selection heuristic (Algorithm 4) ──────────────────────────── */
/* From candidate pairs W (any order), pick up to M diverse neighbours of q.
* Keep c only if it is nearer to q than to every already-chosen neighbour;
* backfill from the pruned set (nearest first) to reach M for connectivity.
* Writes chosen element indices into out[], returns the count. */
static int select_neighbors(VIndex* ix, const float* q, Pair* W, int nW, int M, int* out){
(void)q; /* q's distances are precomputed in W[].d; kept for call-site clarity */
/* sort W ascending by (dist,elem) — deterministic. */
for (int i=1;i<nW;i++){ /* insertion sort (nW small) */
Pair key=W[i]; int j=i-1;
while (j>=0 && !pair_before(W[j],key,0)){ W[j+1]=W[j]; j--; }
W[j+1]=key;
}
int nout = 0;
Pair* pruned = (Pair*)malloc((size_t)(nW?nW:1)*sizeof(Pair));
int npr = 0;
if (!pruned) return -1;
for (int i=0;i<nW && nout<M;i++){
int good = 1;
for (int j=0;j<nout;j++){
float d = vdist(ix, ix->elems[W[i].e].vec, ix->elems[out[j]].vec);
if (d < W[i].d){ good = 0; break; } /* nearer an existing pick → drop */
}
if (good) out[nout++] = W[i].e;
else pruned[npr++] = W[i];
}
for (int i=0;i<npr && nout<M;i++) out[nout++] = pruned[i].e; /* keep-pruned backfill */
free(pruned);
return nout;
}
/* Re-prune a neighbour's over-full adjacency list back to `Mmax`. */
static void prune_links(VIndex* ix, int e, int layer, int Mmax){
NeighList* nl = &ix->elems[e].links[layer];
if (nl->count <= Mmax) return;
const float* base = ix->elems[e].vec;
Pair* W = (Pair*)malloc((size_t)nl->count*sizeof(Pair));
if (!W) return;
int nW = nl->count;
for (int i=0;i<nW;i++) W[i] = (Pair){ vdist(ix, base, ix->elems[nl->ids[i]].vec), nl->ids[i] };
int* keep = (int*)malloc((size_t)nW*sizeof(int));
if (!keep){ free(W); return; }
int nk = select_neighbors(ix, base, W, nW, Mmax, keep);
if (nk >= 0){ nl->count = nk; for (int i=0;i<nk;i++) nl->ids[i]=keep[i]; }
free(keep); free(W);
}
/* ── insert ───────────────────────────────────────────────────────────────── */
static int elems_reserve(VIndex* ix){
if (ix->n < ix->cap) return 0;
size_t nc = ix->cap ? ix->cap*2 : 64;
Elem* ne = (Elem*)realloc(ix->elems, nc*sizeof(Elem));
if (!ne) return -1;
ix->elems = ne; ix->cap = nc;
return visited_ensure(ix);
}
int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
if (!ix || !vec) return -1;
if (elems_reserve(ix)) return -1;
int cur = (int)ix->n;
/* deterministic level assignment, seeded per-node. */
uint64_t seed = VINDEX_FIXED_SEED ^ (node_id + 0x2545F4914F6CDD1DULL*(uint64_t)cur);
int level = (int)(-log(sm_uniform(&seed)) * ix->mL);
if (level < 0) level = 0;
Elem* el = &ix->elems[cur];
el->node_id = node_id;
el->level = level;
el->vec = vec_normalise_copy(vec, ix->dim);
el->links = (NeighList*)calloc((size_t)level+1, sizeof(NeighList));
if (!el->vec || !el->links){ free(el->vec); free(el->links); return -1; }
ix->n++;
if (ix->entry < 0){ /* first element */
ix->entry = cur; ix->max_level = level;
return 0;
}
int ep = ix->entry;
int L = ix->max_level;
/* greedy descent through layers above `level` to refine the entry point. */
for (int lc = L; lc > level; lc--){
Heap r = {0,0,0};
int eps1[1] = { ep };
if (search_layer(ix, el->vec, eps1, 1, 1, lc, &r)){ return -1; }
if (r.n){ ep = r.a[0].e; float bd=r.a[0].d;
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; ep=r.a[i].e;} }
free(r.a);
}
/* from min(L,level) down to 0: connect. Each layer's ef-results seed the next
* layer's entry set; `eps` is heap-owned below the top and freed each step. */
int start = (L < level) ? L : level;
int eps_stack[1] = { ep };
int* eps = eps_stack; /* not owned (stack) until reassigned to malloc'd */
int* eps_owned = NULL;
int neps = 1;
int rc = 0;
for (int lc = start; lc >= 0; lc--){
int Mmax = (lc==0) ? ix->M0 : ix->M;
Heap W = {0,0,0};
if (search_layer(ix, el->vec, eps, neps, ix->ef_construction, lc, &W)){ rc=-1; break; }
int* chosen = (int*)malloc((size_t)(W.n?W.n:1)*sizeof(int));
if (!chosen){ free(W.a); rc=-1; break; }
int nc = select_neighbors(ix, el->vec, W.a, W.n, Mmax, chosen);
if (nc < 0){ free(chosen); free(W.a); rc=-1; break; }
/* link cur <-> chosen (bidirectional), prune neighbours if over-full. */
for (int i=0;i<nc;i++){
int nb = chosen[i];
if (nl_push(&el->links[lc], nb) || nl_push(&ix->elems[nb].links[lc], cur)){
free(chosen); free(W.a); rc=-1; goto done;
}
prune_links(ix, nb, lc, Mmax);
}
free(chosen);
/* next layer's entry points = this layer's ef results. */
if (lc > 0){
int* neweps = (int*)malloc((size_t)(W.n?W.n:1)*sizeof(int));
if (!neweps){ free(W.a); rc=-1; break; }
for (int i=0;i<W.n;i++) neweps[i]=W.a[i].e;
neps = W.n ? W.n : 1;
if (!W.n) neweps[0] = eps[0]; /* fall back to prior ep if empty */
free(eps_owned);
eps = eps_owned = neweps;
}
free(W.a);
}
done:
free(eps_owned);
if (rc) return -1;
if (level > ix->max_level){ ix->max_level = level; ix->entry = cur; }
return 0;
}
/* ── search ───────────────────────────────────────────────────────────────── */
int vindex_search(VIndex* ix, const float* query, int k, int ef_search,
uint64_t* node_id_out, float* dist_out){
if (!ix || !query || k <= 0) return -1;
if (ix->entry < 0) return 0;
if (ef_search <= 0) ef_search = VINDEX_DEFAULT_EF_SEARCH;
if (ef_search < k) ef_search = k;
float* q = vec_normalise_copy(query, ix->dim);
if (!q) return -1;
int ep = ix->entry;
for (int lc = ix->max_level; lc > 0; lc--){
Heap r = {0,0,0};
int eps[1] = { ep };
if (search_layer(ix, q, eps, 1, 1, lc, &r)){ free(q); return -1; }
if (r.n){ int b=r.a[0].e; float bd=r.a[0].d;
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; b=r.a[i].e;}
ep = b; }
free(r.a);
}
Heap res = {0,0,0};
int eps[1] = { ep };
if (search_layer(ix, q, eps, 1, ef_search, 0, &res)){ free(res.a); free(q); return -1; }
free(q);
/* res is a max-heap of size<=ef; pop into ascending order, keep nearest k. */
int total = res.n;
Pair* sorted = (Pair*)malloc((size_t)(total?total:1)*sizeof(Pair));
if (!sorted){ free(res.a); return -1; }
for (int i=total-1;i>=0;i--) sorted[i] = heap_pop(&res,1); /* farthest first out → fill from end */
free(res.a);
int out_n = (k < total) ? k : total;
for (int i=0;i<out_n;i++){
if (node_id_out) node_id_out[i] = ix->elems[sorted[i].e].node_id;
if (dist_out) dist_out[i] = sorted[i].d;
}
free(sorted);
return out_n;
}
size_t vindex_size(const VIndex* ix){ return ix ? ix->n : 0; }
VIndex* vindex_create(int dim, int M, int ef_construction){
if (dim <= 0) return NULL;
if (M <= 0) M = VINDEX_DEFAULT_M;
if (ef_construction <= 0) ef_construction = VINDEX_DEFAULT_EF_CONSTRUCTION;
VIndex* ix = (VIndex*)calloc(1, sizeof(VIndex));
if (!ix) return NULL;
ix->dim = dim;
ix->M = M;
ix->M0 = 2*M;
ix->ef_construction = ef_construction;
ix->mL = 1.0 / log((double)M > 1.0 ? (double)M : 2.0);
ix->entry = -1;
ix->max_level = 0;
ix->visit_epoch = 0;
return ix;
}
void vindex_free(VIndex* ix){
if (!ix) return;
for (size_t i=0;i<ix->n;i++){
Elem* e = &ix->elems[i];
if (e->links) for (int l=0;l<=e->level;l++) free(e->links[l].ids);
free(e->links);
free(e->vec);
}
free(ix->elems);
free(ix->visited);
free(ix);
}
/* ── read-only decode of the paged store node format (design §2.4) ─────────── */
/* Mirrors engram_store.c constants; the on-disk format is PERMANENT so these are
* safe to duplicate for a read-only harvest of emb vectors. */
#define VS_PAGE_SIZE 16384u
#define VS_HDR 32u
#define VS_SLOT_SIZE 6u
#define VS_SLOT_LIVE 1u
#define VS_REC_HDR 4u
#define VS_REC_OVERFLOW 1u
#define VS_PT_NODE 1u
#define VS_OVF_NEXT 32u
#define VS_OVF_LEN 40u
#define VS_OVF_DATA 44u
#define VS_NT_ID 1u
#define VS_NT_EMB 24u
#define VS_NT_EMB_DIM 25u
static uint16_t vg_u16(const uint8_t* p){ return (uint16_t)(p[0] | (p[1]<<8)); }
static uint32_t vg_u32(const uint8_t* p){ uint32_t v=0; for(int i=0;i<4;i++) v|=(uint32_t)p[i]<<(8*i); return v; }
static uint64_t vg_u64(const uint8_t* p){ uint64_t v=0; for(int i=0;i<8;i++) v|=(uint64_t)p[i]<<(8*i); return v; }
static int vs_pread(int fd, uint64_t page, uint8_t* buf){
off_t off = (off_t)page * VS_PAGE_SIZE;
ssize_t r = pread(fd, buf, VS_PAGE_SIZE, off);
return (r == (ssize_t)VS_PAGE_SIZE) ? 0 : -1;
}
/* Read a (possibly overflowed) record body; caller frees *out. */
static int vs_read_body(int fd, const uint8_t* page, uint16_t off, uint16_t len,
uint8_t** out, size_t* outlen){
if (len < VS_REC_HDR) return -1;
uint8_t flags = page[off+3];
if (flags & VS_REC_OVERFLOW){
uint64_t head = vg_u64(page + off + VS_REC_HDR);
uint64_t total = vg_u64(page + off + VS_REC_HDR + 8);
uint8_t* body = (uint8_t*)malloc(total ? total : 1);
if (!body) return -1;
size_t got=0; uint64_t id=head;
uint8_t ov[VS_PAGE_SIZE];
while (id){
if (vs_pread(fd, id, ov)){ free(body); return -1; }
uint32_t chunk = vg_u32(ov + VS_OVF_LEN);
if (got + chunk > total){ free(body); return -1; }
memcpy(body+got, ov+VS_OVF_DATA, chunk); got += chunk;
id = vg_u64(ov + VS_OVF_NEXT);
}
if (got != total){ free(body); return -1; }
*out = body; *outlen = total;
} else {
uint16_t reclen = vg_u16(page + off);
if (reclen < VS_REC_HDR) return -1;
size_t blen = reclen - VS_REC_HDR;
uint8_t* body = (uint8_t*)malloc(blen ? blen : 1);
if (!body) return -1;
memcpy(body, page + off + VS_REC_HDR, blen);
*out = body; *outlen = blen;
}
return 0;
}
/* Extract id (strdup) and emb (malloc'd float[dim]) from a TLV node body. */
static void vs_parse_node(const uint8_t* body, size_t len, char** id_out,
float** emb_out, int* dim_out){
*id_out=NULL; *emb_out=NULL; *dim_out=0;
size_t i=0;
while (i + 5 <= len){
uint8_t tag = body[i];
uint32_t flen = vg_u32(body + i + 1);
if (i + 5 + (size_t)flen > len) break;
const uint8_t* v = body + i + 5;
if (tag == VS_NT_ID){
char* s = (char*)malloc(flen+1);
if (s){ memcpy(s,v,flen); s[flen]=0; free(*id_out); *id_out=s; }
} else if (tag == VS_NT_EMB){
int dim = (int)(flen/4);
float* e = (float*)malloc((size_t)(dim?dim:1)*sizeof(float));
if (e){ for (int k=0;k<dim;k++){ uint32_t u=vg_u32(v+k*4); memcpy(&e[k],&u,4);}
free(*emb_out); *emb_out=e; if(*dim_out==0) *dim_out=dim; }
} else if (tag == VS_NT_EMB_DIM){
*dim_out = (int)vg_u32(v);
}
i += 5 + flen;
}
}
/* Tiny open-addressing string set to dedup ids across live records. */
typedef struct { char** k; size_t cap, n; } StrSet;
static uint64_t vs_fnv(const char* s){ uint64_t h=1469598103934665603ULL; for(;*s;++s){h^=(uint8_t)*s;h*=1099511628211ULL;} return h; }
static int strset_add(StrSet* s, const char* key){ /* 1 added, 0 dup, -1 err */
if (s->n*2 >= s->cap){
size_t nc = s->cap ? s->cap*2 : 1024;
char** nk = (char**)calloc(nc, sizeof(char*));
if (!nk) return -1;
for (size_t i=0;i<s->cap;i++) if (s->k[i]){ size_t j=vs_fnv(s->k[i])&(nc-1); while(nk[j]) j=(j+1)&(nc-1); nk[j]=s->k[i]; }
free(s->k); s->k=nk; s->cap=nc;
}
size_t j = vs_fnv(key)&(s->cap-1);
while (s->k[j]){ if (strcmp(s->k[j],key)==0) return 0; j=(j+1)&(s->cap-1); }
char* d = strdup(key); if(!d) return -1;
s->k[j]=d; s->n++;
return 1;
}
static void strset_free(StrSet* s){ for(size_t i=0;i<s->cap;i++) free(s->k[i]); free(s->k); }
int vindex_build_from_store(VIndex* ix, const char* store_path,
char*** ids_out, int* n_out){
if (!ix || !store_path) return -1;
int fd = open(store_path, O_RDONLY);
if (fd < 0) return -1;
struct stat st;
if (fstat(fd, &st) != 0){ close(fd); return -1; }
uint64_t npages = (uint64_t)st.st_size / VS_PAGE_SIZE;
char** ids = NULL; size_t ids_n = 0, ids_cap = 0;
StrSet seen = {0,0,0};
int inserted = 0;
uint8_t page[VS_PAGE_SIZE];
for (uint64_t pg = 2; pg < npages; pg++){ /* pages 0,1 = superblocks */
if (vs_pread(fd, pg, page)) continue;
if (page[8] != VS_PT_NODE) continue;
int slots = vg_u16(page + 10);
for (int sidx=0; sidx<slots; sidx++){
const uint8_t* sp = page + VS_HDR + (size_t)sidx*VS_SLOT_SIZE;
uint16_t off = vg_u16(sp), len = vg_u16(sp+2), fl = vg_u16(sp+4);
if (fl != VS_SLOT_LIVE) continue;
if ((size_t)off + VS_REC_HDR > VS_PAGE_SIZE) continue;
uint8_t* body=NULL; size_t blen=0;
if (vs_read_body(fd, page, off, len, &body, &blen)) continue;
char* id=NULL; float* emb=NULL; int dim=0;
vs_parse_node(body, blen, &id, &emb, &dim);
free(body);
if (!id || !emb || dim != ix->dim){ free(id); free(emb); continue; }
int add = strset_add(&seen, id);
if (add <= 0){ free(id); free(emb); continue; } /* dup or err */
if (vindex_insert(ix, (uint64_t)inserted, emb) != 0){ free(id); free(emb); break; }
free(emb);
if (ids_n == ids_cap){
size_t nc = ids_cap ? ids_cap*2 : 256;
char** ni = (char**)realloc(ids, nc*sizeof(char*));
if (!ni){ free(id); break; }
ids = ni; ids_cap = nc;
}
ids[ids_n++] = id; /* transfers ownership */
inserted++;
}
}
close(fd);
strset_free(&seen);
if (ids_out){ *ids_out = ids; if (n_out) *n_out = (int)ids_n; }
else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); if (n_out) *n_out=(int)ids_n; }
return inserted;
}
/* ── optional persistence (index is rebuildable; convenience only) ─────────── */
#define VINDEX_SAVE_MAGIC "EGVIDX01"
int vindex_save(const VIndex* ix, const char* path){
if (!ix || !path) return -1;
FILE* f = fopen(path, "wb");
if (!f) return -1;
int ok = 1;
#define WR(p,n) do{ if(fwrite((p),1,(n),f)!=(size_t)(n)) ok=0; }while(0)
WR(VINDEX_SAVE_MAGIC, 8);
int32_t hdr[6] = { ix->dim, ix->M, ix->ef_construction, (int32_t)ix->n, ix->entry, ix->max_level };
WR(hdr, sizeof(hdr));
for (size_t i=0; ok && i<ix->n; i++){
Elem* e = &ix->elems[i];
WR(&e->node_id, sizeof(uint64_t));
int32_t lvl = e->level; WR(&lvl, sizeof(int32_t));
WR(e->vec, (size_t)ix->dim*sizeof(float));
for (int l=0; ok && l<=e->level; l++){
int32_t c = e->links[l].count; WR(&c, sizeof(int32_t));
WR(e->links[l].ids, (size_t)c*sizeof(int));
}
}
#undef WR
fclose(f);
return ok ? 0 : -1;
}
VIndex* vindex_load(const char* path){
FILE* f = fopen(path, "rb");
if (!f) return NULL;
char magic[8];
if (fread(magic,1,8,f)!=8 || memcmp(magic,VINDEX_SAVE_MAGIC,8)!=0){ fclose(f); return NULL; }
int32_t hdr[6];
if (fread(hdr,sizeof(hdr),1,f)!=1){ fclose(f); return NULL; }
VIndex* ix = vindex_create(hdr[0], hdr[1], hdr[2]);
if (!ix){ fclose(f); return NULL; }
size_t N = (size_t)hdr[3];
int ok = 1;
for (size_t i=0; ok && i<N; i++){
if (elems_reserve(ix)){ ok=0; break; }
Elem* e = &ix->elems[ix->n];
int32_t lvl;
if (fread(&e->node_id,sizeof(uint64_t),1,f)!=1 || fread(&lvl,sizeof(int32_t),1,f)!=1){ ok=0; break; }
e->level = lvl;
e->vec = (float*)malloc((size_t)ix->dim*sizeof(float));
e->links = (NeighList*)calloc((size_t)lvl+1, sizeof(NeighList));
if (!e->vec || !e->links){ free(e->vec); free(e->links); ok=0; break; }
if (fread(e->vec,sizeof(float),(size_t)ix->dim,f)!=(size_t)ix->dim){ ok=0; }
for (int l=0; ok && l<=lvl; l++){
int32_t c; if (fread(&c,sizeof(int32_t),1,f)!=1){ ok=0; break; }
e->links[l].ids = (int*)malloc((size_t)(c?c:1)*sizeof(int));
e->links[l].cap = c; e->links[l].count = c;
if (c && fread(e->links[l].ids,sizeof(int),(size_t)c,f)!=(size_t)c){ ok=0; }
}
ix->n++;
}
ix->entry = hdr[4]; ix->max_level = hdr[5];
fclose(f);
if (!ok){ vindex_free(ix); return NULL; }
return ix;
}
lang/runtime/engram_vindex.h
+82
View File
@@ -0,0 +1,82 @@
/* engram_vindex.h — M8 of the engram query engine: an approximate-nearest-
* neighbour (ANN) vector index over the node embedding vectors, for fast
* activation-seed selection.
*
* Replaces the O(n) cosine scan over emb vectors (design §9 M8; backlog #20)
* with an HNSW (Hierarchical Navigable Small World) graph that returns
* high-recall top-k seeds in ~O(log n).
*
* Standalone module: plain C11, stdlib + libm only. It does NOT modify the
* store format or engram_store.{c,h}; vindex_build_from_store() decodes the
* PERMANENT on-disk node format (design §2.4) read-only to harvest emb vectors.
*
* Similarity metric: cosine. Vectors are L2-normalised on insert/query, so
* cosine similarity == dot product. Reported distance = 1 - cosine_similarity
* (range [0,2]); smaller == closer. A query equal to an indexed vector scores
* distance ~0 against it.
*
* The index is fully rebuildable from the store, so persistence is optional for
* this milestone (see vindex_save/vindex_load below — provided as a convenience;
* boot may simply rebuild via vindex_build_from_store()).
*/
#ifndef ENGRAM_VINDEX_H
#define ENGRAM_VINDEX_H
#include <stddef.h>
#include <stdint.h>
/* Tuned defaults (rationale in engram_vindex.c). Pass 0 to vindex_create for
* M / ef_construction to take these; pass ef_search<=0 to vindex_search for
* VINDEX_DEFAULT_EF_SEARCH. */
#define VINDEX_DEFAULT_M 24
#define VINDEX_DEFAULT_EF_CONSTRUCTION 200
#define VINDEX_DEFAULT_EF_SEARCH 128
typedef struct VIndex VIndex;
/* Create an index over `dim`-dimensional f32 vectors.
* M — max neighbours per node on upper layers (2*M on layer 0).
* ef_construction — candidate-list width during insert (recall/build cost).
* Pass M<=0 or ef_construction<=0 to use the VINDEX_DEFAULT_* above.
* Returns NULL on bad args / OOM. */
VIndex* vindex_create(int dim, int M, int ef_construction);
/* Insert one vector under an opaque caller-defined node_id (need not be unique,
* but the caller is responsible for meaning). `vec` has `dim` floats; it is
* copied and L2-normalised internally. A zero vector is accepted (it simply has
* distance ~1 to everything; never produces NaN). Returns 0 on success, <0 on
* error (bad args / OOM). */
int vindex_insert(VIndex* idx, uint64_t node_id, const float* vec);
/* Top-k search by cosine similarity. Writes up to k results (fewer if the index
* holds fewer than k elements) into node_id_out[] / dist_out[], ordered nearest
* first (ascending distance). Either out array may be NULL to skip it.
* ef_search — search-time candidate width; larger == higher recall, slower.
* Pass <=0 for VINDEX_DEFAULT_EF_SEARCH. Internally clamped to >=k.
* Returns the number of results written, or <0 on error. */
int vindex_search(VIndex* idx, const float* query, int k, int ef_search,
uint64_t* node_id_out, float* dist_out);
/* Number of vectors currently indexed. */
size_t vindex_size(const VIndex* idx);
void vindex_free(VIndex* idx);
/* Build an index by scanning every live node record in the paged store at
* `store_path` (the on-disk format is decoded read-only; the store need not be
* open). Nodes without an emb vector, or whose emb_dim != idx->dim, are skipped.
* Each inserted node is assigned node_id = its 0-based insertion ordinal; if
* `ids_out`/`n_out` are non-NULL, *ids_out is set to a malloc'd array of that
* many strdup'd string ids (ids_out[node_id] == the store id) and *n_out to the
* count — the caller frees each string and the array. Returns the number of
* vectors inserted, or <0 on error. */
int vindex_build_from_store(VIndex* idx, const char* store_path,
char*** ids_out, int* n_out);
/* Optional persistence (index is rebuildable from the store; provided for
* convenience). vindex_save writes a self-describing snapshot; vindex_load
* reconstructs an index from one. Return 0 / non-NULL on success. */
int vindex_save(const VIndex* idx, const char* path);
VIndex* vindex_load(const char* path);
#endif /* ENGRAM_VINDEX_H */