neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

engram: fix lazy-embed index gap (#20) and make activate's cosine scan lazy; extract vindex harvest primitive with a bench/oracle harness
El SDK CI - dev / build-and-test (pull_request) Successful in 6m42s
Details

Browse Source 
Adds an O(1) "seen" bitmap so lazily-embedded older nodes get picked up
incrementally instead of only on a full rebuild (embed-gap #20).

Replaces engram_activate's O(N*D) cosine prescan with a lazy-memoized
cosine cache (eg_cosq_at), proven bit-identical to the old path.

Extracts a clean vindex_harvest_from_store primitive (read-only vector
harvest, careful malloc/ownership/error-path handling) reused by both
index-build and the new vindex_bench.c — a read-only proof harness
comparing brute-force vs HNSW recall/latency on both the real store and
synthetic data.

.nsbx-env intentionally excluded — local sandbox config (ports, paths,
dev-only placeholder key), not checked in.
This commit is contained in:
bigmerge
2026-08-15 14:26:16 -05:00
parent 6621a4dbc5
commit 08cbcef5d9
5 changed files with 426 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/runtime/engram_vindex.c
+54 -14
View File
@@ -538,19 +538,20 @@ static int strset_add(StrSet* s, const char* key){ /* 1 added, 0 dup, -1 err *
}
static void strset_free(StrSet* s){ for(size_t i=0;i<s->cap;i++) free(s->k[i]); free(s->k); }
int vindex_build_from_store(VIndex* ix, const char* store_path,
char*** ids_out, int* n_out){
if (!ix || !store_path) return -1;
int vindex_harvest_from_store(const char* store_path, int dim,
float** vecs_out, char*** ids_out, int* n_out){
if (!store_path || dim <= 0 || !vecs_out) return -1;
int fd = open(store_path, O_RDONLY);
if (fd < 0) return -1;
struct stat st;
if (fstat(fd, &st) != 0){ close(fd); return -1; }
uint64_t npages = (uint64_t)st.st_size / VS_PAGE_SIZE;
char** ids = NULL; size_t ids_n = 0, ids_cap = 0;
float* vecs = NULL; size_t vn = 0, vcap = 0; /* row-major float[vn*dim] */
char** ids = NULL; size_t ids_n = 0, ids_cap = 0;
StrSet seen = {0,0,0};
int inserted = 0;
uint8_t page[VS_PAGE_SIZE];
int failed = 0;
for (uint64_t pg = 2; pg < npages; pg++){ /* pages 0,1 = superblocks */
if (vs_pread(fd, pg, page)) continue;
@@ -563,28 +564,67 @@ int vindex_build_from_store(VIndex* ix, const char* store_path,
if ((size_t)off + VS_REC_HDR > VS_PAGE_SIZE) continue;
uint8_t* body=NULL; size_t blen=0;
if (vs_read_body(fd, page, off, len, &body, &blen)) continue;
char* id=NULL; float* emb=NULL; int dim=0;
vs_parse_node(body, blen, &id, &emb, &dim);
char* id=NULL; float* emb=NULL; int edim=0;
vs_parse_node(body, blen, &id, &emb, &edim);
free(body);
if (!id || !emb || dim != ix->dim){ free(id); free(emb); continue; }
if (!id || !emb || edim != dim){ free(id); free(emb); continue; }
int add = strset_add(&seen, id);
if (add <= 0){ free(id); free(emb); continue; } /* dup or err */
if (vindex_insert(ix, (uint64_t)inserted, emb) != 0){ free(id); free(emb); break; }
if (vn == vcap){
size_t nc = vcap ? vcap*2 : 1024;
float* nv = (float*)realloc(vecs, nc*(size_t)dim*sizeof(float));
if (!nv){ free(id); free(emb); failed = 1; goto out; }
vecs = nv; vcap = nc;
}
memcpy(vecs + vn*(size_t)dim, emb, (size_t)dim*sizeof(float));
free(emb);
if (ids_n == ids_cap){
size_t nc = ids_cap ? ids_cap*2 : 256;
size_t nc = ids_cap ? ids_cap*2 : 1024;
char** ni = (char**)realloc(ids, nc*sizeof(char*));
if (!ni){ free(id); break; }
if (!ni){ free(id); failed = 1; goto out; }
ids = ni; ids_cap = nc;
}
ids[ids_n++] = id; /* transfers ownership */
inserted++;
vn++;
}
}
out:
close(fd);
strset_free(&seen);
if (ids_out){ *ids_out = ids; if (n_out) *n_out = (int)ids_n; }
else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); if (n_out) *n_out=(int)ids_n; }
if (failed){
free(vecs);
for (size_t i=0;i<ids_n;i++) free(ids[i]);
free(ids);
return -1;
}
*vecs_out = vecs;
if (n_out) *n_out = (int)vn;
if (ids_out){ *ids_out = ids; }
else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); }
return (int)vn;
}
int vindex_build_from_store(VIndex* ix, const char* store_path,
char*** ids_out, int* n_out){
if (!ix || !store_path) return -1;
float* vecs = NULL; char** ids = NULL; int n = 0;
int h = vindex_harvest_from_store(store_path, ix->dim, &vecs, &ids, &n);
if (h < 0) return -1;
int inserted = 0;
for (int i = 0; i < n; i++){
if (vindex_insert(ix, (uint64_t)inserted, vecs + (size_t)i*ix->dim) != 0) break;
inserted++;
}
free(vecs);
if (ids_out){
*ids_out = ids; if (n_out) *n_out = inserted;
/* free any ids beyond what we inserted (insert failure tail) */
for (int i = inserted; i < n; i++) free(ids[i]);
} else {
for (int i = 0; i < n; i++) free(ids[i]);
free(ids);
if (n_out) *n_out = inserted;
}
return inserted;
}