bigmerge
08cbcef5d9
El SDK CI - dev / build-and-test (pull_request) Successful in 6m42s
Adds an O(1) "seen" bitmap so lazily-embedded older nodes get picked up incrementally instead of only on a full rebuild (embed-gap #20). Replaces engram_activate's O(N*D) cosine prescan with a lazy-memoized cosine cache (eg_cosq_at), proven bit-identical to the old path. Extracts a clean vindex_harvest_from_store primitive (read-only vector harvest, careful malloc/ownership/error-path handling) reused by both index-build and the new vindex_bench.c — a read-only proof harness comparing brute-force vs HNSW recall/latency on both the real store and synthetic data. .nsbx-env intentionally excluded — local sandbox config (ports, paths, dev-only placeholder key), not checked in.
95 lines
4.8 KiB
C
95 lines
4.8 KiB
C
/* engram_vindex.h — M8 of the engram query engine: an approximate-nearest-
|
|
* neighbour (ANN) vector index over the node embedding vectors, for fast
|
|
* activation-seed selection.
|
|
*
|
|
* Replaces the O(n) cosine scan over emb vectors (design §9 M8; backlog #20)
|
|
* with an HNSW (Hierarchical Navigable Small World) graph that returns
|
|
* high-recall top-k seeds in ~O(log n).
|
|
*
|
|
* Standalone module: plain C11, stdlib + libm only. It does NOT modify the
|
|
* store format or engram_store.{c,h}; vindex_build_from_store() decodes the
|
|
* PERMANENT on-disk node format (design §2.4) read-only to harvest emb vectors.
|
|
*
|
|
* Similarity metric: cosine. Vectors are L2-normalised on insert/query, so
|
|
* cosine similarity == dot product. Reported distance = 1 - cosine_similarity
|
|
* (range [0,2]); smaller == closer. A query equal to an indexed vector scores
|
|
* distance ~0 against it.
|
|
*
|
|
* The index is fully rebuildable from the store, so persistence is optional for
|
|
* this milestone (see vindex_save/vindex_load below — provided as a convenience;
|
|
* boot may simply rebuild via vindex_build_from_store()).
|
|
*/
|
|
#ifndef ENGRAM_VINDEX_H
|
|
#define ENGRAM_VINDEX_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* Tuned defaults (rationale in engram_vindex.c). Pass 0 to vindex_create for
|
|
* M / ef_construction to take these; pass ef_search<=0 to vindex_search for
|
|
* VINDEX_DEFAULT_EF_SEARCH. */
|
|
#define VINDEX_DEFAULT_M 24
|
|
#define VINDEX_DEFAULT_EF_CONSTRUCTION 200
|
|
#define VINDEX_DEFAULT_EF_SEARCH 128
|
|
|
|
typedef struct VIndex VIndex;
|
|
|
|
/* Create an index over `dim`-dimensional f32 vectors.
|
|
* M — max neighbours per node on upper layers (2*M on layer 0).
|
|
* ef_construction — candidate-list width during insert (recall/build cost).
|
|
* Pass M<=0 or ef_construction<=0 to use the VINDEX_DEFAULT_* above.
|
|
* Returns NULL on bad args / OOM. */
|
|
VIndex* vindex_create(int dim, int M, int ef_construction);
|
|
|
|
/* Insert one vector under an opaque caller-defined node_id (need not be unique,
|
|
* but the caller is responsible for meaning). `vec` has `dim` floats; it is
|
|
* copied and L2-normalised internally. A zero vector is accepted (it simply has
|
|
* distance ~1 to everything; never produces NaN). Returns 0 on success, <0 on
|
|
* error (bad args / OOM). */
|
|
int vindex_insert(VIndex* idx, uint64_t node_id, const float* vec);
|
|
|
|
/* Top-k search by cosine similarity. Writes up to k results (fewer if the index
|
|
* holds fewer than k elements) into node_id_out[] / dist_out[], ordered nearest
|
|
* first (ascending distance). Either out array may be NULL to skip it.
|
|
* ef_search — search-time candidate width; larger == higher recall, slower.
|
|
* Pass <=0 for VINDEX_DEFAULT_EF_SEARCH. Internally clamped to >=k.
|
|
* Returns the number of results written, or <0 on error. */
|
|
int vindex_search(VIndex* idx, const float* query, int k, int ef_search,
|
|
uint64_t* node_id_out, float* dist_out);
|
|
|
|
/* Number of vectors currently indexed. */
|
|
size_t vindex_size(const VIndex* idx);
|
|
|
|
void vindex_free(VIndex* idx);
|
|
|
|
/* Build an index by scanning every live node record in the paged store at
|
|
* `store_path` (the on-disk format is decoded read-only; the store need not be
|
|
* open). Nodes without an emb vector, or whose emb_dim != idx->dim, are skipped.
|
|
* Each inserted node is assigned node_id = its 0-based insertion ordinal; if
|
|
* `ids_out`/`n_out` are non-NULL, *ids_out is set to a malloc'd array of that
|
|
* many strdup'd string ids (ids_out[node_id] == the store id) and *n_out to the
|
|
* count — the caller frees each string and the array. Returns the number of
|
|
* vectors inserted, or <0 on error. */
|
|
int vindex_build_from_store(VIndex* idx, const char* store_path,
|
|
char*** ids_out, int* n_out);
|
|
|
|
/* Read-only harvest of the raw (un-normalised) emb vectors from a paged store,
|
|
* applying the SAME filtering vindex_build_from_store does (live records only,
|
|
* deduped by store id, emb present with emb_dim == `dim`), in insertion order.
|
|
* On success sets *vecs_out to a malloc'd float[n*dim] (row i == the i-th kept
|
|
* vector) and *n_out to n; if `ids_out` is non-NULL, sets it to a malloc'd array
|
|
* of n strdup'd store ids (ids_out[i] == the id of row i). Caller frees *vecs_out,
|
|
* each id string, and the id array. Returns n, or <0 on error. Used both by
|
|
* vindex_build_from_store (which then inserts each row) and by benchmarks/oracles
|
|
* that need the same vector set the index holds. */
|
|
int vindex_harvest_from_store(const char* store_path, int dim,
|
|
float** vecs_out, char*** ids_out, int* n_out);
|
|
|
|
/* Optional persistence (index is rebuildable from the store; provided for
|
|
* convenience). vindex_save writes a self-describing snapshot; vindex_load
|
|
* reconstructs an index from one. Return 0 / non-NULL on success. */
|
|
int vindex_save(const VIndex* idx, const char* path);
|
|
VIndex* vindex_load(const char* path);
|
|
|
|
#endif /* ENGRAM_VINDEX_H */
|