83 lines
4.0 KiB
C
83 lines
4.0 KiB
C
/* engram_vindex.h — M8 of the engram query engine: an approximate-nearest-
|
|
* neighbour (ANN) vector index over the node embedding vectors, for fast
|
|
* activation-seed selection.
|
|
*
|
|
* Replaces the O(n) cosine scan over emb vectors (design §9 M8; backlog #20)
|
|
* with an HNSW (Hierarchical Navigable Small World) graph that returns
|
|
* high-recall top-k seeds in ~O(log n).
|
|
*
|
|
* Standalone module: plain C11, stdlib + libm only. It does NOT modify the
|
|
* store format or engram_store.{c,h}; vindex_build_from_store() decodes the
|
|
* PERMANENT on-disk node format (design §2.4) read-only to harvest emb vectors.
|
|
*
|
|
* Similarity metric: cosine. Vectors are L2-normalised on insert/query, so
|
|
* cosine similarity == dot product. Reported distance = 1 - cosine_similarity
|
|
* (range [0,2]); smaller == closer. A query equal to an indexed vector scores
|
|
* distance ~0 against it.
|
|
*
|
|
* The index is fully rebuildable from the store, so persistence is optional for
|
|
* this milestone (see vindex_save/vindex_load below — provided as a convenience;
|
|
* boot may simply rebuild via vindex_build_from_store()).
|
|
*/
|
|
#ifndef ENGRAM_VINDEX_H
|
|
#define ENGRAM_VINDEX_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* Tuned defaults (rationale in engram_vindex.c). Pass 0 to vindex_create for
|
|
* M / ef_construction to take these; pass ef_search<=0 to vindex_search for
|
|
* VINDEX_DEFAULT_EF_SEARCH. */
|
|
#define VINDEX_DEFAULT_M 24
|
|
#define VINDEX_DEFAULT_EF_CONSTRUCTION 200
|
|
#define VINDEX_DEFAULT_EF_SEARCH 128
|
|
|
|
typedef struct VIndex VIndex;
|
|
|
|
/* Create an index over `dim`-dimensional f32 vectors.
|
|
* M — max neighbours per node on upper layers (2*M on layer 0).
|
|
* ef_construction — candidate-list width during insert (recall/build cost).
|
|
* Pass M<=0 or ef_construction<=0 to use the VINDEX_DEFAULT_* above.
|
|
* Returns NULL on bad args / OOM. */
|
|
VIndex* vindex_create(int dim, int M, int ef_construction);
|
|
|
|
/* Insert one vector under an opaque caller-defined node_id (need not be unique,
|
|
* but the caller is responsible for meaning). `vec` has `dim` floats; it is
|
|
* copied and L2-normalised internally. A zero vector is accepted (it simply has
|
|
* distance ~1 to everything; never produces NaN). Returns 0 on success, <0 on
|
|
* error (bad args / OOM). */
|
|
int vindex_insert(VIndex* idx, uint64_t node_id, const float* vec);
|
|
|
|
/* Top-k search by cosine similarity. Writes up to k results (fewer if the index
|
|
* holds fewer than k elements) into node_id_out[] / dist_out[], ordered nearest
|
|
* first (ascending distance). Either out array may be NULL to skip it.
|
|
* ef_search — search-time candidate width; larger == higher recall, slower.
|
|
* Pass <=0 for VINDEX_DEFAULT_EF_SEARCH. Internally clamped to >=k.
|
|
* Returns the number of results written, or <0 on error. */
|
|
int vindex_search(VIndex* idx, const float* query, int k, int ef_search,
|
|
uint64_t* node_id_out, float* dist_out);
|
|
|
|
/* Number of vectors currently indexed. */
|
|
size_t vindex_size(const VIndex* idx);
|
|
|
|
void vindex_free(VIndex* idx);
|
|
|
|
/* Build an index by scanning every live node record in the paged store at
|
|
* `store_path` (the on-disk format is decoded read-only; the store need not be
|
|
* open). Nodes without an emb vector, or whose emb_dim != idx->dim, are skipped.
|
|
* Each inserted node is assigned node_id = its 0-based insertion ordinal; if
|
|
* `ids_out`/`n_out` are non-NULL, *ids_out is set to a malloc'd array of that
|
|
* many strdup'd string ids (ids_out[node_id] == the store id) and *n_out to the
|
|
* count — the caller frees each string and the array. Returns the number of
|
|
* vectors inserted, or <0 on error. */
|
|
int vindex_build_from_store(VIndex* idx, const char* store_path,
|
|
char*** ids_out, int* n_out);
|
|
|
|
/* Optional persistence (index is rebuildable from the store; provided for
|
|
* convenience). vindex_save writes a self-describing snapshot; vindex_load
|
|
* reconstructs an index from one. Return 0 / non-NULL on success. */
|
|
int vindex_save(const VIndex* idx, const char* path);
|
|
VIndex* vindex_load(const char* path);
|
|
|
|
#endif /* ENGRAM_VINDEX_H */
|