el/engram/test/test_vindex_concurrency.c

/* test_vindex_concurrency.c — regression test for the 2026-08-16 soul crash.
 *
 * WHAT BROKE: the soul daemon crash-looped (5 crashes in ~100s) with SIGSEGV in
 * search_layer <- vindex_insert <- eg_vindex_sync, a SIGABRT, and a fault inside
 * xzm_realloc's own freelist — i.e. heap corruption. The SIGSEGV address
 * 0x65646f4e6d617267 is little-endian ASCII "gramNode": string bytes being
 * dereferenced as an Elem vector pointer.
 *
 * ROOT CAUSE: VIndex owns its traversal scratch (visited[] + visit_epoch), and
 * search_layer mutates it via visited_reset(). So the index is unsafe for ANY
 * concurrent use — including two concurrent READS. soul.el starts http_serve_async
 * (a thread per connection) and then runs awareness_run() on the main thread, which
 * reaches the same global index through engram_activate; nothing serialized them.
 *
 * Neither hnswlib nor FAISS puts the visited set on the index: hnswlib checks one
 * out of a VisitedListPool per query, FAISS uses a thread_local VisitedTable.
 *
 * THIS TEST HAS TWO HALVES, and they must BOTH be run — the pair is what localises
 * the bug to concurrency rather than to HNSW logic:
 *
 *   single   Insert N clustered vectors on ONE thread and search. Build with ASan.
 *            This is the CONTROL. It must stay clean. When this passes and `concurrent`
 *            fails, the defect is concurrency, not an out-of-bounds/logic error in the
 *            graph code. (On 2026-08-16 this control cleared all 13,820 real dim-768
 *            store vectors under ASan, which DISPROVED an inspection-derived hypothesis
 *            about an out-of-bounds reverse-link write at engram_vindex.c:340.)
 *
 *   concurrent  One writer thread inserting while a reader thread searches the SAME
 *            index. Build with TSan. Until the visited set moves off the index struct
 *            this is EXPECTED TO REPORT A RACE at engram_vindex.c visited_reset —
 *            that is the bug, reproduced. Once a per-query visited buffer lands
 *            (see backlog: "Move VIndex visited-set off the index struct"), this must
 *            become clean, and THAT is the regression this file guards.
 *
 * Absence of a crash does NOT mean absence of a race — always read the sanitizer
 * verdict, never just the exit code.
 *
 * Build/run: engram/test/run_vindex_concurrency_tests.sh
 */
#include "engram_vindex.h"

#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>

#define DIM   128
#define NVEC  3000
#define SEED_N 50

static VIndex* g_ix;
static float*  g_vecs;

/* Deterministic filler. Real embeddings are strongly correlated, not uniform noise;
 * clustering keeps many candidates near-equidistant, which exercises the diversity
 * heuristic and the visited set far harder than random vectors do. */
static void fill_vectors(void) {
    g_vecs = (float*)malloc((size_t)NVEC * DIM * sizeof(float));
    if (!g_vecs) { fprintf(stderr, "OOM\n"); exit(1); }
    for (int i = 0; i < NVEC; i++) {
        int cluster = i % 8;
        for (int d = 0; d < DIM; d++)
            g_vecs[(size_t)i * DIM + d] =
                (float)(((d + cluster * 7) % 13) / 13.0) +
                (float)(((i * 2654435761u + (unsigned)d) % 97) / 9700.0);
    }
}

static void* writer_fn(void* arg) {
    (void)arg;
    for (int i = SEED_N; i < NVEC; i++)
        (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
    return NULL;
}

static void* reader_fn(void* arg) {
    (void)arg;
    uint64_t ids[8]; float ds[8];
    for (int i = 0; i < 20000; i++)
        (void)vindex_search(g_ix, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds);
    return NULL;
}

static int run_single(void) {
    printf("[single] inserting %d vectors on one thread (ASan control)\n", NVEC);
    g_ix = vindex_create(DIM, 0, 0);
    if (!g_ix) { fprintf(stderr, "[single] vindex_create failed\n"); return 1; }
    for (int i = 0; i < NVEC; i++) {
        if (vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM) != 0) {
            fprintf(stderr, "[single] insert %d failed\n", i); return 1;
        }
    }
    if (vindex_size(g_ix) != (size_t)NVEC) {
        fprintf(stderr, "[single] size %zu != %d\n", vindex_size(g_ix), NVEC); return 1;
    }
    uint64_t ids[16]; float ds[16];
    for (int q = 0; q < 200; q++) {
        int k = vindex_search(g_ix, g_vecs + (size_t)((q * 7) % NVEC) * DIM, 16, 0, ids, ds);
        if (k < 0) { fprintf(stderr, "[single] search failed at q=%d\n", q); return 1; }
    }
    vindex_free(g_ix); g_ix = NULL;
    printf("[single] PASS — no memory error (this must ALWAYS pass)\n");
    return 0;
}

static int run_concurrent(void) {
    printf("[concurrent] 1 writer + 1 reader on ONE shared index (TSan probe)\n");
    g_ix = vindex_create(DIM, 0, 0);
    if (!g_ix) { fprintf(stderr, "[concurrent] vindex_create failed\n"); return 1; }
    for (int i = 0; i < SEED_N; i++)
        (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);

    pthread_t w, r;
    if (pthread_create(&w, NULL, writer_fn, NULL) ||
        pthread_create(&r, NULL, reader_fn, NULL)) {
        fprintf(stderr, "[concurrent] pthread_create failed\n"); return 1;
    }
    pthread_join(w, NULL);
    pthread_join(r, NULL);
    vindex_free(g_ix); g_ix = NULL;
    printf("[concurrent] completed — CHECK THE SANITIZER VERDICT, not this line.\n");
    printf("[concurrent] a clean TSan run here is the actual pass condition.\n");
    return 0;
}

int main(int argc, char** argv) {
    const char* mode = (argc > 1) ? argv[1] : "single";
    fill_vectors();
    int rc;
    if      (!strcmp(mode, "single"))     rc = run_single();
    else if (!strcmp(mode, "concurrent")) rc = run_concurrent();
    else { fprintf(stderr, "usage: %s [single|concurrent]\n", argv[0]); rc = 2; }
    free(g_vecs);
    return rc;
}