From e99a4640e2297c64bbcc5f6ee0acc61ac65b6a65 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sun, 16 Aug 2026 08:51:17 -0500 Subject: [PATCH] test: regression harness for the vindex concurrency crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes the two throwaway sanitizer harnesses used to diagnose the 2026-08-16 soul crash into engram/test/ so the bug cannot silently regress. The harness has two halves and the PAIR is the point — it is what localises the defect to concurrency rather than to HNSW logic: single 3000 clustered vectors, one thread, ASan+UBSan. The CONTROL. Must always be clean. During diagnosis this cleared all 13,820 real dim-768 vectors from the live store, which DISPROVED an inspection-derived hypothesis about an out-of-bounds reverse-link write at engram_vindex.c:340. concurrent writer + reader on one shared index, TSan. Currently reports a race at engram_vindex.c:195 (visited_reset) reached from both vindex_search and vindex_insert, because VIndex still owns its visited[]/visit_epoch scratch — so even two concurrent READS corrupt each other's traversal. Verified: half 1 passes, half 2 reproduces the race. Gated on EXPECT_RACE, default 1, so the concurrent half documents the known defect without failing the suite today. When the visited set moves to a per-query checkout pool (hnswlib VisitedListPool style — NOT thread_local, since http_worker is a thread per connection and a __thread buffer would leak ~55KB per connection), flip EXPECT_RACE=0 and it becomes a real gate. --- engram/test/run_vindex_concurrency_tests.sh | 71 ++++++++++ engram/test/test_vindex_concurrency.c | 136 ++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100755 engram/test/run_vindex_concurrency_tests.sh create mode 100644 engram/test/test_vindex_concurrency.c diff --git a/engram/test/run_vindex_concurrency_tests.sh b/engram/test/run_vindex_concurrency_tests.sh new file mode 100755 index 0000000..a79d62b --- /dev/null +++ b/engram/test/run_vindex_concurrency_tests.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# run_vindex_concurrency_tests.sh — regression harness for the 2026-08-16 soul crash. +# +# Runs two halves. The PAIR is the point: the ASan control localises the defect to +# concurrency rather than to HNSW logic. See test_vindex_concurrency.c for the full +# story (SIGSEGV at ASCII address "gramNode", heap corruption in xzm_realloc, etc). +# +# 1. single — ASan+UBSan, one thread. MUST be clean. Always a hard failure. +# 2. concurrent — TSan, writer + reader on one index. Currently EXPECTED to report a +# race at visited_reset, because VIndex still owns its visited[] + +# visit_epoch scratch. Once that moves to a per-query checkout pool +# this must go clean; flip EXPECT_RACE=0 then and it becomes a real +# regression gate. +# +# usage: run_vindex_concurrency_tests.sh +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNTIME="$(cd "$HERE/../../lang/runtime" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +SRC="$HERE/test_vindex_concurrency.c" +VINDEX="$RUNTIME/engram_vindex.c" + +# Flip to 0 once the visited set is per-query; the concurrent half then becomes a gate. +EXPECT_RACE="${EXPECT_RACE:-1}" + +fail=0 + +echo "== [1/2] single-threaded control under AddressSanitizer ==" +cc -std=c11 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \ + -I"$RUNTIME" -o "$WORK/single" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; } +if ASAN_OPTIONS=detect_leaks=0 "$WORK/single" single; then + echo " -> OK" +else + echo " -> FAIL: the single-threaded control must always be clean." + echo " If this fails the bug is NOT (only) concurrency — look for a real" + echo " out-of-bounds or lifetime error in engram_vindex.c." + fail=1 +fi + +echo +echo "== [2/2] concurrent writer+reader under ThreadSanitizer ==" +cc -std=c11 -g -O1 -fsanitize=thread -fno-omit-frame-pointer \ + -I"$RUNTIME" -o "$WORK/conc" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; } + +tsan_log="$WORK/tsan.log" +TSAN_OPTIONS="halt_on_error=0" "$WORK/conc" concurrent >"$tsan_log" 2>&1 +if grep -q "ThreadSanitizer: data race" "$tsan_log"; then + echo " -> RACE DETECTED:" + grep -m1 -A6 "ThreadSanitizer: data race" "$tsan_log" | sed 's/^/ /' + if [ "$EXPECT_RACE" = "1" ]; then + echo " -> EXPECTED (VIndex still owns the shared visited set). Not a failure yet." + echo " Fix = per-query visited buffer (hnswlib VisitedListPool style), then" + echo " re-run with EXPECT_RACE=0." + else + echo " -> REGRESSION: the visited set was supposed to be per-query." + fail=1 + fi +else + echo " -> clean" + if [ "$EXPECT_RACE" = "1" ]; then + echo " -> NOTE: no race reported, but EXPECT_RACE=1. Either the fix landed" + echo " (set EXPECT_RACE=0) or the test did not actually interleave." + fi +fi + +echo +[ "$fail" -eq 0 ] && echo "RESULT: PASS" || echo "RESULT: FAIL" +exit "$fail" diff --git a/engram/test/test_vindex_concurrency.c b/engram/test/test_vindex_concurrency.c new file mode 100644 index 0000000..15751f6 --- /dev/null +++ b/engram/test/test_vindex_concurrency.c @@ -0,0 +1,136 @@ +/* test_vindex_concurrency.c — regression test for the 2026-08-16 soul crash. + * + * WHAT BROKE: the soul daemon crash-looped (5 crashes in ~100s) with SIGSEGV in + * search_layer <- vindex_insert <- eg_vindex_sync, a SIGABRT, and a fault inside + * xzm_realloc's own freelist — i.e. heap corruption. The SIGSEGV address + * 0x65646f4e6d617267 is little-endian ASCII "gramNode": string bytes being + * dereferenced as an Elem vector pointer. + * + * ROOT CAUSE: VIndex owns its traversal scratch (visited[] + visit_epoch), and + * search_layer mutates it via visited_reset(). So the index is unsafe for ANY + * concurrent use — including two concurrent READS. soul.el starts http_serve_async + * (a thread per connection) and then runs awareness_run() on the main thread, which + * reaches the same global index through engram_activate; nothing serialized them. + * + * Neither hnswlib nor FAISS puts the visited set on the index: hnswlib checks one + * out of a VisitedListPool per query, FAISS uses a thread_local VisitedTable. + * + * THIS TEST HAS TWO HALVES, and they must BOTH be run — the pair is what localises + * the bug to concurrency rather than to HNSW logic: + * + * single Insert N clustered vectors on ONE thread and search. Build with ASan. + * This is the CONTROL. It must stay clean. When this passes and `concurrent` + * fails, the defect is concurrency, not an out-of-bounds/logic error in the + * graph code. (On 2026-08-16 this control cleared all 13,820 real dim-768 + * store vectors under ASan, which DISPROVED an inspection-derived hypothesis + * about an out-of-bounds reverse-link write at engram_vindex.c:340.) + * + * concurrent One writer thread inserting while a reader thread searches the SAME + * index. Build with TSan. Until the visited set moves off the index struct + * this is EXPECTED TO REPORT A RACE at engram_vindex.c visited_reset — + * that is the bug, reproduced. Once a per-query visited buffer lands + * (see backlog: "Move VIndex visited-set off the index struct"), this must + * become clean, and THAT is the regression this file guards. + * + * Absence of a crash does NOT mean absence of a race — always read the sanitizer + * verdict, never just the exit code. + * + * Build/run: engram/test/run_vindex_concurrency_tests.sh + */ +#include "engram_vindex.h" + +#include +#include +#include +#include +#include + +#define DIM 128 +#define NVEC 3000 +#define SEED_N 50 + +static VIndex* g_ix; +static float* g_vecs; + +/* Deterministic filler. Real embeddings are strongly correlated, not uniform noise; + * clustering keeps many candidates near-equidistant, which exercises the diversity + * heuristic and the visited set far harder than random vectors do. */ +static void fill_vectors(void) { + g_vecs = (float*)malloc((size_t)NVEC * DIM * sizeof(float)); + if (!g_vecs) { fprintf(stderr, "OOM\n"); exit(1); } + for (int i = 0; i < NVEC; i++) { + int cluster = i % 8; + for (int d = 0; d < DIM; d++) + g_vecs[(size_t)i * DIM + d] = + (float)(((d + cluster * 7) % 13) / 13.0) + + (float)(((i * 2654435761u + (unsigned)d) % 97) / 9700.0); + } +} + +static void* writer_fn(void* arg) { + (void)arg; + for (int i = SEED_N; i < NVEC; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + return NULL; +} + +static void* reader_fn(void* arg) { + (void)arg; + uint64_t ids[8]; float ds[8]; + for (int i = 0; i < 20000; i++) + (void)vindex_search(g_ix, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds); + return NULL; +} + +static int run_single(void) { + printf("[single] inserting %d vectors on one thread (ASan control)\n", NVEC); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[single] vindex_create failed\n"); return 1; } + for (int i = 0; i < NVEC; i++) { + if (vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM) != 0) { + fprintf(stderr, "[single] insert %d failed\n", i); return 1; + } + } + if (vindex_size(g_ix) != (size_t)NVEC) { + fprintf(stderr, "[single] size %zu != %d\n", vindex_size(g_ix), NVEC); return 1; + } + uint64_t ids[16]; float ds[16]; + for (int q = 0; q < 200; q++) { + int k = vindex_search(g_ix, g_vecs + (size_t)((q * 7) % NVEC) * DIM, 16, 0, ids, ds); + if (k < 0) { fprintf(stderr, "[single] search failed at q=%d\n", q); return 1; } + } + vindex_free(g_ix); g_ix = NULL; + printf("[single] PASS — no memory error (this must ALWAYS pass)\n"); + return 0; +} + +static int run_concurrent(void) { + printf("[concurrent] 1 writer + 1 reader on ONE shared index (TSan probe)\n"); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[concurrent] vindex_create failed\n"); return 1; } + for (int i = 0; i < SEED_N; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + + pthread_t w, r; + if (pthread_create(&w, NULL, writer_fn, NULL) || + pthread_create(&r, NULL, reader_fn, NULL)) { + fprintf(stderr, "[concurrent] pthread_create failed\n"); return 1; + } + pthread_join(w, NULL); + pthread_join(r, NULL); + vindex_free(g_ix); g_ix = NULL; + printf("[concurrent] completed — CHECK THE SANITIZER VERDICT, not this line.\n"); + printf("[concurrent] a clean TSan run here is the actual pass condition.\n"); + return 0; +} + +int main(int argc, char** argv) { + const char* mode = (argc > 1) ? argv[1] : "single"; + fill_vectors(); + int rc; + if (!strcmp(mode, "single")) rc = run_single(); + else if (!strcmp(mode, "concurrent")) rc = run_concurrent(); + else { fprintf(stderr, "usage: %s [single|concurrent]\n", argv[0]); rc = 2; } + free(g_vecs); + return rc; +}