test: regression harness for the vindex concurrency crash

Promotes the two throwaway sanitizer harnesses used to diagnose the 2026-08-16 soul crash into engram/test/ so the bug cannot silently regress. The harness has two halves and the PAIR is the point — it is what localises the defect to concurrency rather than to HNSW logic: single 3000 clustered vectors, one thread, ASan+UBSan. The CONTROL. Must always be clean. During diagnosis this cleared all 13,820 real dim-768 vectors from the live store, which DISPROVED an inspection-derived hypothesis about an out-of-bounds reverse-link write at engram_vindex.c:340. concurrent writer + reader on one shared index, TSan. Currently reports a race at engram_vindex.c:195 (visited_reset) reached from both vindex_search and vindex_insert, because VIndex still owns its visited[]/visit_epoch scratch — so even two concurrent READS corrupt each other's traversal. Verified: half 1 passes, half 2 reproduces the race. Gated on EXPECT_RACE, default 1, so the concurrent half documents the known defect without failing the suite today. When the visited set moves to a per-query checkout pool (hnswlib VisitedListPool style — NOT thread_local, since http_worker is a thread per connection and a __thread buffer would leak ~55KB per connection), flip EXPECT_RACE=0 and it becomes a real gate.
2026-08-16 08:51:17 -05:00
parent bdc1f99fb9
commit e99a4640e2
2 changed files with 207 additions and 0 deletions
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# run_vindex_concurrency_tests.sh — regression harness for the 2026-08-16 soul crash.
+#
+# Runs two halves. The PAIR is the point: the ASan control localises the defect to
+# concurrency rather than to HNSW logic. See test_vindex_concurrency.c for the full
+# story (SIGSEGV at ASCII address "gramNode", heap corruption in xzm_realloc, etc).
+#
+#   1. single     — ASan+UBSan, one thread. MUST be clean. Always a hard failure.
+#   2. concurrent — TSan, writer + reader on one index. Currently EXPECTED to report a
+#                   race at visited_reset, because VIndex still owns its visited[] +
+#                   visit_epoch scratch. Once that moves to a per-query checkout pool
+#                   this must go clean; flip EXPECT_RACE=0 then and it becomes a real
+#                   regression gate.
+#
+# usage: run_vindex_concurrency_tests.sh
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RUNTIME="$(cd "$HERE/../../lang/runtime" && pwd)"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+SRC="$HERE/test_vindex_concurrency.c"
+VINDEX="$RUNTIME/engram_vindex.c"
+
+# Flip to 0 once the visited set is per-query; the concurrent half then becomes a gate.
+EXPECT_RACE="${EXPECT_RACE:-1}"
+
+fail=0
+
+echo "== [1/2] single-threaded control under AddressSanitizer =="
+cc -std=c11 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \
+   -I"$RUNTIME" -o "$WORK/single" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
+if ASAN_OPTIONS=detect_leaks=0 "$WORK/single" single; then
+  echo "   -> OK"
+else
+  echo "   -> FAIL: the single-threaded control must always be clean."
+  echo "      If this fails the bug is NOT (only) concurrency — look for a real"
+  echo "      out-of-bounds or lifetime error in engram_vindex.c."
+  fail=1
+fi
+
+echo
+echo "== [2/2] concurrent writer+reader under ThreadSanitizer =="
+cc -std=c11 -g -O1 -fsanitize=thread -fno-omit-frame-pointer \
+   -I"$RUNTIME" -o "$WORK/conc" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
+
+tsan_log="$WORK/tsan.log"
+TSAN_OPTIONS="halt_on_error=0" "$WORK/conc" concurrent >"$tsan_log" 2>&1
+if grep -q "ThreadSanitizer: data race" "$tsan_log"; then
+  echo "   -> RACE DETECTED:"
+  grep -m1 -A6 "ThreadSanitizer: data race" "$tsan_log" | sed 's/^/      /'
+  if [ "$EXPECT_RACE" = "1" ]; then
+    echo "   -> EXPECTED (VIndex still owns the shared visited set). Not a failure yet."
+    echo "      Fix = per-query visited buffer (hnswlib VisitedListPool style), then"
+    echo "      re-run with EXPECT_RACE=0."
+  else
+    echo "   -> REGRESSION: the visited set was supposed to be per-query."
+    fail=1
+  fi
+else
+  echo "   -> clean"
+  if [ "$EXPECT_RACE" = "1" ]; then
+    echo "   -> NOTE: no race reported, but EXPECT_RACE=1. Either the fix landed"
+    echo "      (set EXPECT_RACE=0) or the test did not actually interleave."
+  fi
+fi
+
+echo
+[ "$fail" -eq 0 ] && echo "RESULT: PASS" || echo "RESULT: FAIL"
+exit "$fail"
@@ -0,0 +1,136 @@
+/* test_vindex_concurrency.c — regression test for the 2026-08-16 soul crash.
+ *
+ * WHAT BROKE: the soul daemon crash-looped (5 crashes in ~100s) with SIGSEGV in
+ * search_layer <- vindex_insert <- eg_vindex_sync, a SIGABRT, and a fault inside
+ * xzm_realloc's own freelist — i.e. heap corruption. The SIGSEGV address
+ * 0x65646f4e6d617267 is little-endian ASCII "gramNode": string bytes being
+ * dereferenced as an Elem vector pointer.
+ *
+ * ROOT CAUSE: VIndex owns its traversal scratch (visited[] + visit_epoch), and
+ * search_layer mutates it via visited_reset(). So the index is unsafe for ANY
+ * concurrent use — including two concurrent READS. soul.el starts http_serve_async
+ * (a thread per connection) and then runs awareness_run() on the main thread, which
+ * reaches the same global index through engram_activate; nothing serialized them.
+ *
+ * Neither hnswlib nor FAISS puts the visited set on the index: hnswlib checks one
+ * out of a VisitedListPool per query, FAISS uses a thread_local VisitedTable.
+ *
+ * THIS TEST HAS TWO HALVES, and they must BOTH be run — the pair is what localises
+ * the bug to concurrency rather than to HNSW logic:
+ *
+ *   single   Insert N clustered vectors on ONE thread and search. Build with ASan.
+ *            This is the CONTROL. It must stay clean. When this passes and `concurrent`
+ *            fails, the defect is concurrency, not an out-of-bounds/logic error in the
+ *            graph code. (On 2026-08-16 this control cleared all 13,820 real dim-768
+ *            store vectors under ASan, which DISPROVED an inspection-derived hypothesis
+ *            about an out-of-bounds reverse-link write at engram_vindex.c:340.)
+ *
+ *   concurrent  One writer thread inserting while a reader thread searches the SAME
+ *            index. Build with TSan. Until the visited set moves off the index struct
+ *            this is EXPECTED TO REPORT A RACE at engram_vindex.c visited_reset —
+ *            that is the bug, reproduced. Once a per-query visited buffer lands
+ *            (see backlog: "Move VIndex visited-set off the index struct"), this must
+ *            become clean, and THAT is the regression this file guards.
+ *
+ * Absence of a crash does NOT mean absence of a race — always read the sanitizer
+ * verdict, never just the exit code.
+ *
+ * Build/run: engram/test/run_vindex_concurrency_tests.sh
+ */
+#include "engram_vindex.h"
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#define DIM   128
+#define NVEC  3000
+#define SEED_N 50
+
+static VIndex* g_ix;
+static float*  g_vecs;
+
+/* Deterministic filler. Real embeddings are strongly correlated, not uniform noise;
+ * clustering keeps many candidates near-equidistant, which exercises the diversity
+ * heuristic and the visited set far harder than random vectors do. */
+static void fill_vectors(void) {
+    g_vecs = (float*)malloc((size_t)NVEC * DIM * sizeof(float));
+    if (!g_vecs) { fprintf(stderr, "OOM\n"); exit(1); }
+    for (int i = 0; i < NVEC; i++) {
+        int cluster = i % 8;
+        for (int d = 0; d < DIM; d++)
+            g_vecs[(size_t)i * DIM + d] =
+                (float)(((d + cluster * 7) % 13) / 13.0) +
+                (float)(((i * 2654435761u + (unsigned)d) % 97) / 9700.0);
+    }
+}
+
+static void* writer_fn(void* arg) {
+    (void)arg;
+    for (int i = SEED_N; i < NVEC; i++)
+        (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
+    return NULL;
+}
+
+static void* reader_fn(void* arg) {
+    (void)arg;
+    uint64_t ids[8]; float ds[8];
+    for (int i = 0; i < 20000; i++)
+        (void)vindex_search(g_ix, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds);
+    return NULL;
+}
+
+static int run_single(void) {
+    printf("[single] inserting %d vectors on one thread (ASan control)\n", NVEC);
+    g_ix = vindex_create(DIM, 0, 0);
+    if (!g_ix) { fprintf(stderr, "[single] vindex_create failed\n"); return 1; }
+    for (int i = 0; i < NVEC; i++) {
+        if (vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM) != 0) {
+            fprintf(stderr, "[single] insert %d failed\n", i); return 1;
+        }
+    }
+    if (vindex_size(g_ix) != (size_t)NVEC) {
+        fprintf(stderr, "[single] size %zu != %d\n", vindex_size(g_ix), NVEC); return 1;
+    }
+    uint64_t ids[16]; float ds[16];
+    for (int q = 0; q < 200; q++) {
+        int k = vindex_search(g_ix, g_vecs + (size_t)((q * 7) % NVEC) * DIM, 16, 0, ids, ds);
+        if (k < 0) { fprintf(stderr, "[single] search failed at q=%d\n", q); return 1; }
+    }
+    vindex_free(g_ix); g_ix = NULL;
+    printf("[single] PASS — no memory error (this must ALWAYS pass)\n");
+    return 0;
+}
+
+static int run_concurrent(void) {
+    printf("[concurrent] 1 writer + 1 reader on ONE shared index (TSan probe)\n");
+    g_ix = vindex_create(DIM, 0, 0);
+    if (!g_ix) { fprintf(stderr, "[concurrent] vindex_create failed\n"); return 1; }
+    for (int i = 0; i < SEED_N; i++)
+        (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
+
+    pthread_t w, r;
+    if (pthread_create(&w, NULL, writer_fn, NULL) ||
+        pthread_create(&r, NULL, reader_fn, NULL)) {
+        fprintf(stderr, "[concurrent] pthread_create failed\n"); return 1;
+    }
+    pthread_join(w, NULL);
+    pthread_join(r, NULL);
+    vindex_free(g_ix); g_ix = NULL;
+    printf("[concurrent] completed — CHECK THE SANITIZER VERDICT, not this line.\n");
+    printf("[concurrent] a clean TSan run here is the actual pass condition.\n");
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    const char* mode = (argc > 1) ? argv[1] : "single";
+    fill_vectors();
+    int rc;
+    if      (!strcmp(mode, "single"))     rc = run_single();
+    else if (!strcmp(mode, "concurrent")) rc = run_concurrent();
+    else { fprintf(stderr, "usage: %s [single|concurrent]\n", argv[0]); rc = 2; }
+    free(g_vecs);
+    return rc;
+}