bigmerge
e99a4640e2
Promotes the two throwaway sanitizer harnesses used to diagnose the
2026-08-16 soul crash into engram/test/ so the bug cannot silently regress.
The harness has two halves and the PAIR is the point — it is what localises
the defect to concurrency rather than to HNSW logic:
single 3000 clustered vectors, one thread, ASan+UBSan. The CONTROL.
Must always be clean. During diagnosis this cleared all 13,820
real dim-768 vectors from the live store, which DISPROVED an
inspection-derived hypothesis about an out-of-bounds
reverse-link write at engram_vindex.c:340.
concurrent writer + reader on one shared index, TSan. Currently reports a
race at engram_vindex.c:195 (visited_reset) reached from both
vindex_search and vindex_insert, because VIndex still owns its
visited[]/visit_epoch scratch — so even two concurrent READS
corrupt each other's traversal.
Verified: half 1 passes, half 2 reproduces the race.
Gated on EXPECT_RACE, default 1, so the concurrent half documents the known
defect without failing the suite today. When the visited set moves to a
per-query checkout pool (hnswlib VisitedListPool style — NOT thread_local,
since http_worker is a thread per connection and a __thread buffer would leak
~55KB per connection), flip EXPECT_RACE=0 and it becomes a real gate.
72 lines
2.9 KiB
Bash
Executable File
72 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# run_vindex_concurrency_tests.sh — regression harness for the 2026-08-16 soul crash.
|
|
#
|
|
# Runs two halves. The PAIR is the point: the ASan control localises the defect to
|
|
# concurrency rather than to HNSW logic. See test_vindex_concurrency.c for the full
|
|
# story (SIGSEGV at ASCII address "gramNode", heap corruption in xzm_realloc, etc).
|
|
#
|
|
# 1. single — ASan+UBSan, one thread. MUST be clean. Always a hard failure.
|
|
# 2. concurrent — TSan, writer + reader on one index. Currently EXPECTED to report a
|
|
# race at visited_reset, because VIndex still owns its visited[] +
|
|
# visit_epoch scratch. Once that moves to a per-query checkout pool
|
|
# this must go clean; flip EXPECT_RACE=0 then and it becomes a real
|
|
# regression gate.
|
|
#
|
|
# usage: run_vindex_concurrency_tests.sh
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
RUNTIME="$(cd "$HERE/../../lang/runtime" && pwd)"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
SRC="$HERE/test_vindex_concurrency.c"
|
|
VINDEX="$RUNTIME/engram_vindex.c"
|
|
|
|
# Flip to 0 once the visited set is per-query; the concurrent half then becomes a gate.
|
|
EXPECT_RACE="${EXPECT_RACE:-1}"
|
|
|
|
fail=0
|
|
|
|
echo "== [1/2] single-threaded control under AddressSanitizer =="
|
|
cc -std=c11 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \
|
|
-I"$RUNTIME" -o "$WORK/single" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
|
|
if ASAN_OPTIONS=detect_leaks=0 "$WORK/single" single; then
|
|
echo " -> OK"
|
|
else
|
|
echo " -> FAIL: the single-threaded control must always be clean."
|
|
echo " If this fails the bug is NOT (only) concurrency — look for a real"
|
|
echo " out-of-bounds or lifetime error in engram_vindex.c."
|
|
fail=1
|
|
fi
|
|
|
|
echo
|
|
echo "== [2/2] concurrent writer+reader under ThreadSanitizer =="
|
|
cc -std=c11 -g -O1 -fsanitize=thread -fno-omit-frame-pointer \
|
|
-I"$RUNTIME" -o "$WORK/conc" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
|
|
|
|
tsan_log="$WORK/tsan.log"
|
|
TSAN_OPTIONS="halt_on_error=0" "$WORK/conc" concurrent >"$tsan_log" 2>&1
|
|
if grep -q "ThreadSanitizer: data race" "$tsan_log"; then
|
|
echo " -> RACE DETECTED:"
|
|
grep -m1 -A6 "ThreadSanitizer: data race" "$tsan_log" | sed 's/^/ /'
|
|
if [ "$EXPECT_RACE" = "1" ]; then
|
|
echo " -> EXPECTED (VIndex still owns the shared visited set). Not a failure yet."
|
|
echo " Fix = per-query visited buffer (hnswlib VisitedListPool style), then"
|
|
echo " re-run with EXPECT_RACE=0."
|
|
else
|
|
echo " -> REGRESSION: the visited set was supposed to be per-query."
|
|
fail=1
|
|
fi
|
|
else
|
|
echo " -> clean"
|
|
if [ "$EXPECT_RACE" = "1" ]; then
|
|
echo " -> NOTE: no race reported, but EXPECT_RACE=1. Either the fix landed"
|
|
echo " (set EXPECT_RACE=0) or the test did not actually interleave."
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
[ "$fail" -eq 0 ] && echo "RESULT: PASS" || echo "RESULT: FAIL"
|
|
exit "$fail"
|