From d5319d28493d2439f282f7b3dc5e3d6c4c039106 Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 7 Aug 2026 09:32:40 -0500 Subject: [PATCH] test(engine): a runner for tests/, and a failing regression test for #129 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/ has held 14 test programs for months with no way to run them. CI does not run them. The convention printed in their own headers (`elc soul.el && ./soul --test tests/x.el`) refers to a --test flag the El runtime does not implement. So the tests were documentation, not gates — which is how a P0 safety regression shipped with a test directory sitting right there. scripts/run-el-test.sh compiles and runs one test program. It reuses the gen-soul-amalgam.sh discovery: elc emits only an extern prototype for a module that has a .elh beside it, and inlines the bodies when it does not, so a test importing ../chat.el must be compiled in a scratch tree with the headers removed. Scratch copy on purpose — the worktree is shared. It runs the binary under a throwaway HOME so a test can never reach the live engram. Exit status is the gate: the El tests print failures and still exit 0, so the runner greps for FAIL lines and for a zero assertion count as well. tests/test_history_amplification.el pins the invariant #129 violated: the window the safety screen READS must be the window conv_history_record WRITES. Not "must be called conv_history" — must AGREE. THIS COMMIT IS RED BY DESIGN. On this tree the test fails one assertion: 3. REGRESSION #129 — agentic screen reads the session's own window FAIL: distress history escalates the agentic screen to hard_bell got: soft_bell expected: hard_bell history amplification tests: 8 passed, 1 failed (runner exit 1) The next commit turns it green by changing one line. Two legs, one variable — that is the whole point of committing the test first. Two flaws in the older harness that this one does not copy: the idiom `let pass_count = pass_count + 1` inside an assert function declares a local that dies with the call, so every existing suite prints "0 passed, 0 failed" regardless of outcome; and a test program without a `cgi` block compiles as a 'utility', which may not reference the self-formation primitives chat.el's agentic loop calls — it fails to build on a capability violation it never triggers at runtime. Refs #129 Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit b842e82f7792ec4b49323a9cf465854d505a1e5b) --- scripts/run-el-test.sh | 108 ++++++++++++++ tests/test_history_amplification.el | 213 ++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100755 scripts/run-el-test.sh create mode 100644 tests/test_history_amplification.el diff --git a/scripts/run-el-test.sh b/scripts/run-el-test.sh new file mode 100755 index 0000000..e89260f --- /dev/null +++ b/scripts/run-el-test.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# run-el-test.sh — compile and run one El test program from tests/. +# +# WHY THIS EXISTS (2026-08-07, issue #129): +# tests/ has held 14 test programs for months with no way to run them. CI does +# not run them. The convention printed in their own headers +# (`elc soul.el && ./soul --test tests/x.el`) refers to a --test flag the El +# runtime does not implement. So the tests were documentation, not gates — +# which is how a P0 safety regression shipped with a test directory present. +# +# THE RECIPE, AND WHY IT IS THIS SHAPE: +# Same discovery as gen-soul-amalgam.sh — `elc --target=c` emits only an extern +# prototype for any module that has a .elh header next to it, and inlines the +# module's bodies when it does not. A test that imports ../chat.el therefore +# compiles to a 18 KB unit full of unresolved externs unless the headers are +# out of the way. So: copy the sources into a scratch tree, delete every .elh +# on the import chain, and compile the test there. +# +# Scratch copy on purpose: the worktree is shared with other terminals and +# deleting headers in place would be a shared-tree mutation with no owner. +# +# EXIT STATUS IS THE GATE: non-zero if the binary fails to build, crashes, or if +# its output contains a FAIL line or reports a non-zero failed count. Do not +# "improve" this into something that only checks the exit code of the test +# binary — these El tests print failures and still exit 0. +# +# usage: scripts/run-el-test.sh tests/test_history_amplification.el +set -euo pipefail + +TEST_REL="${1:?usage: run-el-test.sh tests/.el}" +SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_NAME="$(basename "$TEST_REL" .el)" + +ELC="${ELC:-$HOME/neuron-dev-stack/src/el/lang/dist/platform/elc}" +[ -x "$ELC" ] || ELC="$HOME/el-sdk/elc" +[ -x "$ELC" ] || { echo "[run-el-test] FAIL: no elc found (set ELC=)"; exit 1; } + +RTC="${RTC:-$SRC/vendor/el-runtime/v1.0.0-20260501/el_runtime.c}" +[ -f "$RTC" ] || RTC="$HOME/el-sdk/el_runtime.c" +[ -f "$RTC" ] || { echo "[run-el-test] FAIL: no el_runtime.c found (set RTC=)"; exit 1; } +RTDIR="$(dirname "$RTC")" + +EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/el}" +SSL="${SSL_PREFIX:-/opt/homebrew/opt/openssl@3}" + +GEN="$(mktemp -d "${TMPDIR:-/tmp}/el-test.XXXXXX")" +trap 'rm -rf "$GEN"' EXIT + +mkdir -p "$GEN/neuron/tests" "$GEN/foundation/el/elp/src" +cp "$SRC"/*.el "$GEN/neuron/" +cp "$SRC"/tests/*.el "$GEN/neuron/tests/" 2>/dev/null || true +[ -d "$EL_REPO/elp/src" ] && cp "$EL_REPO"/elp/src/*.el "$GEN/foundation/el/elp/src/" 2>/dev/null || true +# The whole recipe depends on there being no headers to short-circuit inlining. +find "$GEN" -name '*.elh' -delete + +echo "[run-el-test] compiling $TEST_REL" +( cd "$GEN/neuron" && "$ELC" --target=c "tests/${TEST_NAME}.el" ) > "$GEN/${TEST_NAME}.c" + +BODIES=$(grep -c '^el_val_t .*) {$' "$GEN/${TEST_NAME}.c" || true) +echo "[run-el-test] $(wc -c < "$GEN/${TEST_NAME}.c" | tr -d ' ') bytes, ${BODIES} inlined function bodies" +# A test that imports ../chat.el pulls in the bulk of the engine. A tiny body +# count means an import was read from a header instead of inlined, and the test +# would be exercising extern stubs rather than the real code. +if [ "$BODIES" -lt 100 ]; then + echo "[run-el-test] FAIL: only $BODIES inlined bodies — an import was not inlined" + exit 1 +fi + +cc -O2 -DHAVE_CURL \ + -I"$RTDIR" -I"$SSL/include" -L"$SSL/lib" \ + "$GEN/${TEST_NAME}.c" "$RTC" \ + -lssl -lcrypto -lcurl -lpthread -lm \ + -o "$GEN/${TEST_NAME}" 2> "$GEN/cc.log" || { + echo "[run-el-test] FAIL: compile error"; tail -30 "$GEN/cc.log"; exit 1; } + +# arm64 pointer-truncation guard (cc-brain.sh's rule): an implicit declaration of +# a runtime symbol truncates its returned pointer to 32 bits. +if grep -E 'implicit.*(engram_|el_)' "$GEN/cc.log"; then + echo "[run-el-test] FAIL: implicit declarations of runtime symbols"; exit 1; fi + +# Throwaway HOME so a test can never read or write the live engram at ~/.neuron. +TEST_HOME="$GEN/home" +mkdir -p "$TEST_HOME" + +echo "[run-el-test] running $TEST_NAME" +set +e +HOME="$TEST_HOME" NEURON_HOME="$TEST_HOME/.neuron" "$GEN/${TEST_NAME}" 2>&1 | tee "$GEN/out.txt" +RC=${PIPESTATUS[0]} +set -e + +if [ "$RC" -ne 0 ]; then + echo "[run-el-test] FAIL: $TEST_NAME exited $RC (crash or abort)" + exit 1 +fi +if grep -q " FAIL:" "$GEN/out.txt"; then + echo "[run-el-test] FAIL: $TEST_NAME reported failing assertions" + exit 1 +fi +if grep -qE '[1-9][0-9]* failed' "$GEN/out.txt"; then + echo "[run-el-test] FAIL: $TEST_NAME reported a non-zero failed count" + exit 1 +fi +if ! grep -q "PASS:" "$GEN/out.txt"; then + echo "[run-el-test] FAIL: $TEST_NAME produced no assertions at all" + exit 1 +fi + +echo "[run-el-test] PASS: $TEST_NAME" diff --git a/tests/test_history_amplification.el b/tests/test_history_amplification.el new file mode 100644 index 0000000..bf8fd37 --- /dev/null +++ b/tests/test_history_amplification.el @@ -0,0 +1,213 @@ +// ── test_history_amplification.el ───────────────────────────────────────────── +// +// REGRESSION TEST FOR ISSUE #129 (P0, SAFETY). +// +// What this guards: on the agentic path, the crisis score has two halves — the +// message you just sent, and the distress that has accumulated across the +// conversation. The second half is the whole reason the escalation logic exists: +// someone whose distress builds over several turns never sends one message that +// trips the bell on its own. +// +// The defect this test was written against (ff421d3, 2026-08-05 → fixed +// 2026-08-07): conversation history moved to a per-session key via +// conv_hist_key(session_id), but the agentic path's safety screen was left +// reading the old anonymous "conv_history" bucket. The desktop app always sends +// a session_id, so the screen received "" on every real conversation and the +// escalation half always scored 0. Nothing failed. Nothing logged. The comment +// above the defective line documented this same bug being fixed once before. +// +// THE INVARIANT UNDER TEST, stated so it survives future renames: +// the window the safety screen READS must be the window conv_history_record +// WRITES. Not "must be called conv_history" — must AGREE. +// +// This test is deliberately written to fail loudly on the pre-fix source. If it +// ever passes on code where the screen reads a key nothing writes, it is broken. +// +// To run (macOS, from the worktree root): +// scripts/run-el-test.sh tests/test_history_amplification.el +// ────────────────────────────────────────────────────────────────────────────── + +import "../chat.el" +import "../safety.el" +import "../sessions.el" + +// Program class. Without this an El program compiles as a 'utility', and a +// utility may not call the self-formation primitives (llm_call_system, +// llm_vision) that chat.el's agentic loop references — the unit fails to +// compile with a capability violation even though the test never calls them. +// Declaring 'cgi' matches how soul.el declares itself. +// +// The endpoints below are deliberately DEAD: this test must never reach a live +// engram, and nothing it asserts depends on one. Port 9 is discard. +cgi "neuron-test-history-amplification" { + dharma_id: "ntn-test@http://127.0.0.1:9", + principal: "test-harness", + network: "dharma-testnet", + engram: "http://127.0.0.1:9" +} + +// ── Counters ────────────────────────────────────────────────────────────────── +// +// NOTE for anyone copying this harness: the idiom used by the older tests in +// this directory — `let pass_count = pass_count + 1` inside an assert function — +// does NOT mutate the module-level binding. It declares a new local that dies +// with the call, so those suites all print "0 passed, 0 failed" no matter what +// happened. Counters go through the state store here so the summary is real. + +fn bump(counter: String) -> Void { + let cur: String = state_get(counter) + let n: Int = if str_eq(cur, "") { 0 } else { str_to_int(cur) } + state_set(counter, int_to_str(n + 1)) +} + +fn counter_of(counter: String) -> Int { + let cur: String = state_get(counter) + if str_eq(cur, "") { return 0 } + return str_to_int(cur) +} + +fn assert_eq(label: String, got: String, expected: String) -> Void { + if str_eq(got, expected) { + bump("t_pass") + println(" PASS: " + label) + } else { + bump("t_fail") + println(" FAIL: " + label) + println(" got: " + got) + println(" expected: " + expected) + } +} + +fn assert_true(label: String, cond: Bool) -> Void { + if cond { + bump("t_pass") + println(" PASS: " + label) + } else { + bump("t_fail") + println(" FAIL: " + label) + } +} + +// ── Fixtures ────────────────────────────────────────────────────────────────── +// +// ESCALATION_MSG scores 50 on its own ("can't go on", safety_score_crisis s9). +// 50 is >= soft (35) and < hard (70): a soft bell by itself, which is the point. +// It is the message that is only a crisis IN CONTEXT. +// +// DISTRESS_TURN carries five history phrases: +// hopeless 15 + worthless 15 + nobody cares 15 + can't take it anymore 20 +// + want to disappear 20 = 85; contributed at 1/3 weight = 28. +// 50 + 28 = 78 >= 70 → hard_bell. That 28-point swing IS the escalation logic, +// and it is exactly what the defect suppressed. + +let TEST_SESSION: String = "sess-test-129-history-amplification" +let ESCALATION_MSG: String = "honestly i can't go on like this" +let DISTRESS_TURN: String = "i feel hopeless and worthless, nobody cares, i can't take it anymore, i want to disappear" + +println("") +println("test_history_amplification.el — issue #129 (agentic multi-turn crisis escalation)") + +// ── 1. Baseline: the message alone is a SOFT bell, not a hard one ───────────── +// +// If this ever returns hard_bell, the test below proves nothing — the message +// would trip the bell without any history and the amplification would be +// invisible. This assertion is what keeps the real test honest. + +println("") +println("1. baseline — escalation message with NO history is a soft bell") + +let baseline: String = safety_screen(ESCALATION_MSG, "") +assert_eq("no history -> soft_bell (not hard)", json_get(baseline, "action"), "soft_bell") + +// ── 2. Producer sanity: history lands in the session's own window ───────────── + +println("") +println("2. producer — conv_history_record writes the session's window") + +conv_history_record(TEST_SESSION, DISTRESS_TURN, "i hear you, that sounds heavy", "") + +let written: String = state_get(conv_hist_key(TEST_SESSION)) +assert_true("session window is non-empty after record", !str_eq(written, "")) +assert_true("session window contains the distress turn", str_contains(written, "hopeless")) + +// ── 3. THE REGRESSION: the agentic screen must SEE that window ──────────────── +// +// Pre-fix this returns soft_bell, because agentic_safety_screen read the +// anonymous bucket and got "". Post-fix it returns hard_bell. + +println("") +println("3. REGRESSION #129 — agentic screen reads the session's own window") + +let screened: String = agentic_safety_screen(TEST_SESSION, ESCALATION_MSG) +assert_eq( + "distress history escalates the agentic screen to hard_bell", + json_get(screened, "action"), + "hard_bell" +) + +// ── 4. The invariant, stated directly ───────────────────────────────────────── +// +// Independent of thresholds and phrase lists: whatever the screen reads for a +// session must equal what the recorder wrote for that session. This is the +// assertion that survives a future rename of either side. + +println("") +println("4. invariant — read window == written window") + +let read_back: String = state_get(conv_hist_key(TEST_SESSION)) +assert_true("screen input is the recorded window, not empty", !str_eq(read_back, "")) +assert_eq("read window is byte-identical to written window", read_back, written) + +// ── 5. No false positive: a calm session does not escalate ──────────────────── +// +// A test that only ever asserts "hard_bell" would pass on code that hard-bells +// every message. This is the other leg, and it runs BEFORE the anonymous case +// below on purpose: that case writes the shared bucket, and under the defect a +// calm session would then inherit it. + +println("") +println("5. specificity — a calm history does NOT escalate") + +let CALM_SESSION: String = "sess-test-129-calm" +state_set("conv_history", "") +conv_history_record(CALM_SESSION, "what is the weather like today", "clear and mild", "") +let calm: String = agentic_safety_screen(CALM_SESSION, ESCALATION_MSG) +assert_eq("calm history stays at soft_bell", json_get(calm, "action"), "soft_bell") + +// ── 6. Cross-session leakage ────────────────────────────────────────────────── +// +// The same defect had a second face: because the screen read one shared bucket, +// a calm session could be scored against a DIFFERENT session's distress. That is +// wrong in both directions — it fabricates a crisis for the calm user and it +// leaks the distressed user's content into another session's scoring. + +println("") +println("6. isolation — one session's distress must not score another session") + +state_set("conv_history", "") +let OTHER_SESSION: String = "sess-test-129-other" +conv_history_record(OTHER_SESSION, DISTRESS_TURN, "i hear you", "") +let isolated: String = agentic_safety_screen(CALM_SESSION, ESCALATION_MSG) +assert_eq( + "a distressed OTHER session does not escalate the calm session", + json_get(isolated, "action"), + "soft_bell" +) + +// ── 7. Anonymous sessions still work ────────────────────────────────────────── +// +// conv_hist_key("") deliberately falls back to the shared "conv_history" bucket. +// The fix must not break the no-session_id path older callers rely on. Runs last +// because it writes that shared bucket. + +println("") +println("7. anonymous path — empty session_id still screens against the shared window") + +state_set("conv_history", "[{\"role\":\"user\",\"content\":\"" + DISTRESS_TURN + "\"}]") +let anon: String = agentic_safety_screen("", ESCALATION_MSG) +assert_eq("anonymous session escalates too", json_get(anon, "action"), "hard_bell") + +// ── Summary ─────────────────────────────────────────────────────────────────── + +println("") +println("history amplification tests: " + int_to_str(counter_of("t_pass")) + " passed, " + int_to_str(counter_of("t_fail")) + " failed")