diff --git a/engram/test/run_http_sigpipe_test.sh b/engram/test/run_http_sigpipe_test.sh new file mode 100755 index 0000000..426f0ca --- /dev/null +++ b/engram/test/run_http_sigpipe_test.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Build + RUN the SIGPIPE integration probe: a client that hangs up mid-response +# must not kill the server. Pure C11, no Python, no third-party anything. +# +# This is an INTEGRATION probe — it needs a running engram, and it needs the PID +# so it can tell "still serving" from "restarted by a supervisor underneath me". +# Point it at a SCRATCH instance, never at production: +# +# cp -Rc ~/.neuron/engram /tmp/engram-scratch # APFS clone, instant +# EL_SINGLETON_DIR=/tmp ENGRAM_DATA_DIR=/tmp/engram-scratch \ +# ENGRAM_STORE=1 ENGRAM_API_KEY=ntn-user-2026 ENGRAM_BIND=":18753" ./engram & +# lsof -nP -iTCP:18753 -sTCP:LISTEN # confirm YOUR pid owns the port +# ./engram/test/run_http_sigpipe_test.sh 18753 +# +# NEGATIVE CONTROL (invariant §8.6 — no test without one). This probe was shown +# to FAIL on the pre-change build before the fix was accepted. Measured, same +# data, same endpoint, same probe: +# unpatched -> SERVER DIED 43.3s after the hang-up (exit 1) +# patched -> survived 2 rounds, still listening, still serving (exit 0) +# To reproduce the failing side, build the runtime at the parent commit and run +# this against it. +set -e +HERE=$(cd "$(dirname "$0")" && pwd) +CC=${CC:-cc} +PORT=${1:?usage: run_http_sigpipe_test.sh [rounds] [settle_seconds]} +PID=${2:?usage: run_http_sigpipe_test.sh [rounds] [settle_seconds]} +ROUNDS=${3:-2} +SETTLE=${4:-45} +TMP=$(mktemp -d) + +$CC -std=c11 -Wall -Wextra -O2 "$HERE/test_http_sigpipe.c" -o "$TMP/probe" +"$TMP/probe" "$PORT" "$PID" "$ROUNDS" "$SETTLE" diff --git a/engram/test/test_http_sigpipe.c b/engram/test/test_http_sigpipe.c new file mode 100644 index 0000000..1b4f458 --- /dev/null +++ b/engram/test/test_http_sigpipe.c @@ -0,0 +1,119 @@ +/* test_http_sigpipe.c — a client that hangs up mid-response must not kill the + * server. Integration probe: point it at a running engram. + * + * WHAT THIS REPRODUCES. `send()` to a socket whose peer has closed raises + * SIGPIPE, whose default disposition is terminate. Nothing in the runtime + * suppressed it, so any abandoned request could kill the process. In production + * the abandoning client was the heartbeat: ai.neuron.engram-tick.plist runs on + * StartInterval 600 and calls `curl -s -m10 -X POST /api/tick`; when the endpoint + * exceeded ten seconds curl hung up, and the eventual response write killed the + * server. 254 restarts between 2026-08-13T19:37 and 2026-08-16T18:16, with + * `launchctl list ai.neuron.engram` reporting LastExitStatus = 13 — raw wait + * status 13, killed by signal 13, SIGPIPE. + * + * TWO THINGS THIS PROBE GETS RIGHT, both of which a naive version gets wrong and + * both of which cost me a false PASS on the unpatched build before I caught them: + * + * 1. CLOSE MODE. Closing with SO_LINGER=0 emits RST, and the server's FIRST + * write then returns ECONNRESET rather than raising SIGPIPE: http_send_all + * sees w <= 0, returns -1, and stops. No signal, no crash — the test passes + * on the UNPATCHED build and proves nothing. A NORMAL close (FIN) is what + * kills: the first write succeeds, the peer answers RST because it is fully + * closed, and the SECOND write raises SIGPIPE. A response is emitted as four + * http_send_all calls (status line, headers, tail, body), so the sequence is + * always reached. + * + * 2. TIMING. The kill does not land when the client closes. It lands when the + * server reaches its write, which on a heavy endpoint is many seconds later + * (~16s measured on /api/activate over a 13,632-node store). Checking + * liveness a second after the hang-up finds the process healthy and reports + * a false pass. + * + * Usage: test_http_sigpipe [rounds] [settle_seconds] + * Exit 0 = server survived every round (PASS), 1 = server died (FAIL), + * 2 = bad precondition. + */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The request is deliberately expensive: the body must be large enough and slow + * enough that the server is still working when the client walks away. */ +static const char* REQ_PATH = + "/api/activate?q=will%20anderson&depth=3&_auth=ntn-user-2026"; + +static int alive(pid_t pid) { return kill(pid, 0) == 0 || errno == EPERM; } + +static void sleep_ms(long ms) { + struct timespec ts = { ms / 1000, (ms % 1000) * 1000000L }; + nanosleep(&ts, NULL); +} + +/* Connect, ask for the expensive body, then hang up NORMALLY without reading. */ +static int abort_request(int port) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + struct sockaddr_in a; + memset(&a, 0, sizeof a); + a.sin_family = AF_INET; + a.sin_port = htons((uint16_t)port); + a.sin_addr.s_addr = inet_addr("127.0.0.1"); + if (connect(fd, (struct sockaddr*)&a, sizeof a) != 0) { close(fd); return -1; } + + char req[512]; + int n = snprintf(req, sizeof req, + "GET %s HTTP/1.1\r\nHost: probe\r\nConnection: close\r\nAccept: */*\r\n\r\n", + REQ_PATH); + if (write(fd, req, (size_t)n) != n) { close(fd); return -1; } + + sleep_ms(200); /* ensure the request landed and work has begun */ + close(fd); /* NORMAL close -> FIN. Never SO_LINGER=0 / RST. */ + return 0; +} + +int main(int argc, char** argv) { + if (argc < 3) { + fprintf(stderr, + "usage: %s [rounds=1] [settle_seconds=45]\n", argv[0]); + return 2; + } + int port = atoi(argv[1]); + pid_t pid = (pid_t)atoi(argv[2]); + int rounds = argc > 3 ? atoi(argv[3]) : 1; + int settle = argc > 4 ? atoi(argv[4]) : 45; + + if (!alive(pid)) { + printf(" PRECONDITION FAILED: pid %d is not running before the probe\n", pid); + return 2; + } + printf(" precondition: pid %d alive, target port %d\n", pid, port); + + for (int i = 1; i <= rounds; i++) { + if (abort_request(port) != 0) { + printf(" round %d: could not reach the server on port %d\n", i, port); + return 2; + } + printf(" round %d: client hung up; waiting up to %ds for the server to " + "reach its write...\n", i, settle); + for (int s = 0; s < settle * 2; s++) { + if (!alive(pid)) { + printf(" round %d: SERVER DIED %.1fs after the hang-up *** FAIL ***\n", + i, s / 2.0); + return 1; + } + sleep_ms(500); + } + printf(" round %d: server survived\n", i); + } + printf(" PASS: the server survived %d abandoned request(s)\n", rounds); + return 0; +} diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index a1cbce5..2c59d3c 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -1335,10 +1335,64 @@ static const char* http_reason_phrase(int status) { } } -/* Best-effort send with retry on partial writes. */ +/* ── SIGPIPE MUST NEVER REACH THE PROCESS (2026-08-16) ─────────────────────── + * + * A client that hangs up before we finish writing its response was KILLING THE + * SERVER. `send()` to a socket whose peer has closed raises SIGPIPE, whose + * default disposition is terminate, and nothing in this runtime suppressed it: + * `grep -rn "SIGPIPE\|MSG_NOSIGNAL\|SO_NOSIGPIPE\|sigaction" lang/runtime/` + * returned zero hits. + * + * MEASURED IN PRODUCTION. The engram restarted 254 times between 2026-08-13T19:37 + * and 2026-08-16T18:16 on a ~10 minute cadence, with `launchctl list + * ai.neuron.engram` reporting `LastExitStatus = 13` — raw wait status 13, i.e. + * killed by signal 13, SIGPIPE. The trigger was ai.neuron.engram-tick.plist + * (`StartInterval 600`, matching the cadence exactly) running + * `curl -s -m10 -X POST /api/tick`: when /api/tick exceeded curl's 10 second + * timeout the client hung up, and the response write then killed the process. + * The tick log recorded an EMPTY response — curl having returned nothing — in + * 279 of 448 ticks. launchd `KeepAlive: true` restarted it each time, so the + * loop was invisible except as a PID that kept changing. + * + * Nothing was ever logged about it, and could not have been: a signal-killed + * process never reaches a line where it could write one. ~/.neuron/logs/engram.log + * is 2.7 MB of nothing but repeated "[http] listening on [::]:8742" — that is the + * signature of this bug, not a logging gap. + * + * Note what was already correct below: the `w <= 0` branch handles a dead peer + * properly. It had simply never once executed, because the signal killed the + * process before send() could return -1/EPIPE. The error handling was written + * and unreachable. + * + * The suppression is deliberately PER-SOCKET / PER-CALL rather than a global + * signal(SIGPIPE, SIG_IGN). Both forms fix the crash; only this one has zero + * blast radius. A global handler would also change the disposition of writes to + * ordinary pipes in the exec/fs paths, which is a semantic change to code this + * fix has no business touching. SO_NOSIGPIPE (macOS/BSD) and MSG_NOSIGNAL + * (Linux) each affect only the socket and call sites named here, and each is a + * no-op on the platform that lacks it. Every HTTP response in the runtime funnels + * through this one function, so covering it covers all three accept loops. */ +#ifdef MSG_NOSIGNAL +# define EL_SEND_FLAGS MSG_NOSIGNAL /* Linux: suppress per send() call */ +#else +# define EL_SEND_FLAGS 0 +#endif +static void el_sock_nosigpipe(int fd) { +#ifdef SO_NOSIGPIPE /* macOS/BSD: suppress per socket */ + int on = 1; + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on)); +#else + (void)fd; +#endif +} + +/* Best-effort send with retry on partial writes. A peer that has gone away now + * surfaces as a -1 return (EPIPE) and drops the connection, instead of killing + * the server. */ static int http_send_all(int fd, const char* p, size_t left) { + el_sock_nosigpipe(fd); while (left > 0) { - ssize_t w = send(fd, p, left, 0); + ssize_t w = send(fd, p, left, EL_SEND_FLAGS); if (w <= 0) return -1; p += w; left -= (size_t)w; }