engram/test/run_http_sigpipe_test.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Build + RUN the SIGPIPE integration probe: a client that hangs up mid-response
+# must not kill the server. Pure C11, no Python, no third-party anything.
+#
+# This is an INTEGRATION probe — it needs a running engram, and it needs the PID
+# so it can tell "still serving" from "restarted by a supervisor underneath me".
+# Point it at a SCRATCH instance, never at production:
+#
+#   cp -Rc ~/.neuron/engram /tmp/engram-scratch          # APFS clone, instant
+#   EL_SINGLETON_DIR=/tmp ENGRAM_DATA_DIR=/tmp/engram-scratch \
+#     ENGRAM_STORE=1 ENGRAM_API_KEY=ntn-user-2026 ENGRAM_BIND=":18753" ./engram &
+#   lsof -nP -iTCP:18753 -sTCP:LISTEN     # confirm YOUR pid owns the port
+#   ./engram/test/run_http_sigpipe_test.sh 18753 <pid>
+#
+# NEGATIVE CONTROL (invariant §8.6 — no test without one). This probe was shown
+# to FAIL on the pre-change build before the fix was accepted. Measured, same
+# data, same endpoint, same probe:
+#     unpatched  -> SERVER DIED 43.3s after the hang-up   (exit 1)
+#     patched    -> survived 2 rounds, still listening, still serving (exit 0)
+# To reproduce the failing side, build the runtime at the parent commit and run
+# this against it.
+set -e
+HERE=$(cd "$(dirname "$0")" && pwd)
+CC=${CC:-cc}
+PORT=${1:?usage: run_http_sigpipe_test.sh <port> <pid> [rounds] [settle_seconds]}
+PID=${2:?usage: run_http_sigpipe_test.sh <port> <pid> [rounds] [settle_seconds]}
+ROUNDS=${3:-2}
+SETTLE=${4:-45}
+TMP=$(mktemp -d)
+
+$CC -std=c11 -Wall -Wextra -O2 "$HERE/test_http_sigpipe.c" -o "$TMP/probe"
+"$TMP/probe" "$PORT" "$PID" "$ROUNDS" "$SETTLE"

engram/test/test_http_sigpipe.c

@@ -0,0 +1,119 @@
+/* test_http_sigpipe.c — a client that hangs up mid-response must not kill the
+ * server. Integration probe: point it at a running engram.
+ *
+ * WHAT THIS REPRODUCES. `send()` to a socket whose peer has closed raises
+ * SIGPIPE, whose default disposition is terminate. Nothing in the runtime
+ * suppressed it, so any abandoned request could kill the process. In production
+ * the abandoning client was the heartbeat: ai.neuron.engram-tick.plist runs on
+ * StartInterval 600 and calls `curl -s -m10 -X POST /api/tick`; when the endpoint
+ * exceeded ten seconds curl hung up, and the eventual response write killed the
+ * server. 254 restarts between 2026-08-13T19:37 and 2026-08-16T18:16, with
+ * `launchctl list ai.neuron.engram` reporting LastExitStatus = 13 — raw wait
+ * status 13, killed by signal 13, SIGPIPE.
+ *
+ * TWO THINGS THIS PROBE GETS RIGHT, both of which a naive version gets wrong and
+ * both of which cost me a false PASS on the unpatched build before I caught them:
+ *
+ *   1. CLOSE MODE. Closing with SO_LINGER=0 emits RST, and the server's FIRST
+ *      write then returns ECONNRESET rather than raising SIGPIPE: http_send_all
+ *      sees w <= 0, returns -1, and stops. No signal, no crash — the test passes
+ *      on the UNPATCHED build and proves nothing. A NORMAL close (FIN) is what
+ *      kills: the first write succeeds, the peer answers RST because it is fully
+ *      closed, and the SECOND write raises SIGPIPE. A response is emitted as four
+ *      http_send_all calls (status line, headers, tail, body), so the sequence is
+ *      always reached.
+ *
+ *   2. TIMING. The kill does not land when the client closes. It lands when the
+ *      server reaches its write, which on a heavy endpoint is many seconds later
+ *      (~16s measured on /api/activate over a 13,632-node store). Checking
+ *      liveness a second after the hang-up finds the process healthy and reports
+ *      a false pass.
+ *
+ * Usage: test_http_sigpipe <port> <pid> [rounds] [settle_seconds]
+ * Exit 0 = server survived every round (PASS), 1 = server died (FAIL),
+ *      2 = bad precondition.
+ */
+#define _POSIX_C_SOURCE 200809L
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+/* The request is deliberately expensive: the body must be large enough and slow
+ * enough that the server is still working when the client walks away. */
+static const char* REQ_PATH =
+    "/api/activate?q=will%20anderson&depth=3&_auth=ntn-user-2026";
+
+static int alive(pid_t pid) { return kill(pid, 0) == 0 || errno == EPERM; }
+
+static void sleep_ms(long ms) {
+    struct timespec ts = { ms / 1000, (ms % 1000) * 1000000L };
+    nanosleep(&ts, NULL);
+}
+
+/* Connect, ask for the expensive body, then hang up NORMALLY without reading. */
+static int abort_request(int port) {
+    int fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd < 0) return -1;
+    struct sockaddr_in a;
+    memset(&a, 0, sizeof a);
+    a.sin_family = AF_INET;
+    a.sin_port   = htons((uint16_t)port);
+    a.sin_addr.s_addr = inet_addr("127.0.0.1");
+    if (connect(fd, (struct sockaddr*)&a, sizeof a) != 0) { close(fd); return -1; }
+
+    char req[512];
+    int n = snprintf(req, sizeof req,
+        "GET %s HTTP/1.1\r\nHost: probe\r\nConnection: close\r\nAccept: */*\r\n\r\n",
+        REQ_PATH);
+    if (write(fd, req, (size_t)n) != n) { close(fd); return -1; }
+
+    sleep_ms(200);        /* ensure the request landed and work has begun */
+    close(fd);            /* NORMAL close -> FIN. Never SO_LINGER=0 / RST. */
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    if (argc < 3) {
+        fprintf(stderr,
+            "usage: %s <port> <pid> [rounds=1] [settle_seconds=45]\n", argv[0]);
+        return 2;
+    }
+    int   port   = atoi(argv[1]);
+    pid_t pid    = (pid_t)atoi(argv[2]);
+    int   rounds = argc > 3 ? atoi(argv[3]) : 1;
+    int   settle = argc > 4 ? atoi(argv[4]) : 45;
+
+    if (!alive(pid)) {
+        printf("  PRECONDITION FAILED: pid %d is not running before the probe\n", pid);
+        return 2;
+    }
+    printf("  precondition: pid %d alive, target port %d\n", pid, port);
+
+    for (int i = 1; i <= rounds; i++) {
+        if (abort_request(port) != 0) {
+            printf("  round %d: could not reach the server on port %d\n", i, port);
+            return 2;
+        }
+        printf("  round %d: client hung up; waiting up to %ds for the server to "
+               "reach its write...\n", i, settle);
+        for (int s = 0; s < settle * 2; s++) {
+            if (!alive(pid)) {
+                printf("  round %d: SERVER DIED %.1fs after the hang-up  *** FAIL ***\n",
+                       i, s / 2.0);
+                return 1;
+            }
+            sleep_ms(500);
+        }
+        printf("  round %d: server survived\n", i);
+    }
+    printf("  PASS: the server survived %d abandoned request(s)\n", rounds);
+    return 0;
+}

lang/runtime/el_runtime.c

@@ -1335,10 +1335,64 @@ static const char* http_reason_phrase(int status) {
     }
 }
 
-/* Best-effort send with retry on partial writes. */
+/* ── SIGPIPE MUST NEVER REACH THE PROCESS (2026-08-16) ───────────────────────
+ *
+ * A client that hangs up before we finish writing its response was KILLING THE
+ * SERVER. `send()` to a socket whose peer has closed raises SIGPIPE, whose
+ * default disposition is terminate, and nothing in this runtime suppressed it:
+ * `grep -rn "SIGPIPE\|MSG_NOSIGNAL\|SO_NOSIGPIPE\|sigaction" lang/runtime/`
+ * returned zero hits.
+ *
+ * MEASURED IN PRODUCTION. The engram restarted 254 times between 2026-08-13T19:37
+ * and 2026-08-16T18:16 on a ~10 minute cadence, with `launchctl list
+ * ai.neuron.engram` reporting `LastExitStatus = 13` — raw wait status 13, i.e.
+ * killed by signal 13, SIGPIPE. The trigger was ai.neuron.engram-tick.plist
+ * (`StartInterval 600`, matching the cadence exactly) running
+ * `curl -s -m10 -X POST /api/tick`: when /api/tick exceeded curl's 10 second
+ * timeout the client hung up, and the response write then killed the process.
+ * The tick log recorded an EMPTY response — curl having returned nothing — in
+ * 279 of 448 ticks. launchd `KeepAlive: true` restarted it each time, so the
+ * loop was invisible except as a PID that kept changing.
+ *
+ * Nothing was ever logged about it, and could not have been: a signal-killed
+ * process never reaches a line where it could write one. ~/.neuron/logs/engram.log
+ * is 2.7 MB of nothing but repeated "[http] listening on [::]:8742" — that is the
+ * signature of this bug, not a logging gap.
+ *
+ * Note what was already correct below: the `w <= 0` branch handles a dead peer
+ * properly. It had simply never once executed, because the signal killed the
+ * process before send() could return -1/EPIPE. The error handling was written
+ * and unreachable.
+ *
+ * The suppression is deliberately PER-SOCKET / PER-CALL rather than a global
+ * signal(SIGPIPE, SIG_IGN). Both forms fix the crash; only this one has zero
+ * blast radius. A global handler would also change the disposition of writes to
+ * ordinary pipes in the exec/fs paths, which is a semantic change to code this
+ * fix has no business touching. SO_NOSIGPIPE (macOS/BSD) and MSG_NOSIGNAL
+ * (Linux) each affect only the socket and call sites named here, and each is a
+ * no-op on the platform that lacks it. Every HTTP response in the runtime funnels
+ * through this one function, so covering it covers all three accept loops. */
+#ifdef MSG_NOSIGNAL
+#  define EL_SEND_FLAGS MSG_NOSIGNAL      /* Linux: suppress per send() call     */
+#else
+#  define EL_SEND_FLAGS 0
+#endif
+static void el_sock_nosigpipe(int fd) {
+#ifdef SO_NOSIGPIPE                        /* macOS/BSD: suppress per socket      */
+    int on = 1;
+    setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on));
+#else
+    (void)fd;
+#endif
+}
+
+/* Best-effort send with retry on partial writes. A peer that has gone away now
+ * surfaces as a -1 return (EPIPE) and drops the connection, instead of killing
+ * the server. */
 static int http_send_all(int fd, const char* p, size_t left) {
+    el_sock_nosigpipe(fd);
     while (left > 0) {
-        ssize_t w = send(fd, p, left, 0);
+        ssize_t w = send(fd, p, left, EL_SEND_FLAGS);
         if (w <= 0) return -1;
         p += w; left -= (size_t)w;
     }