Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Neuron
|
cace6a5ebf | ||
 will.anderson
|
d41645388a | ||
|
Neuron
|
8a307dfd42 |
+168
-22
@@ -40,6 +40,7 @@
|
||||
#include <sys/stat.h>
|
||||
#include <netinet/in.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <signal.h> /* SIGPIPE disposition: a hung-up client must not kill us */
|
||||
#include <dlfcn.h> /* dlsym for http_set_handler fallback */
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
@@ -1335,10 +1336,63 @@ static const char* http_reason_phrase(int status) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Best-effort send with retry on partial writes. */
|
||||
/* A DISCONNECTING CLIENT MUST NOT KILL THE SERVER (2026-08-16).
|
||||
*
|
||||
* There was no SIGPIPE handling anywhere in this runtime: no signal disposition,
|
||||
* no MSG_NOSIGNAL, no SO_NOSIGPIPE, and send() called with bare flags. The
|
||||
* default disposition of SIGPIPE is to TERMINATE THE PROCESS, so any client that
|
||||
* hung up mid-response — a curl that hit its timeout, a browser tab closed
|
||||
* during a large read, a proxy giving up — took the whole engram down with it.
|
||||
*
|
||||
* Measured on the live instance: 18 boots in the log, and `launchctl list`
|
||||
* reporting the previous exit for ai.neuron.engram as -13, i.e. killed by
|
||||
* signal 13 = SIGPIPE. Reproduced by the cause: pulling /api/nodes/list (26 MB)
|
||||
* with a client-side timeout. launchd's KeepAlive then restarts it, so the
|
||||
* failure looks like a mysterious restart rather than a crash, and the graph
|
||||
* silently reloads under whatever was mid-flight.
|
||||
*
|
||||
* This is an exemption in the §8 sense: the write never checked whether the
|
||||
* peer was still there, and the consequence of not checking was fatal rather
|
||||
* than merely wrong.
|
||||
*
|
||||
* Two layers, because neither alone is portable:
|
||||
* - SO_NOSIGPIPE per socket (Darwin/BSD) and MSG_NOSIGNAL per send (Linux),
|
||||
* so the signal is never raised for socket writes in the first place.
|
||||
* - A process-wide SIG_IGN as the backstop for platforms/paths with neither,
|
||||
* installed once and idempotent. With the signal ignored, send() returns
|
||||
* -1/EPIPE and the existing error path closes the connection. */
|
||||
#ifndef MSG_NOSIGNAL
|
||||
#define MSG_NOSIGNAL 0
|
||||
#endif
|
||||
|
||||
static void el_ignore_sigpipe_once(void) {
|
||||
static int done = 0;
|
||||
if (done) return;
|
||||
done = 1;
|
||||
#ifndef _WIN32
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Per-socket suppression where the platform offers it. Best-effort: a failure
|
||||
* here is not fatal because el_ignore_sigpipe_once() already covers the case. */
|
||||
static void el_sock_nosigpipe(int fd) {
|
||||
#if defined(SO_NOSIGPIPE)
|
||||
int on = 1;
|
||||
setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on));
|
||||
#else
|
||||
(void)fd;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Best-effort send with retry on partial writes. EPIPE/ECONNRESET are a client
|
||||
* that left, not a server fault: return -1 so the caller closes the connection,
|
||||
* and never let it reach the process as a signal. */
|
||||
static int http_send_all(int fd, const char* p, size_t left) {
|
||||
el_ignore_sigpipe_once();
|
||||
while (left > 0) {
|
||||
ssize_t w = send(fd, p, left, 0);
|
||||
ssize_t w = send(fd, p, left, MSG_NOSIGNAL);
|
||||
if (w < 0 && errno == EINTR) continue;
|
||||
if (w <= 0) return -1;
|
||||
p += w; left -= (size_t)w;
|
||||
}
|
||||
@@ -1788,6 +1842,7 @@ void http_serve(el_val_t port, el_val_t handler) {
|
||||
pthread_mutex_unlock(&_http_conn_mu);
|
||||
HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg));
|
||||
if (!arg) { el_closesocket(cfd); continue; }
|
||||
el_sock_nosigpipe(cfd);
|
||||
arg->fd = cfd;
|
||||
pthread_t tid;
|
||||
if (pthread_create(&tid, NULL, http_worker, arg) != 0) {
|
||||
@@ -1834,6 +1889,7 @@ static void* _http_serve_async_loop(void* raw) {
|
||||
pthread_mutex_unlock(&_http_conn_mu);
|
||||
HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg));
|
||||
if (!arg) { close(cfd); continue; }
|
||||
el_sock_nosigpipe(cfd);
|
||||
arg->fd = cfd;
|
||||
pthread_t tid;
|
||||
if (pthread_create(&tid, NULL, http_worker, arg) != 0) {
|
||||
@@ -2134,6 +2190,7 @@ void http_serve_v2(el_val_t port, el_val_t handler) {
|
||||
pthread_mutex_unlock(&_http_conn_mu);
|
||||
HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg));
|
||||
if (!arg) { el_closesocket(cfd); continue; }
|
||||
el_sock_nosigpipe(cfd);
|
||||
arg->fd = cfd;
|
||||
pthread_t tid;
|
||||
if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) {
|
||||
@@ -3479,28 +3536,74 @@ static void jb_puts(JsonBuf* b, const char* s) {
|
||||
b->buf[b->len] = '\0';
|
||||
}
|
||||
|
||||
/* UTF-8 VALIDITY IS THE EMITTER'S CONTRACT (2026-08-16 self-review).
|
||||
*
|
||||
* This copied every byte >= 0x20 through verbatim, so a malformed sequence
|
||||
* anywhere in the store became malformed output. Measured against the live
|
||||
* graph: three nodes carry labels truncated to exactly 80 bytes ending in a
|
||||
* lone 0xE2 — the first byte of an em-dash, cut mid-sequence by some producer
|
||||
* that is NOT this runtime (no 80-byte truncation exists here; the content
|
||||
* itself is 2572 and 2746 bytes). Those three nodes made the ENTIRE 26 MB
|
||||
* /api/nodes/list response undecodable, so a strict parser could not read the
|
||||
* graph at all.
|
||||
*
|
||||
* Fixing only the writer would not have helped: the store already contains the
|
||||
* damage, and it accepts data from importers, other producers and older
|
||||
* binaries. A serializer that promises JSON owes valid UTF-8 regardless of what
|
||||
* it is handed — so validate here, at the boundary that makes the promise.
|
||||
* Invalid bytes become U+FFFD rather than being dropped, so damage stays
|
||||
* visible in the output instead of being silently papered over.
|
||||
*
|
||||
* Well-formed input is byte-identical to before: valid sequences are copied
|
||||
* verbatim, and only structurally invalid ones (bad lead byte, missing or bad
|
||||
* continuation, overlong encoding, UTF-16 surrogate, or > U+10FFFF) are
|
||||
* replaced. */
|
||||
static void jb_emit_escaped(JsonBuf* b, const char* s) {
|
||||
jb_putc(b, '"');
|
||||
for (; *s; s++) {
|
||||
unsigned char c = (unsigned char)*s;
|
||||
const unsigned char* p = (const unsigned char*)s;
|
||||
while (*p) {
|
||||
unsigned char c = *p;
|
||||
switch (c) {
|
||||
case '"': jb_puts(b, "\\\""); break;
|
||||
case '\\': jb_puts(b, "\\\\"); break;
|
||||
case '\b': jb_puts(b, "\\b"); break;
|
||||
case '\f': jb_puts(b, "\\f"); break;
|
||||
case '\n': jb_puts(b, "\\n"); break;
|
||||
case '\r': jb_puts(b, "\\r"); break;
|
||||
case '\t': jb_puts(b, "\\t"); break;
|
||||
default:
|
||||
if (c < 0x20) {
|
||||
char tmp[8];
|
||||
snprintf(tmp, sizeof(tmp), "\\u%04x", c);
|
||||
jb_puts(b, tmp);
|
||||
} else {
|
||||
jb_putc(b, (char)c);
|
||||
}
|
||||
break;
|
||||
case '"': jb_puts(b, "\\\""); p++; continue;
|
||||
case '\\': jb_puts(b, "\\\\"); p++; continue;
|
||||
case '\b': jb_puts(b, "\\b"); p++; continue;
|
||||
case '\f': jb_puts(b, "\\f"); p++; continue;
|
||||
case '\n': jb_puts(b, "\\n"); p++; continue;
|
||||
case '\r': jb_puts(b, "\\r"); p++; continue;
|
||||
case '\t': jb_puts(b, "\\t"); p++; continue;
|
||||
default: break;
|
||||
}
|
||||
if (c < 0x20) {
|
||||
char tmp[8];
|
||||
snprintf(tmp, sizeof(tmp), "\\u%04x", c);
|
||||
jb_puts(b, tmp);
|
||||
p++;
|
||||
continue;
|
||||
}
|
||||
if (c < 0x80) { jb_putc(b, (char)c); p++; continue; }
|
||||
|
||||
/* Multi-byte: validate the whole sequence before emitting any of it. */
|
||||
int len; unsigned int cp;
|
||||
if ((c & 0xE0) == 0xC0) { len = 2; cp = c & 0x1Fu; }
|
||||
else if ((c & 0xF0) == 0xE0) { len = 3; cp = c & 0x0Fu; }
|
||||
else if ((c & 0xF8) == 0xF0) { len = 4; cp = c & 0x07u; }
|
||||
else { jb_puts(b, "\\ufffd"); p++; continue; }
|
||||
|
||||
int ok = 1;
|
||||
for (int i = 1; i < len; i++) {
|
||||
if ((p[i] & 0xC0) != 0x80) { ok = 0; break; } /* also catches NUL */
|
||||
cp = (cp << 6) | (unsigned int)(p[i] & 0x3F);
|
||||
}
|
||||
if (ok) {
|
||||
if (len == 2 && cp < 0x80) ok = 0; /* overlong */
|
||||
else if (len == 3 && cp < 0x800) ok = 0; /* overlong */
|
||||
else if (len == 4 && cp < 0x10000) ok = 0; /* overlong */
|
||||
else if (cp >= 0xD800 && cp <= 0xDFFF) ok = 0; /* UTF-16 surrogate */
|
||||
else if (cp > 0x10FFFF) ok = 0; /* out of range */
|
||||
}
|
||||
if (!ok) { jb_puts(b, "\\ufffd"); p++; continue; }
|
||||
for (int i = 0; i < len; i++) jb_putc(b, (char)p[i]);
|
||||
p += len;
|
||||
}
|
||||
jb_putc(b, '"');
|
||||
}
|
||||
@@ -5517,6 +5620,45 @@ el_val_t str_count(el_val_t sv, el_val_t subv) {
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
/* el_utf8_safe_len — the largest byte length <= max_bytes that does NOT split a
|
||||
* UTF-8 codepoint.
|
||||
*
|
||||
* WHY (2026-08-16 self-review): engram_first_n_chars truncated with a plain
|
||||
* `if (l > n) l = n; memcpy(...)`, i.e. by BYTES despite its name. Any content
|
||||
* carrying a multi-byte character across the 60-byte boundary produced a label
|
||||
* ending in a half codepoint. That label is copied verbatim into every JSON
|
||||
* document containing the node, so a single such node makes the WHOLE response
|
||||
* invalid UTF-8 — /api/nodes/list failed to decode at byte 89261 against the
|
||||
* live store, which breaks any strict parser reading the graph.
|
||||
*
|
||||
* This lives beside str_count_chars rather than in the engram because the rest
|
||||
* of el's string layer is already codepoint-aware (str_count_chars counts
|
||||
* codepoints, str_reverse walks codepoint lengths). Byte-truncation was the
|
||||
* outlier, and the concern is a string concern. Bounded by BYTES, not
|
||||
* codepoints, so existing labels never grow — only stop splitting.
|
||||
*
|
||||
* A lead byte with no room for its full sequence is dropped entirely; a stray
|
||||
* continuation byte (already-invalid input) is passed through unchanged rather
|
||||
* than silently repaired, so this never manufactures data. */
|
||||
size_t el_utf8_safe_len(const char* s, size_t max_bytes) {
|
||||
if (!s) return 0;
|
||||
size_t len = strlen(s);
|
||||
if (len <= max_bytes) return len;
|
||||
size_t i = 0;
|
||||
while (i < max_bytes) {
|
||||
unsigned char c = (unsigned char)s[i];
|
||||
size_t cp_len;
|
||||
if ((c & 0x80) == 0x00) cp_len = 1;
|
||||
else if ((c & 0xE0) == 0xC0) cp_len = 2;
|
||||
else if ((c & 0xF0) == 0xE0) cp_len = 3;
|
||||
else if ((c & 0xF8) == 0xF0) cp_len = 4;
|
||||
else cp_len = 1; /* stray continuation: passthrough */
|
||||
if (i + cp_len > max_bytes) break; /* would split — stop before it */
|
||||
i += cp_len;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
/* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */
|
||||
el_val_t str_count_chars(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
@@ -8017,10 +8159,14 @@ static double engram_decode_score(el_val_t v) {
|
||||
return (double)n;
|
||||
}
|
||||
|
||||
/* Truncate to at most n BYTES without splitting a UTF-8 codepoint. The old
|
||||
* implementation was `if (l > n) l = n;` — a byte cut that could land inside a
|
||||
* multi-byte character and emit a half codepoint into the node's label, which
|
||||
* then propagated into every JSON document containing that node. See
|
||||
* el_utf8_safe_len for the measurement. */
|
||||
static char* engram_first_n_chars(const char* s, size_t n) {
|
||||
if (!s) return el_strdup("");
|
||||
size_t l = strlen(s);
|
||||
if (l > n) l = n;
|
||||
size_t l = el_utf8_safe_len(s, n);
|
||||
char* out = el_strbuf(l);
|
||||
memcpy(out, s, l);
|
||||
out[l] = '\0';
|
||||
|
||||
@@ -666,6 +666,10 @@ el_val_t engram_get_node(el_val_t id);
|
||||
void engram_strengthen(el_val_t node_id);
|
||||
void engram_forget(el_val_t node_id);
|
||||
el_val_t engram_prune_telemetry(el_val_t older_than_ms);
|
||||
/* Largest byte length <= max_bytes that does not split a UTF-8 codepoint.
|
||||
* Bounded by bytes, not codepoints, so truncated strings never grow. */
|
||||
size_t el_utf8_safe_len(const char* s, size_t max_bytes);
|
||||
|
||||
el_val_t engram_node_count(void);
|
||||
/* Attach a Geometry to an existing node, and read the attached width back.
|
||||
* Named for the operation, not the store: a node acquires geometry. This is
|
||||
|
||||
Reference in New Issue
Block a user