diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 81829bd..cb52abd 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -3478,28 +3478,74 @@ static void jb_puts(JsonBuf* b, const char* s) { b->buf[b->len] = '\0'; } +/* UTF-8 VALIDITY IS THE EMITTER'S CONTRACT (2026-08-16 self-review). + * + * This copied every byte >= 0x20 through verbatim, so a malformed sequence + * anywhere in the store became malformed output. Measured against the live + * graph: three nodes carry labels truncated to exactly 80 bytes ending in a + * lone 0xE2 — the first byte of an em-dash, cut mid-sequence by some producer + * that is NOT this runtime (no 80-byte truncation exists here; the content + * itself is 2572 and 2746 bytes). Those three nodes made the ENTIRE 26 MB + * /api/nodes/list response undecodable, so a strict parser could not read the + * graph at all. + * + * Fixing only the writer would not have helped: the store already contains the + * damage, and it accepts data from importers, other producers and older + * binaries. A serializer that promises JSON owes valid UTF-8 regardless of what + * it is handed — so validate here, at the boundary that makes the promise. + * Invalid bytes become U+FFFD rather than being dropped, so damage stays + * visible in the output instead of being silently papered over. + * + * Well-formed input is byte-identical to before: valid sequences are copied + * verbatim, and only structurally invalid ones (bad lead byte, missing or bad + * continuation, overlong encoding, UTF-16 surrogate, or > U+10FFFF) are + * replaced. */ static void jb_emit_escaped(JsonBuf* b, const char* s) { jb_putc(b, '"'); - for (; *s; s++) { - unsigned char c = (unsigned char)*s; + const unsigned char* p = (const unsigned char*)s; + while (*p) { + unsigned char c = *p; switch (c) { - case '"': jb_puts(b, "\\\""); break; - case '\\': jb_puts(b, "\\\\"); break; - case '\b': jb_puts(b, "\\b"); break; - case '\f': jb_puts(b, "\\f"); break; - case '\n': jb_puts(b, "\\n"); break; - case '\r': jb_puts(b, "\\r"); break; - case '\t': jb_puts(b, "\\t"); break; - default: - if (c < 0x20) { - char tmp[8]; - snprintf(tmp, sizeof(tmp), "\\u%04x", c); - jb_puts(b, tmp); - } else { - jb_putc(b, (char)c); - } - break; + case '"': jb_puts(b, "\\\""); p++; continue; + case '\\': jb_puts(b, "\\\\"); p++; continue; + case '\b': jb_puts(b, "\\b"); p++; continue; + case '\f': jb_puts(b, "\\f"); p++; continue; + case '\n': jb_puts(b, "\\n"); p++; continue; + case '\r': jb_puts(b, "\\r"); p++; continue; + case '\t': jb_puts(b, "\\t"); p++; continue; + default: break; } + if (c < 0x20) { + char tmp[8]; + snprintf(tmp, sizeof(tmp), "\\u%04x", c); + jb_puts(b, tmp); + p++; + continue; + } + if (c < 0x80) { jb_putc(b, (char)c); p++; continue; } + + /* Multi-byte: validate the whole sequence before emitting any of it. */ + int len; unsigned int cp; + if ((c & 0xE0) == 0xC0) { len = 2; cp = c & 0x1Fu; } + else if ((c & 0xF0) == 0xE0) { len = 3; cp = c & 0x0Fu; } + else if ((c & 0xF8) == 0xF0) { len = 4; cp = c & 0x07u; } + else { jb_puts(b, "\\ufffd"); p++; continue; } + + int ok = 1; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) { ok = 0; break; } /* also catches NUL */ + cp = (cp << 6) | (unsigned int)(p[i] & 0x3F); + } + if (ok) { + if (len == 2 && cp < 0x80) ok = 0; /* overlong */ + else if (len == 3 && cp < 0x800) ok = 0; /* overlong */ + else if (len == 4 && cp < 0x10000) ok = 0; /* overlong */ + else if (cp >= 0xD800 && cp <= 0xDFFF) ok = 0; /* UTF-16 surrogate */ + else if (cp > 0x10FFFF) ok = 0; /* out of range */ + } + if (!ok) { jb_puts(b, "\\ufffd"); p++; continue; } + for (int i = 0; i < len; i++) jb_putc(b, (char)p[i]); + p += len; } jb_putc(b, '"'); } @@ -5516,6 +5562,45 @@ el_val_t str_count(el_val_t sv, el_val_t subv) { return (el_val_t)count; } +/* el_utf8_safe_len — the largest byte length <= max_bytes that does NOT split a + * UTF-8 codepoint. + * + * WHY (2026-08-16 self-review): engram_first_n_chars truncated with a plain + * `if (l > n) l = n; memcpy(...)`, i.e. by BYTES despite its name. Any content + * carrying a multi-byte character across the 60-byte boundary produced a label + * ending in a half codepoint. That label is copied verbatim into every JSON + * document containing the node, so a single such node makes the WHOLE response + * invalid UTF-8 — /api/nodes/list failed to decode at byte 89261 against the + * live store, which breaks any strict parser reading the graph. + * + * This lives beside str_count_chars rather than in the engram because the rest + * of el's string layer is already codepoint-aware (str_count_chars counts + * codepoints, str_reverse walks codepoint lengths). Byte-truncation was the + * outlier, and the concern is a string concern. Bounded by BYTES, not + * codepoints, so existing labels never grow — only stop splitting. + * + * A lead byte with no room for its full sequence is dropped entirely; a stray + * continuation byte (already-invalid input) is passed through unchanged rather + * than silently repaired, so this never manufactures data. */ +size_t el_utf8_safe_len(const char* s, size_t max_bytes) { + if (!s) return 0; + size_t len = strlen(s); + if (len <= max_bytes) return len; + size_t i = 0; + while (i < max_bytes) { + unsigned char c = (unsigned char)s[i]; + size_t cp_len; + if ((c & 0x80) == 0x00) cp_len = 1; + else if ((c & 0xE0) == 0xC0) cp_len = 2; + else if ((c & 0xF0) == 0xE0) cp_len = 3; + else if ((c & 0xF8) == 0xF0) cp_len = 4; + else cp_len = 1; /* stray continuation: passthrough */ + if (i + cp_len > max_bytes) break; /* would split — stop before it */ + i += cp_len; + } + return i; +} + /* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */ el_val_t str_count_chars(el_val_t sv) { const char* s = EL_CSTR(sv); @@ -7714,10 +7799,14 @@ static double engram_decode_score(el_val_t v) { return (double)n; } +/* Truncate to at most n BYTES without splitting a UTF-8 codepoint. The old + * implementation was `if (l > n) l = n;` — a byte cut that could land inside a + * multi-byte character and emit a half codepoint into the node's label, which + * then propagated into every JSON document containing that node. See + * el_utf8_safe_len for the measurement. */ static char* engram_first_n_chars(const char* s, size_t n) { if (!s) return el_strdup(""); - size_t l = strlen(s); - if (l > n) l = n; + size_t l = el_utf8_safe_len(s, n); char* out = el_strbuf(l); memcpy(out, s, l); out[l] = '\0'; diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 10e337b..66484a5 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -612,6 +612,10 @@ el_val_t engram_get_node(el_val_t id); void engram_strengthen(el_val_t node_id); void engram_forget(el_val_t node_id); el_val_t engram_prune_telemetry(el_val_t older_than_ms); +/* Largest byte length <= max_bytes that does not split a UTF-8 codepoint. + * Bounded by bytes, not codepoints, so truncated strings never grow. */ +size_t el_utf8_safe_len(const char* s, size_t max_bytes); + el_val_t engram_node_count(void); /* Attach geometry to an existing node. `hex` is little-endian float32, * exactly dim*8 hex chars — the encoding realizers already emit. Lets a