runtime: make valid UTF-8 the JSON emitter's contract (#148)

lang/runtime/el_runtime.c

@@ -3479,28 +3479,74 @@ static void jb_puts(JsonBuf* b, const char* s) {
     b->buf[b->len] = '\0';
 }
 
+/* UTF-8 VALIDITY IS THE EMITTER'S CONTRACT (2026-08-16 self-review).
+ *
+ * This copied every byte >= 0x20 through verbatim, so a malformed sequence
+ * anywhere in the store became malformed output. Measured against the live
+ * graph: three nodes carry labels truncated to exactly 80 bytes ending in a
+ * lone 0xE2 — the first byte of an em-dash, cut mid-sequence by some producer
+ * that is NOT this runtime (no 80-byte truncation exists here; the content
+ * itself is 2572 and 2746 bytes). Those three nodes made the ENTIRE 26 MB
+ * /api/nodes/list response undecodable, so a strict parser could not read the
+ * graph at all.
+ *
+ * Fixing only the writer would not have helped: the store already contains the
+ * damage, and it accepts data from importers, other producers and older
+ * binaries. A serializer that promises JSON owes valid UTF-8 regardless of what
+ * it is handed — so validate here, at the boundary that makes the promise.
+ * Invalid bytes become U+FFFD rather than being dropped, so damage stays
+ * visible in the output instead of being silently papered over.
+ *
+ * Well-formed input is byte-identical to before: valid sequences are copied
+ * verbatim, and only structurally invalid ones (bad lead byte, missing or bad
+ * continuation, overlong encoding, UTF-16 surrogate, or > U+10FFFF) are
+ * replaced. */
 static void jb_emit_escaped(JsonBuf* b, const char* s) {
     jb_putc(b, '"');
-    for (; *s; s++) {
-        unsigned char c = (unsigned char)*s;
+    const unsigned char* p = (const unsigned char*)s;
+    while (*p) {
+        unsigned char c = *p;
         switch (c) {
-            case '"':  jb_puts(b, "\\\""); break;
-            case '\\': jb_puts(b, "\\\\"); break;
-            case '\b': jb_puts(b, "\\b");  break;
-            case '\f': jb_puts(b, "\\f");  break;
-            case '\n': jb_puts(b, "\\n");  break;
-            case '\r': jb_puts(b, "\\r");  break;
-            case '\t': jb_puts(b, "\\t");  break;
-            default:
-                if (c < 0x20) {
-                    char tmp[8];
-                    snprintf(tmp, sizeof(tmp), "\\u%04x", c);
-                    jb_puts(b, tmp);
-                } else {
-                    jb_putc(b, (char)c);
-                }
-                break;
+            case '"':  jb_puts(b, "\\\""); p++; continue;
+            case '\\': jb_puts(b, "\\\\"); p++; continue;
+            case '\b': jb_puts(b, "\\b");  p++; continue;
+            case '\f': jb_puts(b, "\\f");  p++; continue;
+            case '\n': jb_puts(b, "\\n");  p++; continue;
+            case '\r': jb_puts(b, "\\r");  p++; continue;
+            case '\t': jb_puts(b, "\\t");  p++; continue;
+            default: break;
         }
+        if (c < 0x20) {
+            char tmp[8];
+            snprintf(tmp, sizeof(tmp), "\\u%04x", c);
+            jb_puts(b, tmp);
+            p++;
+            continue;
+        }
+        if (c < 0x80) { jb_putc(b, (char)c); p++; continue; }
+
+        /* Multi-byte: validate the whole sequence before emitting any of it. */
+        int len; unsigned int cp;
+        if      ((c & 0xE0) == 0xC0) { len = 2; cp = c & 0x1Fu; }
+        else if ((c & 0xF0) == 0xE0) { len = 3; cp = c & 0x0Fu; }
+        else if ((c & 0xF8) == 0xF0) { len = 4; cp = c & 0x07u; }
+        else                         { jb_puts(b, "\\ufffd"); p++; continue; }
+
+        int ok = 1;
+        for (int i = 1; i < len; i++) {
+            if ((p[i] & 0xC0) != 0x80) { ok = 0; break; }   /* also catches NUL */
+            cp = (cp << 6) | (unsigned int)(p[i] & 0x3F);
+        }
+        if (ok) {
+            if      (len == 2 && cp < 0x80)     ok = 0;     /* overlong */
+            else if (len == 3 && cp < 0x800)    ok = 0;     /* overlong */
+            else if (len == 4 && cp < 0x10000)  ok = 0;     /* overlong */
+            else if (cp >= 0xD800 && cp <= 0xDFFF) ok = 0;  /* UTF-16 surrogate */
+            else if (cp > 0x10FFFF)             ok = 0;     /* out of range */
+        }
+        if (!ok) { jb_puts(b, "\\ufffd"); p++; continue; }
+        for (int i = 0; i < len; i++) jb_putc(b, (char)p[i]);
+        p += len;
     }
     jb_putc(b, '"');
 }
@@ -5517,6 +5563,45 @@ el_val_t str_count(el_val_t sv, el_val_t subv) {
     return (el_val_t)count;
 }
 
+/* el_utf8_safe_len — the largest byte length <= max_bytes that does NOT split a
+ * UTF-8 codepoint.
+ *
+ * WHY (2026-08-16 self-review): engram_first_n_chars truncated with a plain
+ * `if (l > n) l = n; memcpy(...)`, i.e. by BYTES despite its name. Any content
+ * carrying a multi-byte character across the 60-byte boundary produced a label
+ * ending in a half codepoint. That label is copied verbatim into every JSON
+ * document containing the node, so a single such node makes the WHOLE response
+ * invalid UTF-8 — /api/nodes/list failed to decode at byte 89261 against the
+ * live store, which breaks any strict parser reading the graph.
+ *
+ * This lives beside str_count_chars rather than in the engram because the rest
+ * of el's string layer is already codepoint-aware (str_count_chars counts
+ * codepoints, str_reverse walks codepoint lengths). Byte-truncation was the
+ * outlier, and the concern is a string concern. Bounded by BYTES, not
+ * codepoints, so existing labels never grow — only stop splitting.
+ *
+ * A lead byte with no room for its full sequence is dropped entirely; a stray
+ * continuation byte (already-invalid input) is passed through unchanged rather
+ * than silently repaired, so this never manufactures data. */
+size_t el_utf8_safe_len(const char* s, size_t max_bytes) {
+    if (!s) return 0;
+    size_t len = strlen(s);
+    if (len <= max_bytes) return len;
+    size_t i = 0;
+    while (i < max_bytes) {
+        unsigned char c = (unsigned char)s[i];
+        size_t cp_len;
+        if      ((c & 0x80) == 0x00) cp_len = 1;
+        else if ((c & 0xE0) == 0xC0) cp_len = 2;
+        else if ((c & 0xF0) == 0xE0) cp_len = 3;
+        else if ((c & 0xF8) == 0xF0) cp_len = 4;
+        else                         cp_len = 1;   /* stray continuation: passthrough */
+        if (i + cp_len > max_bytes) break;         /* would split — stop before it */
+        i += cp_len;
+    }
+    return i;
+}
+
 /* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */
 el_val_t str_count_chars(el_val_t sv) {
     const char* s = EL_CSTR(sv);
@@ -8017,10 +8102,14 @@ static double engram_decode_score(el_val_t v) {
     return (double)n;
 }
 
+/* Truncate to at most n BYTES without splitting a UTF-8 codepoint. The old
+ * implementation was `if (l > n) l = n;` — a byte cut that could land inside a
+ * multi-byte character and emit a half codepoint into the node's label, which
+ * then propagated into every JSON document containing that node. See
+ * el_utf8_safe_len for the measurement. */
 static char* engram_first_n_chars(const char* s, size_t n) {
     if (!s) return el_strdup("");
-    size_t l = strlen(s);
-    if (l > n) l = n;
+    size_t l = el_utf8_safe_len(s, n);
     char* out = el_strbuf(l);
     memcpy(out, s, l);
     out[l] = '\0';

lang/runtime/el_runtime.h

@@ -666,6 +666,10 @@ el_val_t  engram_get_node(el_val_t id);
 void      engram_strengthen(el_val_t node_id);
 void      engram_forget(el_val_t node_id);
 el_val_t  engram_prune_telemetry(el_val_t older_than_ms);
+/* Largest byte length <= max_bytes that does not split a UTF-8 codepoint.
+ * Bounded by bytes, not codepoints, so truncated strings never grow. */
+size_t    el_utf8_safe_len(const char* s, size_t max_bytes);
+
 el_val_t  engram_node_count(void);
 /* Attach a Geometry to an existing node, and read the attached width back.
  * Named for the operation, not the store: a node acquires geometry. This is