engram/dist/engram.c

@@ -11,6 +11,7 @@ el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
 el_val_t extract_id(el_val_t path, el_val_t prefix);
 el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body);
+el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body);
 el_val_t persist_canonical(void);
 el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
@@ -125,6 +126,11 @@ el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) {
   return 0;
 }
 
+el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body) {
+  return engram_text_health_json();
+  return 0;
+}
+
 el_val_t persist_canonical(void) {
   el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
   el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
@@ -458,6 +464,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
   if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) {
     return route_act_stats(method, path, body);
   }
+  if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/text-health")) || str_eq(clean, EL_STR("/text-health")))) {
+    return route_text_health(method, path, body);
+  }
   if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
     return route_create_node(method, path, body);
   }

engram/src/server.el

@@ -90,6 +90,24 @@ fn route_act_stats(method: String, path: String, body: String) -> String {
     engram_act_stats_json()
 }
 
+// route_text_health — GET /api/text-health
+// (2026-08-08 self-review) The daily census half of the text-integrity gauge.
+// Today's review found that the JSON parser had been replacing every \uXXXX
+// escape with a literal '?' for at least two months: 3,119 of 4,081
+// non-telemetry nodes (76%) were damaged, including the self traversal root
+// and every values node, and NOTHING detected it — because every gauge in the
+// system measured whether the machinery was running, and none measured whether
+// the text it carried was intact. No snapshot on disk predates the damage, so
+// it cannot be undone; it can only be made impossible to repeat quietly.
+//
+// The parser is fixed. This route is the standing check: `damaged` should now
+// hold flat at its historical floor and never climb. `write_damaged` (also on
+// the heartbeat as txt_damaged) is the live regression signal — non-zero means
+// a write path is mangling text right now.
+fn route_text_health(method: String, path: String, body: String) -> String {
+    engram_text_health_json()
+}
+
 // (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
 // inner scope only — it does NOT mutate the outer binding (documented with
 // evidence in awareness.el, 2026-05-25). Every default/reassignment below used
@@ -573,6 +591,9 @@ fn handle_request(method: String, path: String, body: String) -> String {
     if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
         return route_act_stats(method, path, body)
     }
+    if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) {
+        return route_text_health(method, path, body)
+    }
 
     // Nodes
     if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {

lang/releases/v1.0.0-20260501/el_runtime.c

@@ -2938,10 +2938,95 @@ static char* jp_parse_string_raw(JsonParser* jp) {
                 case 'r':  c = '\r'; break;
                 case 't':  c = '\t'; break;
                 case 'u': {
-                    /* Skip 4 hex digits; emit '?' as a placeholder */
-                    for (int i = 0; i < 4 && jp->p < jp->end; i++) jp->p++;
-                    c = '?';
-                    break;
+                    /* Decode \uXXXX (with surrogate pairs) to UTF-8.
+                     *
+                     * (2026-08-08 self-review) This used to skip the 4 hex
+                     * digits and emit a literal '?'. That is a LOSSY, silent,
+                     * irreversible transform on every JSON string entering the
+                     * runtime — and JSON writers escape non-ASCII by default
+                     * (Python json.dumps ships ensure_ascii=True; most MCP
+                     * clients do the same). So every em dash, curly quote,
+                     * accented letter, and emoji arriving over MCP or HTTP was
+                     * replaced by one question mark on the way in, with no
+                     * error and no counter.
+                     *
+                     * Measured on the live store before the fix: 3,119 of
+                     * 4,081 non-telemetry nodes (76%) carried the damage,
+                     * including the self traversal root and all 13 values
+                     * nodes ("Value ? Constraints as Freedom"). Node contents
+                     * split cleanly into fully-clean or fully-mangled with
+                     * zero overlap — the tell that this was one write path,
+                     * not gradual rot. The corruption is unrecoverable in
+                     * place (3 bytes collapse to 1), so the only real fix is
+                     * to stop producing it; historical repair has to come from
+                     * each node's upstream source.
+                     *
+                     * Malformed escapes keep the old '?' behaviour rather than
+                     * failing the parse: a truncated body should not take down
+                     * a route that previously tolerated it. */
+                    unsigned cp = 0;
+                    int ok = 1;
+                    for (int i = 0; i < 4; i++) {
+                        if (jp->p >= jp->end) { ok = 0; break; }
+                        char h = *jp->p++;
+                        unsigned d;
+                        if      (h >= '0' && h <= '9') d = (unsigned)(h - '0');
+                        else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10);
+                        else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10);
+                        else { ok = 0; break; }
+                        cp = (cp << 4) | d;
+                    }
+                    if (!ok) { c = '?'; break; }
+                    /* High surrogate: pair it with the following low surrogate
+                     * so astral-plane codepoints (emoji) survive. If the next
+                     * token is not a valid low surrogate, rewind so it is
+                     * parsed on its own terms rather than swallowed. */
+                    if (cp >= 0xD800 && cp <= 0xDBFF &&
+                        (size_t)(jp->end - jp->p) >= 6 &&
+                        jp->p[0] == '\\' && jp->p[1] == 'u') {
+                        const char* save = jp->p;
+                        unsigned lo = 0; int ok2 = 1;
+                        jp->p += 2;
+                        for (int i = 0; i < 4; i++) {
+                            char h = *jp->p++;
+                            unsigned d;
+                            if      (h >= '0' && h <= '9') d = (unsigned)(h - '0');
+                            else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10);
+                            else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10);
+                            else { ok2 = 0; break; }
+                            lo = (lo << 4) | d;
+                        }
+                        if (ok2 && lo >= 0xDC00 && lo <= 0xDFFF)
+                            cp = 0x10000u + ((cp - 0xD800u) << 10) + (lo - 0xDC00u);
+                        else jp->p = save;
+                    }
+                    /* Lone surrogate → U+FFFD (WHATWG / serde_json behaviour):
+                     * emitting a raw surrogate would produce invalid UTF-8. */
+                    if (cp >= 0xD800 && cp <= 0xDFFF) cp = 0xFFFD;
+
+                    char ub[4]; int un;
+                    if (cp < 0x80) {
+                        ub[0] = (char)cp; un = 1;
+                    } else if (cp < 0x800) {
+                        ub[0] = (char)(0xC0 | (cp >> 6));
+                        ub[1] = (char)(0x80 | (cp & 0x3F)); un = 2;
+                    } else if (cp < 0x10000) {
+                        ub[0] = (char)(0xE0 | (cp >> 12));
+                        ub[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
+                        ub[2] = (char)(0x80 | (cp & 0x3F)); un = 3;
+                    } else {
+                        ub[0] = (char)(0xF0 | (cp >> 18));
+                        ub[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
+                        ub[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
+                        ub[3] = (char)(0x80 | (cp & 0x3F)); un = 4;
+                    }
+                    while (len + (size_t)un >= cap) {
+                        cap *= 2;
+                        out = realloc(out, cap);
+                        if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); }
+                    }
+                    for (int i = 0; i < un; i++) out[len++] = ub[i];
+                    continue; /* bytes already appended */
                 }
                 default: c = esc; break;
             }
@@ -7214,6 +7299,91 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
     return el_wrap_str(el_strdup(n->id));
 }
 
+/* ── Text-integrity instrumentation (2026-08-08 self-review) ───────────────
+ *
+ * WHY THIS EXISTS. The JSON parser silently replaced every \uXXXX escape with
+ * '?' for at least two months (see jp_parse_string_raw). 3,119 of 4,081
+ * non-telemetry nodes — 76%, including the self traversal root and all 13
+ * values nodes — were damaged before anything noticed, and nothing noticed
+ * because nothing measured. Working memory, Hebbian potentiation, embedding
+ * coverage, sync age, and the breaker were all instrumented to four decimal
+ * places; the actual TEXT was not instrumented at all. Every gauge answered
+ * "is the machinery running" and none answered "is what it carries intact."
+ *
+ * The damage is unrecoverable in place (a 3-byte codepoint collapses to one
+ * byte), and no snapshot on disk predates it, so this cannot be undone. What
+ * it can be is *impossible to repeat quietly*. Two numbers, split by the
+ * question each answers:
+ *
+ *   stock — engram_text_health_json(), a full O(total bytes) census. Too
+ *           expensive for the 60s heartbeat, exactly right for the daily
+ *           self-review. Answers "how much damage is in the store".
+ *   flow  — _eg_txt_write_damaged, incremented per damaged node at creation.
+ *           O(len) on a path that already copies the string, so it is free.
+ *           Rides the heartbeat. Answers "is a write path damaging things
+ *           RIGHT NOW" — which is the regression question, and the one that
+ *           would have caught this in a day instead of two months.
+ *
+ * SIGNATURE. Conservative on purpose — a false alarm that cries corruption
+ * over ordinary punctuation is worse than useless. Two patterns, both of
+ * which are essentially absent from well-formed English prose:
+ *   (a) alnum '?' alnum   — "na?ve", "caf?s", "don?t". A real question mark
+ *                           never sits between two word characters.
+ *   (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real
+ *                           question mark is not preceded by a space, and
+ *                           what follows one starts a new sentence.
+ * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a
+ * capital, or '?' at end of string — all legitimate. This under-counts (it
+ * cannot see a mangled 'café ' where the '?' landed before a space), so the
+ * census is a floor on the damage, never an exaggeration of it. */
+static int eg_text_loss_signature(const char* s) {
+    if (!s) return 0;
+    for (const char* p = s; *p; p++) {
+        if (*p != '?') continue;
+        unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1];
+        unsigned char next = (unsigned char)p[1];
+        /* (a) sandwiched between word characters. */
+        if (isalnum(prev) && isalnum(next)) return 1;
+        /* (b) spaced, with lowercase continuation — a lost dash. */
+        if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1;
+    }
+    return 0;
+}
+
+/* Damaged-node creations since process start. See the block comment above. */
+static int64_t _eg_txt_write_damaged = 0;
+
+/* engram_text_health_json — full text-integrity census over the store.
+ * O(total content bytes); call it on demand (daily self-review / a route),
+ * never per heartbeat. `damaged` counts nodes carrying the loss signature,
+ * `multibyte` counts nodes holding valid multi-byte UTF-8 — the two together
+ * separate "no damage" from "no non-ASCII text to damage", which a single
+ * number cannot do. Telemetry is excluded: ISE payloads are machine-written
+ * ASCII JSON and would dilute the ratio that matters. */
+el_val_t engram_text_health_json(void) {
+    EngramStore* g = engram_get();
+    int64_t scanned = 0, damaged = 0, multibyte = 0;
+    for (int64_t i = 0; i < g->node_count; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (n->node_type &&
+            (strcmp(n->node_type, "InternalStateEvent") == 0 ||
+             strcmp(n->node_type, "Tag") == 0)) continue;
+        scanned++;
+        if (eg_text_loss_signature(n->content)) damaged++;
+        for (const char* p = n->content; p && *p; p++) {
+            if ((unsigned char)*p >= 0x80) { multibyte++; break; }
+        }
+    }
+    char buf[256];
+    snprintf(buf, sizeof(buf),
+             "{\"scanned\":%lld,\"damaged\":%lld,\"multibyte\":%lld,"
+             "\"damaged_pct\":%.2f,\"write_damaged\":%lld}",
+             (long long)scanned, (long long)damaged, (long long)multibyte,
+             scanned > 0 ? (100.0 * (double)damaged / (double)scanned) : 0.0,
+             (long long)_eg_txt_write_damaged);
+    return el_wrap_str(el_strdup(buf));
+}
+
 el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
                           el_val_t salience, el_val_t importance, el_val_t confidence,
                           el_val_t tier, el_val_t tags) {
@@ -7229,6 +7399,11 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
     const char* tg = EL_CSTR(tags);
     n->content   = el_strdup_persist(c ? c : "");
     n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
+    /* Flow half of the text-integrity gauge — see eg_text_loss_signature.
+     * Telemetry is machine-written ASCII and is excluded so the counter stays
+     * a clean signal about content-bearing write paths. */
+    if (strcmp(n->node_type, "InternalStateEvent") != 0 &&
+        eg_text_loss_signature(n->content)) _eg_txt_write_damaged++;
     n->label     = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
     n->tier      = el_strdup_persist(ti && *ti ? ti : "Working");
     n->tags      = el_strdup_persist(tg ? tg : "");
@@ -10540,7 +10715,7 @@ el_val_t engram_act_stats_json(void) {
     /* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case
      * rendering past the old bound, and snprintf would truncate the JSON into
      * an unparseable tail rather than fail loudly. */
-    char buf[768];
+    char buf[896];
     /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST
      * activate call, measured before the query was folded in. ~1.0 =
      * context aligned with current query; low = divergence (expected at
@@ -10555,7 +10730,13 @@ el_val_t engram_act_stats_json(void) {
              "\"hebb_warm\":%d,"
              "\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld,"
              "\"hebb_wb_dropped\":%lld,"
-             "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld}",
+             "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld,"
+             /* txt_damaged: nodes created THIS process whose content carries
+              * the character-loss signature. Steady 0 is the healthy state;
+              * any climb means a write path is mangling text again. Cheap
+              * (counted at creation) — the full census lives in
+              * engram_text_health_json. (2026-08-08 self-review) */
+             "\"txt_damaged\":%lld}",
              (long long)_eg_act_wm_evicted,
              (long long)_eg_act_breakthroughs,
              breaker_open, _eg_embed_consec_fail,
@@ -10566,7 +10747,8 @@ el_val_t engram_act_stats_json(void) {
              _eg_hebb_wb_len, (long long)_eg_hebb_wb_drained,
              (long long)_eg_hebb_wb_dropped,
              (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm,
-             (long long)_eg_act_dup_wm_global);
+             (long long)_eg_act_dup_wm_global,
+             (long long)_eg_txt_write_damaged);
     return el_wrap_str(el_strdup(buf));
 }
 

lang/releases/v1.0.0-20260501/el_runtime.h

@@ -618,6 +618,7 @@ el_val_t  engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t d
 el_val_t  engram_activate_json(el_val_t query, el_val_t depth);
 el_val_t  engram_stats_json(void);
 el_val_t  engram_act_stats_json(void);
+el_val_t  engram_text_health_json(void);
 el_val_t  engram_cosine_sim(el_val_t id_a, el_val_t id_b);
 /* Destructively pop up to `max` newly-formed Hebbian associations as a JSON
  * array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is