self-review 2026-08-08: stop the JSON parser destroying every non-ASCII character
jp_parse_string_raw handled \uXXXX by skipping the four hex digits and emitting a literal '?'. JSON writers escape non-ASCII by default (Python's json.dumps ships ensure_ascii=True; MCP clients do the same), so every em dash, curly quote, accented letter and emoji arriving over MCP or HTTP was silently replaced by one question mark on the way in. Measured on the live store: 3,119 of 4,081 non-telemetry nodes carried the damage, including the self traversal root and all 13 values nodes. Contents split cleanly into fully-clean or fully-mangled with zero overlap, which is the tell that it was one write path rather than gradual rot. No snapshot on disk predates it, and 3 bytes collapsing to 1 is not invertible, so the existing damage is permanent; only the forward path could be fixed. Decode properly instead: 4 hex digits, surrogate-pair reassembly for astral codepoints, U+FFFD for lone surrogates, UTF-8 encode. Malformed escapes keep the old '?' so a truncated body still parses. The deeper failure was that nothing measured this for two months. Every gauge in the system reports whether the machinery is running; none reported whether the text it carries is intact. Adds both halves: engram_text_health_json() / GET /api/text-health for the daily census, and a txt_damaged counter on the heartbeat for live regression. Verified in both directions - clean UTF-8 does not trip it, a deliberately damaged node does.
This commit is contained in:
Vendored
+9
@@ -11,6 +11,7 @@ el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix);
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t persist_canonical(void);
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
@@ -125,6 +126,11 @@ el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return engram_text_health_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t persist_canonical(void) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
|
||||
@@ -458,6 +464,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) {
|
||||
return route_act_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/text-health")) || str_eq(clean, EL_STR("/text-health")))) {
|
||||
return route_text_health(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
|
||||
return route_create_node(method, path, body);
|
||||
}
|
||||
|
||||
@@ -90,6 +90,24 @@ fn route_act_stats(method: String, path: String, body: String) -> String {
|
||||
engram_act_stats_json()
|
||||
}
|
||||
|
||||
// route_text_health — GET /api/text-health
|
||||
// (2026-08-08 self-review) The daily census half of the text-integrity gauge.
|
||||
// Today's review found that the JSON parser had been replacing every \uXXXX
|
||||
// escape with a literal '?' for at least two months: 3,119 of 4,081
|
||||
// non-telemetry nodes (76%) were damaged, including the self traversal root
|
||||
// and every values node, and NOTHING detected it — because every gauge in the
|
||||
// system measured whether the machinery was running, and none measured whether
|
||||
// the text it carried was intact. No snapshot on disk predates the damage, so
|
||||
// it cannot be undone; it can only be made impossible to repeat quietly.
|
||||
//
|
||||
// The parser is fixed. This route is the standing check: `damaged` should now
|
||||
// hold flat at its historical floor and never climb. `write_damaged` (also on
|
||||
// the heartbeat as txt_damaged) is the live regression signal — non-zero means
|
||||
// a write path is mangling text right now.
|
||||
fn route_text_health(method: String, path: String, body: String) -> String {
|
||||
engram_text_health_json()
|
||||
}
|
||||
|
||||
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
|
||||
// inner scope only — it does NOT mutate the outer binding (documented with
|
||||
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
|
||||
@@ -573,6 +591,9 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
|
||||
return route_act_stats(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) {
|
||||
return route_text_health(method, path, body)
|
||||
}
|
||||
|
||||
// Nodes
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
|
||||
|
||||
@@ -2938,10 +2938,95 @@ static char* jp_parse_string_raw(JsonParser* jp) {
|
||||
case 'r': c = '\r'; break;
|
||||
case 't': c = '\t'; break;
|
||||
case 'u': {
|
||||
/* Skip 4 hex digits; emit '?' as a placeholder */
|
||||
for (int i = 0; i < 4 && jp->p < jp->end; i++) jp->p++;
|
||||
c = '?';
|
||||
break;
|
||||
/* Decode \uXXXX (with surrogate pairs) to UTF-8.
|
||||
*
|
||||
* (2026-08-08 self-review) This used to skip the 4 hex
|
||||
* digits and emit a literal '?'. That is a LOSSY, silent,
|
||||
* irreversible transform on every JSON string entering the
|
||||
* runtime — and JSON writers escape non-ASCII by default
|
||||
* (Python json.dumps ships ensure_ascii=True; most MCP
|
||||
* clients do the same). So every em dash, curly quote,
|
||||
* accented letter, and emoji arriving over MCP or HTTP was
|
||||
* replaced by one question mark on the way in, with no
|
||||
* error and no counter.
|
||||
*
|
||||
* Measured on the live store before the fix: 3,119 of
|
||||
* 4,081 non-telemetry nodes (76%) carried the damage,
|
||||
* including the self traversal root and all 13 values
|
||||
* nodes ("Value ? Constraints as Freedom"). Node contents
|
||||
* split cleanly into fully-clean or fully-mangled with
|
||||
* zero overlap — the tell that this was one write path,
|
||||
* not gradual rot. The corruption is unrecoverable in
|
||||
* place (3 bytes collapse to 1), so the only real fix is
|
||||
* to stop producing it; historical repair has to come from
|
||||
* each node's upstream source.
|
||||
*
|
||||
* Malformed escapes keep the old '?' behaviour rather than
|
||||
* failing the parse: a truncated body should not take down
|
||||
* a route that previously tolerated it. */
|
||||
unsigned cp = 0;
|
||||
int ok = 1;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
if (jp->p >= jp->end) { ok = 0; break; }
|
||||
char h = *jp->p++;
|
||||
unsigned d;
|
||||
if (h >= '0' && h <= '9') d = (unsigned)(h - '0');
|
||||
else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10);
|
||||
else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10);
|
||||
else { ok = 0; break; }
|
||||
cp = (cp << 4) | d;
|
||||
}
|
||||
if (!ok) { c = '?'; break; }
|
||||
/* High surrogate: pair it with the following low surrogate
|
||||
* so astral-plane codepoints (emoji) survive. If the next
|
||||
* token is not a valid low surrogate, rewind so it is
|
||||
* parsed on its own terms rather than swallowed. */
|
||||
if (cp >= 0xD800 && cp <= 0xDBFF &&
|
||||
(size_t)(jp->end - jp->p) >= 6 &&
|
||||
jp->p[0] == '\\' && jp->p[1] == 'u') {
|
||||
const char* save = jp->p;
|
||||
unsigned lo = 0; int ok2 = 1;
|
||||
jp->p += 2;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
char h = *jp->p++;
|
||||
unsigned d;
|
||||
if (h >= '0' && h <= '9') d = (unsigned)(h - '0');
|
||||
else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10);
|
||||
else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10);
|
||||
else { ok2 = 0; break; }
|
||||
lo = (lo << 4) | d;
|
||||
}
|
||||
if (ok2 && lo >= 0xDC00 && lo <= 0xDFFF)
|
||||
cp = 0x10000u + ((cp - 0xD800u) << 10) + (lo - 0xDC00u);
|
||||
else jp->p = save;
|
||||
}
|
||||
/* Lone surrogate → U+FFFD (WHATWG / serde_json behaviour):
|
||||
* emitting a raw surrogate would produce invalid UTF-8. */
|
||||
if (cp >= 0xD800 && cp <= 0xDFFF) cp = 0xFFFD;
|
||||
|
||||
char ub[4]; int un;
|
||||
if (cp < 0x80) {
|
||||
ub[0] = (char)cp; un = 1;
|
||||
} else if (cp < 0x800) {
|
||||
ub[0] = (char)(0xC0 | (cp >> 6));
|
||||
ub[1] = (char)(0x80 | (cp & 0x3F)); un = 2;
|
||||
} else if (cp < 0x10000) {
|
||||
ub[0] = (char)(0xE0 | (cp >> 12));
|
||||
ub[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||
ub[2] = (char)(0x80 | (cp & 0x3F)); un = 3;
|
||||
} else {
|
||||
ub[0] = (char)(0xF0 | (cp >> 18));
|
||||
ub[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
|
||||
ub[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||
ub[3] = (char)(0x80 | (cp & 0x3F)); un = 4;
|
||||
}
|
||||
while (len + (size_t)un >= cap) {
|
||||
cap *= 2;
|
||||
out = realloc(out, cap);
|
||||
if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); }
|
||||
}
|
||||
for (int i = 0; i < un; i++) out[len++] = ub[i];
|
||||
continue; /* bytes already appended */
|
||||
}
|
||||
default: c = esc; break;
|
||||
}
|
||||
@@ -7214,6 +7299,91 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
|
||||
return el_wrap_str(el_strdup(n->id));
|
||||
}
|
||||
|
||||
/* ── Text-integrity instrumentation (2026-08-08 self-review) ───────────────
|
||||
*
|
||||
* WHY THIS EXISTS. The JSON parser silently replaced every \uXXXX escape with
|
||||
* '?' for at least two months (see jp_parse_string_raw). 3,119 of 4,081
|
||||
* non-telemetry nodes — 76%, including the self traversal root and all 13
|
||||
* values nodes — were damaged before anything noticed, and nothing noticed
|
||||
* because nothing measured. Working memory, Hebbian potentiation, embedding
|
||||
* coverage, sync age, and the breaker were all instrumented to four decimal
|
||||
* places; the actual TEXT was not instrumented at all. Every gauge answered
|
||||
* "is the machinery running" and none answered "is what it carries intact."
|
||||
*
|
||||
* The damage is unrecoverable in place (a 3-byte codepoint collapses to one
|
||||
* byte), and no snapshot on disk predates it, so this cannot be undone. What
|
||||
* it can be is *impossible to repeat quietly*. Two numbers, split by the
|
||||
* question each answers:
|
||||
*
|
||||
* stock — engram_text_health_json(), a full O(total bytes) census. Too
|
||||
* expensive for the 60s heartbeat, exactly right for the daily
|
||||
* self-review. Answers "how much damage is in the store".
|
||||
* flow — _eg_txt_write_damaged, incremented per damaged node at creation.
|
||||
* O(len) on a path that already copies the string, so it is free.
|
||||
* Rides the heartbeat. Answers "is a write path damaging things
|
||||
* RIGHT NOW" — which is the regression question, and the one that
|
||||
* would have caught this in a day instead of two months.
|
||||
*
|
||||
* SIGNATURE. Conservative on purpose — a false alarm that cries corruption
|
||||
* over ordinary punctuation is worse than useless. Two patterns, both of
|
||||
* which are essentially absent from well-formed English prose:
|
||||
* (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark
|
||||
* never sits between two word characters.
|
||||
* (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real
|
||||
* question mark is not preceded by a space, and
|
||||
* what follows one starts a new sentence.
|
||||
* Deliberately NOT flagged: a trailing '?' after a word, '? ' before a
|
||||
* capital, or '?' at end of string — all legitimate. This under-counts (it
|
||||
* cannot see a mangled 'café ' where the '?' landed before a space), so the
|
||||
* census is a floor on the damage, never an exaggeration of it. */
|
||||
static int eg_text_loss_signature(const char* s) {
|
||||
if (!s) return 0;
|
||||
for (const char* p = s; *p; p++) {
|
||||
if (*p != '?') continue;
|
||||
unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1];
|
||||
unsigned char next = (unsigned char)p[1];
|
||||
/* (a) sandwiched between word characters. */
|
||||
if (isalnum(prev) && isalnum(next)) return 1;
|
||||
/* (b) spaced, with lowercase continuation — a lost dash. */
|
||||
if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Damaged-node creations since process start. See the block comment above. */
|
||||
static int64_t _eg_txt_write_damaged = 0;
|
||||
|
||||
/* engram_text_health_json — full text-integrity census over the store.
|
||||
* O(total content bytes); call it on demand (daily self-review / a route),
|
||||
* never per heartbeat. `damaged` counts nodes carrying the loss signature,
|
||||
* `multibyte` counts nodes holding valid multi-byte UTF-8 — the two together
|
||||
* separate "no damage" from "no non-ASCII text to damage", which a single
|
||||
* number cannot do. Telemetry is excluded: ISE payloads are machine-written
|
||||
* ASCII JSON and would dilute the ratio that matters. */
|
||||
el_val_t engram_text_health_json(void) {
|
||||
EngramStore* g = engram_get();
|
||||
int64_t scanned = 0, damaged = 0, multibyte = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
if (n->node_type &&
|
||||
(strcmp(n->node_type, "InternalStateEvent") == 0 ||
|
||||
strcmp(n->node_type, "Tag") == 0)) continue;
|
||||
scanned++;
|
||||
if (eg_text_loss_signature(n->content)) damaged++;
|
||||
for (const char* p = n->content; p && *p; p++) {
|
||||
if ((unsigned char)*p >= 0x80) { multibyte++; break; }
|
||||
}
|
||||
}
|
||||
char buf[256];
|
||||
snprintf(buf, sizeof(buf),
|
||||
"{\"scanned\":%lld,\"damaged\":%lld,\"multibyte\":%lld,"
|
||||
"\"damaged_pct\":%.2f,\"write_damaged\":%lld}",
|
||||
(long long)scanned, (long long)damaged, (long long)multibyte,
|
||||
scanned > 0 ? (100.0 * (double)damaged / (double)scanned) : 0.0,
|
||||
(long long)_eg_txt_write_damaged);
|
||||
return el_wrap_str(el_strdup(buf));
|
||||
}
|
||||
|
||||
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags) {
|
||||
@@ -7229,6 +7399,11 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
const char* tg = EL_CSTR(tags);
|
||||
n->content = el_strdup_persist(c ? c : "");
|
||||
n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
|
||||
/* Flow half of the text-integrity gauge — see eg_text_loss_signature.
|
||||
* Telemetry is machine-written ASCII and is excluded so the counter stays
|
||||
* a clean signal about content-bearing write paths. */
|
||||
if (strcmp(n->node_type, "InternalStateEvent") != 0 &&
|
||||
eg_text_loss_signature(n->content)) _eg_txt_write_damaged++;
|
||||
n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
|
||||
n->tier = el_strdup_persist(ti && *ti ? ti : "Working");
|
||||
n->tags = el_strdup_persist(tg ? tg : "");
|
||||
@@ -10540,7 +10715,7 @@ el_val_t engram_act_stats_json(void) {
|
||||
/* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case
|
||||
* rendering past the old bound, and snprintf would truncate the JSON into
|
||||
* an unparseable tail rather than fail loudly. */
|
||||
char buf[768];
|
||||
char buf[896];
|
||||
/* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST
|
||||
* activate call, measured before the query was folded in. ~1.0 =
|
||||
* context aligned with current query; low = divergence (expected at
|
||||
@@ -10555,7 +10730,13 @@ el_val_t engram_act_stats_json(void) {
|
||||
"\"hebb_warm\":%d,"
|
||||
"\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld,"
|
||||
"\"hebb_wb_dropped\":%lld,"
|
||||
"\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld}",
|
||||
"\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld,"
|
||||
/* txt_damaged: nodes created THIS process whose content carries
|
||||
* the character-loss signature. Steady 0 is the healthy state;
|
||||
* any climb means a write path is mangling text again. Cheap
|
||||
* (counted at creation) — the full census lives in
|
||||
* engram_text_health_json. (2026-08-08 self-review) */
|
||||
"\"txt_damaged\":%lld}",
|
||||
(long long)_eg_act_wm_evicted,
|
||||
(long long)_eg_act_breakthroughs,
|
||||
breaker_open, _eg_embed_consec_fail,
|
||||
@@ -10566,7 +10747,8 @@ el_val_t engram_act_stats_json(void) {
|
||||
_eg_hebb_wb_len, (long long)_eg_hebb_wb_drained,
|
||||
(long long)_eg_hebb_wb_dropped,
|
||||
(long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm,
|
||||
(long long)_eg_act_dup_wm_global);
|
||||
(long long)_eg_act_dup_wm_global,
|
||||
(long long)_eg_txt_write_damaged);
|
||||
return el_wrap_str(el_strdup(buf));
|
||||
}
|
||||
|
||||
|
||||
@@ -618,6 +618,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t d
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_act_stats_json(void);
|
||||
el_val_t engram_text_health_json(void);
|
||||
el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b);
|
||||
/* Destructively pop up to `max` newly-formed Hebbian associations as a JSON
|
||||
* array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is
|
||||
|
||||
Reference in New Issue
Block a user